Skip to content

M
models

PaddlePaddle / models
大约 2 年 前同步成功

通知 232
Star 6828
Fork 2962
M
models
提交 2bb4e4a6 编写于 作者: Y Yancey1989
浏览文件
操作

polish code

上级 2e110d79
隐藏空白更改
内联 并排
Showing with 173 addition and 236 deletion
fluid/PaddleCV/image_classification/fast_resnet/torchvision_reader.py
浏览文件 @ 2bb4e4a6
......@@ -3,33 +3,30 @@ import os
import numpy as np
import math
import random
import torchvision
import torch
import torch.utils.data
from torch.utils.data.distributed import DistributedSampler
import torchvision.transforms as transforms
import torchvision.datasets as datasets
from torch.utils.data.sampler import Sampler
import torchvision
import pickle
from tqdm import tqdm
import time
import multiprocessing
TRAINER_NUMS = int(os.getenv("PADDLE_TRAINERS_NUM", "1"))
TRAINER_ID = int(os.getenv("PADDLE_TRAINER_ID", "0"))
FINISH_EVENT = "FINISH_EVENT"
class PaddleDataLoader(object):
def __init__(self, torch_dataset, indices=None, concurrent=16, queue_size=3072, shuffle=True, batch_size=224, is_distributed=True):
def __init__(self, torch_dataset, indices=None, concurrent=24, queue_size=3072, shuffle=True):
self.torch_dataset = torch_dataset
self.data_queue = multiprocessing.Queue(queue_size)
self.indices = indices
self.concurrent = concurrent
self.shuffle_seed = 0
self.shuffle = shuffle
self.is_distributed = is_distributed
self.batch_size = batch_size
def _worker_loop(self, dataset, worker_indices, worker_id):
cnt = 0
print("worker [%d], len: [%d], indices: [%s]"%(worker_id, len(worker_indices), worker_indices[:10]))
for idx in worker_indices:
cnt += 1
img, label = self.torch_dataset[idx]
......@@ -43,28 +40,21 @@ class PaddleDataLoader(object):
worker_processes = []
total_img = len(self.torch_dataset)
print("total image: ", total_img)
if self.indices is None:
self.indices = [i for i in xrange(total_img)]
#if self.indices is None:
if self.shuffle:
random.seed(self.shuffle_seed)
self.indices = [i for i in xrange(total_img)]
random.seed(time.time())
random.shuffle(self.indices)
worker_indices = self.indices
if self.is_distributed:
cnt_per_node = len(self.indices) / TRAINER_NUMS
offset = TRAINER_ID * cnt_per_node
worker_indices = self.indices[offset: (offset + cnt_per_node)]
if len(worker_indices) % self.batch_size != 0:
worker_indices += worker_indices[-(self.batch_size - (len(worker_indices) % self.batch_size)):]
print("shuffle: [%d], shuffle seed: [%d], worker indices len: [%d], %s" % (self.shuffle, self.shuffle_seed, len(worker_indices), worker_indices[:10]))
cnt_per_thread = int(math.ceil(len(worker_indices) / self.concurrent))
print("shuffle indices: %s ..." % self.indices[:10])
imgs_per_worker = int(math.ceil(total_img / self.concurrent))
for i in xrange(self.concurrent):
offset = i * cnt_per_thread
thread_incides = worker_indices[offset: (offset + cnt_per_thread)]
print("loader thread: [%d] start idx: [%d], end idx: [%d], len: [%d]" % (i, offset, (offset + cnt_per_thread), len(thread_incides)))
start = i * imgs_per_worker
end = (i + 1) * imgs_per_worker if i != self.concurrent - 1 else None
sliced_indices = self.indices[start:end]
w = multiprocessing.Process(
target=self._worker_loop,
args=(self.torch_dataset, thread_incides, i)
args=(self.torch_dataset, sliced_indices, i)
)
w.daemon = True
w.start()
......@@ -80,13 +70,13 @@ class PaddleDataLoader(object):
return _reader_creator
def train(traindir, bs, sz, min_scale=0.08):
def train(traindir, sz, min_scale=0.08):
train_tfms = [
transforms.RandomResizedCrop(sz, scale=(min_scale, 1.0)),
transforms.RandomHorizontalFlip()
]
train_dataset = datasets.ImageFolder(traindir, transforms.Compose(train_tfms))
return PaddleDataLoader(train_dataset, batch_size=bs)
return PaddleDataLoader(train_dataset).reader()
def test(valdir, bs, sz, rect_val=False):
if rect_val:
......@@ -96,12 +86,12 @@ def test(valdir, bs, sz, rect_val=False):
ar_tfms = [transforms.Resize(int(sz* 1.14)), CropArTfm(idx2ar, sz)]
val_dataset = ValDataset(valdir, transform=ar_tfms)
return PaddleDataLoader(val_dataset, concurrent=1, indices=idx_sorted, shuffle=False, is_distributed=False)
return PaddleDataLoader(val_dataset, concurrent=1, indices=idx_sorted, shuffle=False).reader()
val_tfms = [transforms.Resize(int(sz* 1.14)), transforms.CenterCrop(sz)]
val_dataset = datasets.ImageFolder(valdir, transforms.Compose(val_tfms))
return PaddleDataLoader(val_dataset, is_distributed=False)
return PaddleDataLoader(val_dataset).reader()
class ValDataset(datasets.ImageFolder):
......@@ -122,6 +112,7 @@ class ValDataset(datasets.ImageFolder):
return sample, target
class CropArTfm(object):
def __init__(self, idx2ar, target_size):
self.idx2ar, self.target_size = idx2ar, target_size
......@@ -134,7 +125,7 @@ class CropArTfm(object):
else:
h = int(self.target_size * target_ar)
size = (self.target_size, h // 8 * 8)
return transforms.functional.center_crop(img, size)
return torchvision.transforms.functional.center_crop(img, size)
def sort_ar(valdir):
......@@ -166,5 +157,15 @@ def map_idx2ar(idx_ar_sorted, batch_size):
return idx2ar
if __name__ == "__main__":
reader = test("/work/fast_resnet_data", 64, 128).reader()
print(next(reader()))
\ No newline at end of file
#ds, sampler = create_validation_set("/data/imagenet/validation", 128, 288, True, True)
#for item in sampler:
# for idx in item:
# ds[idx]
import time
test_reader = test(valdir="/data/imagenet/validation", bs=64, sz=288, rect_val=True)
start_ts = time.time()
for idx, data in enumerate(test_reader()):
print(idx, data[0], data[0].shape, data[1])
if idx == 2:
break
\ No newline at end of file
fluid/PaddleCV/image_classification/fast_resnet/train.py
浏览文件 @ 2bb4e4a6
......@@ -19,9 +19,8 @@ import os
import traceback
import numpy as np
import torch
import torchvision_reader
import torch
import paddle
import paddle.fluid as fluid
import paddle.fluid.core as core
......@@ -32,25 +31,9 @@ import sys
sys.path.append("..")
from utility import add_arguments, print_arguments
import functools
import models
from models.fast_resnet import FastResNet, lr_decay
import utils
from env import dist_env
import reader as imagenet_reader
def is_mp_mode():
return True if os.getenv("FLAGS_selected_gpus") else False
def nccl2_prepare(args, startup_prog):
config = fluid.DistributeTranspilerConfig()
config.mode = "nccl2"
t = fluid.DistributeTranspiler(config=config)
envs = args.dist_env
t.transpile(envs["trainer_id"],
trainers=','.join(envs["trainer_endpoints"]),
current_endpoint=envs["current_endpoint"],
startup_program=startup_prog)
def parse_args():
parser = argparse.ArgumentParser(description=__doc__)
......@@ -62,7 +45,7 @@ def parse_args():
add_arg('model_save_dir', str, "output", "model save directory")
add_arg('pretrained_model', str, None, "Whether to use pretrained model.")
add_arg('checkpoint', str, None, "Whether to resume checkpoint.")
add_arg('lr', float, 0.1, "set learning rate.")
add_arg('lr', float, 1.0, "set learning rate.")
add_arg('lr_strategy', str, "piecewise_decay", "Set the learning rate decay strategy.")
add_arg('model', str, "FastResNet", "Set the network to use.")
add_arg('data_dir', str, "./data/ILSVRC2012", "The ImageNet dataset root dir.")
......@@ -89,83 +72,7 @@ def get_device_num():
['nvidia-smi', '-L']).decode().count('\n')
return device_num
def linear_lr_decay(lr_values, epochs, bs_values, total_images):
from paddle.fluid.layers.learning_rate_scheduler import _decay_step_counter
import paddle.fluid.layers.tensor as tensor
import math
with paddle.fluid.default_main_program()._lr_schedule_guard():
global_step = _decay_step_counter()
lr = tensor.create_global_var(
shape=[1],
value=0.0,
dtype='float32',
persistable=True,
name="learning_rate")
with fluid.layers.control_flow.Switch() as switch:
last_steps = 0
for idx, epoch_bound in enumerate(epochs):
start_epoch, end_epoch = epoch_bound
linear_epoch = end_epoch - start_epoch
start_lr, end_lr = lr_values[idx]
linear_lr = end_lr - start_lr
steps = last_steps + linear_epoch * total_images / bs_values[idx] + 1
with switch.case(global_step < steps):
decayed_lr = start_lr + linear_lr * ((global_step - last_steps)* 1.0/(steps - last_steps))
last_steps = steps
fluid.layers.tensor.assign(decayed_lr, lr)
last_value_var = tensor.fill_constant(
shape=[1],
dtype='float32',
value=float(lr_values[-1]))
with switch.default():
fluid.layers.tensor.assign(last_value_var, lr)
return lr
def linear_lr_decay_by_epoch(lr_values, epochs, bs_values, total_images):
from paddle.fluid.layers.learning_rate_scheduler import _decay_step_counter
import paddle.fluid.layers.tensor as tensor
import math
with paddle.fluid.default_main_program()._lr_schedule_guard():
global_step = _decay_step_counter()
lr = tensor.create_global_var(
shape=[1],
value=0.0,
dtype='float32',
persistable=True,
name="learning_rate")
with fluid.layers.control_flow.Switch() as switch:
last_steps = 0
for idx, epoch_bound in enumerate(epochs):
start_epoch, end_epoch = epoch_bound
linear_epoch = end_epoch - start_epoch
start_lr, end_lr = lr_values[idx]
linear_lr = end_lr - start_lr
for epoch_step in xrange(linear_epoch):
steps = last_steps + (1 + epoch_step) * total_images / bs_values[idx]
boundary_val = tensor.fill_constant(
shape=[1],
dtype='float32',
value=float(steps),
force_cpu=True)
decayed_lr = start_lr + epoch_step * linear_lr * 1.0 / linear_epoch
with switch.case(global_step < boundary_val):
value_var = tensor.fill_constant(shape=[1], dtype='float32', value=float(decayed_lr))
print("steps: [%d], epoch : [%d], decayed_lr: [%f]" % (steps, start_epoch + epoch_step, decayed_lr))
fluid.layers.tensor.assign(value_var, lr)
last_steps = steps
last_value_var = tensor.fill_constant(
shape=[1],
dtype='float32',
value=float(lr_values[-1]))
with switch.default():
fluid.layers.tensor.assign(last_value_var, lr)
return lr
DEVICE_NUM = get_device_num()
def test_parallel(exe, test_args, args, test_prog, feeder, bs):
acc_evaluators = []
......@@ -183,69 +90,51 @@ def test_parallel(exe, test_args, args, test_prog, feeder, bs):
for i, e in enumerate(acc_evaluators):
e.update(
value=np.array(acc_rets[i]), weight=bs)
num_samples = batch_id * bs * get_device_num()
print_train_time(start_ts, time.time(), num_samples, "Test")
num_samples = batch_id * bs * DEVICE_NUM
print_train_time(start_ts, time.time(), num_samples)
return [e.eval() for e in acc_evaluators]
def test_single(exe, test_args, args, test_prog, feeder, bs):
test_reader = test_args[3]
to_fetch = [v.name for v in test_args[2]]
acc1 = fluid.metrics.Accuracy()
acc5 = fluid.metrics.Accuracy()
start_ts = time.time()
for batch_id, data in enumerate(test_reader()):
batch_size = len(data[0])
acc_rets = exe.run(test_prog, fetch_list=to_fetch, feed=feeder.feed(data))
acc1.update(value=np.array(acc_rets[0]), weight=batch_size)
acc5.update(value=np.array(acc_rets[1]), weight=batch_size)
if batch_id % 30 == 0:
print("Test batch: [%d], acc_rets: [%s]" % (batch_id, acc_rets))
num_samples = batch_id * bs
print_train_time(start_ts, time.time(), num_samples, "Test")
return np.mean(acc1.eval()), np.mean(acc5.eval())
def build_program(args, is_train, main_prog, startup_prog, py_reader_startup_prog, img_size, trn_dir, batch_size, min_scale, rect_val):
dataloader = None
def build_program(args, is_train, main_prog, startup_prog, py_reader_startup_prog, sz, trn_dir, bs, min_scale, rect_val=False):
dshape=[3, sz, sz]
class_dim=1000
if is_train:
dataloader = torchvision_reader.train(traindir=os.path.join(args.data_dir, trn_dir, "train"), bs=batch_size if is_mp_mode() else batch_size * get_device_num(), sz=img_size, min_scale=min_scale)
reader = torchvision_reader.train(
traindir="/data/imagenet/%strain" % trn_dir, sz=sz, min_scale=min_scale)
else:
dataloader = torchvision_reader.test(valdir=os.path.join(args.data_dir, trn_dir, "validation"), bs=batch_size if is_mp_mode() else batch_size * get_device_num(), sz=img_size, rect_val=rect_val)
dshape = [3, img_size, img_size]
class_dim = 1000
reader = torchvision_reader.test(
valdir="/data/imagenet/%svalidation" % trn_dir, bs=bs*DEVICE_NUM, sz=sz, rect_val=rect_val)
pyreader = None
batched_reader = None
model_name = args.model
model_list = [m for m in dir(models) if "__" not in m]
assert model_name in model_list, "{} is not in lists: {}".format(args.model,
model_list)
model = models.__dict__[model_name]()
with fluid.program_guard(main_prog, startup_prog):
with fluid.unique_name.guard():
if is_train:
with fluid.program_guard(main_prog, py_reader_startup_prog):
with fluid.unique_name.guard():
pyreader = fluid.layers.py_reader(
capacity=batch_size if is_mp_mode() else batch_size * get_device_num(),
capacity=bs * DEVICE_NUM,
shapes=([-1] + dshape, (-1, 1)),
dtypes=('uint8', 'int64'),
name="train_reader_" + str(img_size) if is_train else "test_reader_" + str(img_size),
name="train_reader_" + str(sz) if is_train else "test_reader_" + str(sz),
use_double_buffer=True)
input, label = fluid.layers.read_file(pyreader)
pyreader.decorate_paddle_reader(paddle.batch(reader, batch_size=bs))
else:
input = fluid.layers.data(name="image", shape=[3, 244, 244], dtype="uint8")
label = fluid.layers.data(name="label", shape=[1], dtype="int64")
batched_reader = paddle.batch(reader, batch_size=bs * DEVICE_NUM)
cast_img_type = "float16" if args.fp16 else "float32"
cast = fluid.layers.cast(input, cast_img_type)
img_mean = fluid.layers.create_global_var([3, 1, 1], 0.0, cast_img_type, name="img_mean", persistable=True)
img_std = fluid.layers.create_global_var([3, 1, 1], 0.0, cast_img_type, name="img_std", persistable=True)
#image = (image - (mean * 255.0)) / (std * 255.0)
# image = (image - (mean * 255.0)) / (std * 255.0)
t1 = fluid.layers.elementwise_sub(cast, img_mean, axis=1)
t2 = fluid.layers.elementwise_div(t1, img_std, axis=1)
predict = model.net(t2, class_dim=class_dim, img_size=img_size, is_train=is_train)
model = FastResNet(is_train=is_train)
predict = model.net(t2, class_dim=class_dim, img_size=sz)
cost, pred = fluid.layers.softmax_with_cross_entropy(predict, label, return_softmax=True)
if args.scale_loss > 1:
avg_cost = fluid.layers.mean(x=cost) * float(args.scale_loss)
......@@ -258,15 +147,19 @@ def build_program(args, is_train, main_prog, startup_prog, py_reader_startup_pro
# configure optimize
optimizer = None
if is_train:
total_images = args.total_images
lr = args.lr
epochs = [(0,7), (7,13), (13, 22), (22, 25), (25, 28)]
bs_epoch = [x if is_mp_mode() else x * get_device_num() for x in [224, 224, 96, 96, 50]]
lrs = [(1.0, 2.0), (2.0, 0.25), (0.42857142857142855, 0.04285714285714286), (0.04285714285714286, 0.004285714285714286), (0.0022321428571428575, 0.00022321428571428573), 0.00022321428571428573]
images_per_worker = args.total_images / get_device_num() if is_mp_mode() else args.total_images
bs_epoch = [bs*DEVICE_NUM for bs in [224, 224, 96, 96, 50]]
bs_scale = [bs*1.0 / bs_epoch[0] for bs in bs_epoch]
lrs = [(lr, lr*2), (lr*2, lr/4), (lr*bs_scale[2], lr/10*bs_scale[2]), (lr/10*bs_scale[2], lr/100*bs_scale[2]), (lr/100*bs_scale[4], lr/1000*bs_scale[4]), lr/1000*bs_scale[4]]
boundaries, values = lr_decay(lrs, epochs, bs_epoch, total_images)
optimizer = fluid.optimizer.Momentum(
learning_rate=linear_lr_decay_by_epoch(lrs, epochs, bs_epoch, images_per_worker),
learning_rate=fluid.layers.piecewise_decay(boundaries=boundaries, values=values),
momentum=0.9)
#regularization=fluid.regularizer.L2Decay(1e-4))
if args.fp16:
params_grads = optimizer.backward(avg_cost)
master_params_grads = utils.create_master_params_grads(
......@@ -276,41 +169,32 @@ def build_program(args, is_train, main_prog, startup_prog, py_reader_startup_pro
else:
optimizer.minimize(avg_cost)
if args.memory_optimize:
fluid.memory_optimize(main_prog, skip_grads=True)
if is_train:
pyreader.decorate_paddle_reader(paddle.batch(dataloader.reader(), batch_size=batch_size, drop_last=True))
else:
batched_reader = paddle.batch(dataloader.reader(), batch_size=batch_size if is_mp_mode() else batch_size * get_device_num(), drop_last=True)
if args.memory_optimize:
fluid.memory_optimize(main_prog, skip_grads=True)
return avg_cost, optimizer, [batch_acc1,
batch_acc5], batched_reader, pyreader, py_reader_startup_prog, dataloader
batch_acc5], batched_reader, pyreader, py_reader_startup_prog
def refresh_program(args, epoch, sz, trn_dir, bs, val_bs, need_update_start_prog=False, min_scale=0.08, rect_val=False):
print('program changed: epoch: [%d], image size: [%d], trn_dir: [%s], batch_size:[%d]' % (epoch, sz, trn_dir, bs))
print('refresh program: epoch: [%d], image size: [%d], trn_dir: [%s], batch_size:[%d]' % (epoch, sz, trn_dir, bs))
train_prog = fluid.Program()
test_prog = fluid.Program()
startup_prog = fluid.Program()
py_reader_startup_prog = fluid.Program()
num_trainers = args.dist_env["num_trainers"]
trainer_id = args.dist_env["trainer_id"]
train_args = build_program(args, True, train_prog, startup_prog, py_reader_startup_prog, sz, trn_dir, bs, min_scale, False)
test_args = build_program(args, False, test_prog, startup_prog, py_reader_startup_prog, sz, trn_dir, val_bs, min_scale, rect_val)
gpu_id = int(os.getenv("FLAGS_selected_gpus")) if is_mp_mode() else 0
place = core.CUDAPlace(gpu_id)
train_args = build_program(args, True, train_prog, startup_prog, py_reader_startup_prog, sz, trn_dir, bs, min_scale)
test_args = build_program(args, False, test_prog, startup_prog, py_reader_startup_prog, sz, trn_dir, val_bs, min_scale, rect_val=rect_val)
place = core.CUDAPlace(0)
startup_exe = fluid.Executor(place)
print("execute py_reader startup program")
startup_exe.run(py_reader_startup_prog)
if need_update_start_prog:
print("execute startup program")
if is_mp_mode():
nccl2_prepare(args, startup_prog)
startup_exe.run(startup_prog)
conv2d_w_vars = [var for var in startup_prog.global_block().vars.values() if var.name.startswith('conv2d_')]
for var in conv2d_w_vars:
torch_w = torch.empty(var.shape)
#print("initialize %s, shape: %s, with kaiming normalization." % (var.name, var.shape))
kaiming_np = torch.nn.init.kaiming_normal_(torch_w, mode='fan_out', nonlinearity='relu').numpy()
tensor = fluid.global_scope().find_var(var.name).get_tensor()
if args.fp16:
......@@ -328,28 +212,24 @@ def refresh_program(args, epoch, sz, trn_dir, bs, val_bs, need_update_start_prog
else:
var.get_tensor().set(np_tensor, place)
strategy = fluid.ExecutionStrategy()
strategy.num_threads = args.num_threads
strategy.allow_op_delay = False
strategy.num_iteration_per_drop_scope = 1
build_strategy = fluid.BuildStrategy()
build_strategy.reduce_strategy = fluid.BuildStrategy().ReduceStrategy.AllReduce
avg_loss = train_args[0]
train_exe = fluid.ParallelExecutor(
True,
avg_loss.name,
main_program=train_prog,
exec_strategy=strategy,
build_strategy=build_strategy,
num_trainers=num_trainers,
trainer_id=trainer_id)
build_strategy=build_strategy)
test_exe = fluid.ParallelExecutor(
True, main_program=test_prog, share_vars_from=train_exe)
#return train_args, test_args, test_prog, train_exe, test_exe
return train_args, test_args, test_prog, train_exe, test_exe
# NOTE: only need to benchmark using parallelexe
......@@ -363,18 +243,17 @@ def train_parallel(args):
test_args = None
bs = 224
val_bs = 64
for pass_id in range(args.num_epochs):
for epoch_id in range(args.num_epochs):
# program changed
if pass_id == 0:
train_args, test_args, test_prog, exe, test_exe = refresh_program(args, pass_id, sz=128, trn_dir="sz/160/", bs=bs, val_bs=val_bs, need_update_start_prog=True)
elif pass_id == 13: #13
if epoch_id == 0:
train_args, test_args, test_prog, exe, test_exe = refresh_program(args, epoch_id, sz=128, trn_dir="sz/160/", bs=bs, val_bs=val_bs, need_update_start_prog=True)
elif epoch_id == 13: #13
bs = 96
val_bs = 32
train_args, test_args, test_prog, exe, test_exe = refresh_program(args, pass_id, sz=224, trn_dir="sz/352/", bs=bs, val_bs=val_bs, min_scale=0.087)
elif pass_id == 25: #25
train_args, test_args, test_prog, exe, test_exe = refresh_program(args, epoch_id, sz=224, trn_dir="sz/352/", bs=bs, val_bs=val_bs, min_scale=0.087)
elif epoch_id == 25: #25
bs = 50
val_bs=4
train_args, test_args, test_prog, exe, test_exe = refresh_program(args, pass_id, sz=288, trn_dir="", bs=bs, val_bs=val_bs, min_scale=0.5, rect_val=True)
val_bs=8
train_args, test_args, test_prog, exe, test_exe = refresh_program(args, epoch_id, sz=288, trn_dir="", bs=bs, val_bs=val_bs, min_scale=0.5, rect_val=True)
else:
pass
......@@ -382,17 +261,14 @@ def train_parallel(args):
num_samples = 0
iters = 0
start_time = time.time()
train_dataloader = train_args[6] # Paddle DataLoader
train_dataloader.shuffle_seed = pass_id + 1
train_args[4].start() # start pyreader
batch_time_start = time.time()
samples_per_step = bs if is_mp_mode() else bs * get_device_num()
batch_start_time = time.time()
while True:
fetch_list = [avg_loss.name]
acc_name_list = [v.name for v in train_args[2]]
fetch_list.extend(acc_name_list)
fetch_list.append("learning_rate")
if iters > 0 and iters % args.log_period == 0:
if iters % args.log_period == 0:
should_print = True
else:
should_print = False
......@@ -410,35 +286,36 @@ def train_parallel(args):
except fluid.core.EnforceNotMet as ex:
traceback.print_exc()
exit(1)
num_samples += samples_per_step
num_samples += bs * DEVICE_NUM
if should_print:
fetched_data = [np.mean(np.array(d)) for d in fetch_ret]
print("Pass %d, batch %d, loss %s, accucacys: %s, learning_rate %s, py_reader queue_size: %d, avg batch time: %0.4f secs" %
(pass_id, iters, fetched_data[0], fetched_data[1:-1], fetched_data[-1], train_args[4].queue.size(), (time.time() - batch_time_start) * 1.0 / args.log_period ))
batch_time_start = time.time()
print("Epoch %d, batch %d, loss %s, accucacys: %s, learning_rate %s, py_reader queue_size: %d, avg batch time: %0.4f secs" %
(epoch_id, iters, fetched_data[0], fetched_data[1:-1], fetched_data[-1], train_args[4].queue.size(), (time.time() - batch_start_time)*1.0/args.log_period))
batch_start_time = time.time()
iters += 1
print_train_time(start_time, time.time(), num_samples, "Train")
print_train_time(start_time, time.time(), num_samples)
feed_list = [test_prog.global_block().var(varname) for varname in ("image", "label")]
gpu_id = int(os.getenv("FLAGS_selected_gpus")) if is_mp_mode() else 0
test_feeder = fluid.DataFeeder(feed_list=feed_list, place=fluid.CUDAPlace(gpu_id))
#test_ret = test_single(test_exe, test_args, args, test_prog, test_feeder, val_bs)
test_feeder = fluid.DataFeeder(feed_list=feed_list, place=fluid.CUDAPlace(0))
test_ret = test_parallel(test_exe, test_args, args, test_prog, test_feeder, val_bs)
print("Pass: %d, Test Accuracy: %s, Spend %.2f hours\n" %
(pass_id, [np.mean(np.array(v)) for v in test_ret], (time.time() - over_all_start) / 3600))
print("Epoch: %d, Test Accuracy: %s, Spend %.2f hours\n" %
(epoch_id, [np.mean(np.array(v)) for v in test_ret], (time.time() - over_all_start) / 3600))
print("total train time: ", time.time() - over_all_start)
def print_train_time(start_time, end_time, num_samples, prefix_text=""):
def print_train_time(start_time, end_time, num_samples):
train_elapsed = end_time - start_time
examples_per_sec = num_samples / train_elapsed
print('\n%s Total examples: %d, total time: %.5f, %.5f examples/sed\n' %
(prefix_text, num_samples, train_elapsed, examples_per_sec))
print('\nTotal examples: %d, total time: %.5f, %.5f examples/sed\n' %
(num_samples, train_elapsed, examples_per_sec))
def print_paddle_envs():
print('----------- Configuration envs -----------')
print("DEVICE_NUM: %d" % DEVICE_NUM)
for k in os.environ:
if "PADDLE_" in k:
print "ENV %s:%s" % (k, os.environ[k])
......@@ -447,7 +324,6 @@ def print_paddle_envs():
def main():
args = parse_args()
args.dist_env = dist_env()
print_arguments(args)
print_paddle_envs()
train_parallel(args)
......
fluid/PaddleCV/image_classification/models/fast_resnet.py
浏览文件 @ 2bb4e4a6
......@@ -30,14 +30,12 @@ import paddle.fluid.core as core
import paddle.fluid.profiler as profiler
import utils
## visreader for imagenet
import torchvision_reader
__all__ = ["FastResNet"]
class FastResNet():
def __init__(self, layers=50):
def __init__(self, layers=50, is_train=True):
self.layers = layers
self.is_train = is_train
def net(self, input, class_dim=1000, img_size=224, is_train=True):
layers = self.layers
......@@ -54,7 +52,7 @@ class FastResNet():
num_filters = [64, 128, 256, 512]
conv = self.conv_bn_layer(
input=input, num_filters=64, filter_size=7, stride=2, act='relu', is_train=is_train)
input=input, num_filters=64, filter_size=7, stride=2, act='relu')
conv = fluid.layers.pool2d(
input=conv,
pool_size=3,
......@@ -73,6 +71,7 @@ class FastResNet():
input=conv, pool_size=pool_size, pool_type='avg', global_pooling=True)
out = fluid.layers.fc(input=pool,
size=class_dim,
act=None,
param_attr=fluid.param_attr.ParamAttr(
initializer=fluid.initializer.NormalInitializer(0.0, 0.01),
regularizer=fluid.regularizer.L2Decay(1e-4)),
......@@ -87,8 +86,7 @@ class FastResNet():
stride=1,
groups=1,
act=None,
bn_init_value=1.0,
is_train=True):
bn_init_value=1.0):
conv = fluid.layers.conv2d(
input=input,
num_filters=num_filters,
......@@ -98,10 +96,8 @@ class FastResNet():
groups=groups,
act=None,
bias_attr=False,
param_attr=fluid.ParamAttr(
initializer=fluid.initializer.MSRAInitializer(),
regularizer=fluid.regularizer.L2Decay(1e-4)))
return fluid.layers.batch_norm(input=conv, act=act, is_test=not is_train,
param_attr=fluid.ParamAttr(regularizer=fluid.regularizer.L2Decay(1e-4)))
return fluid.layers.batch_norm(input=conv, act=act, is_test=not self.is_train,
param_attr=fluid.param_attr.ParamAttr(
initializer=fluid.initializer.Constant(bn_init_value),
regularizer=None))
......@@ -129,3 +125,67 @@ class FastResNet():
short = self.shortcut(input, num_filters * 4, stride)
return fluid.layers.elementwise_add(x=short, y=conv2, act='relu')
def lr_decay(lrs, epochs, bs, total_image):
boundaries = []
values = []
for idx, epoch in enumerate(epochs):
step = total_image // bs[idx]
if step * bs[idx] < total_image:
step += 1
ratio = (lrs[idx][1] - lrs[idx][0])*1.0 / (epoch[1] - epoch[0])
lr_base = lrs[idx][0]
for s in xrange(epoch[0], epoch[1]):
if boundaries:
boundaries.append(boundaries[-1] + step)
else:
boundaries = [step]
lr = lr_base + ratio * (s - epoch[0])
values.append(lr)
print("epoch: [%d], steps: [%d], lr: [%f]" % (s, boundaries[-1], values[-1]))
values.append(lrs[-1])
print("epoch: [%d:], steps: [%d:], lr:[%f]" % (epochs[-1][-1], boundaries[-1], values[-1]))
return boundaries, values
def linear_lr_decay_by_epoch(lr_values, epochs, bs_values, total_images):
from paddle.fluid.layers.learning_rate_scheduler import _decay_step_counter
import paddle.fluid.layers.tensor as tensor
import math
with paddle.fluid.default_main_program()._lr_schedule_guard():
global_step = _decay_step_counter()
lr = tensor.create_global_var(
shape=[1],
value=0.0,
dtype='float32',
persistable=True,
name="learning_rate")
with fluid.layers.control_flow.Switch() as switch:
last_steps = 0
for idx, epoch_bound in enumerate(epochs):
start_epoch, end_epoch = epoch_bound
linear_epoch = end_epoch - start_epoch
start_lr, end_lr = lr_values[idx]
linear_lr = end_lr - start_lr
for epoch_step in xrange(linear_epoch):
steps = last_steps + (1 + epoch_step) * total_images / bs_values[idx] + 1
boundary_val = tensor.fill_constant(
shape=[1],
dtype='float32',
value=float(steps),
force_cpu=True)
decayed_lr = start_lr + epoch_step * linear_lr * 1.0 / linear_epoch
with switch.case(global_step < boundary_val):
value_var = tensor.fill_constant(shape=[1], dtype='float32', value=float(decayed_lr))
print("steps: [%d], epoch : [%d], decayed_lr: [%f]" % (steps, start_epoch + epoch_step, decayed_lr))
fluid.layers.tensor.assign(value_var, lr)
last_steps = steps
last_value_var = tensor.fill_constant(
shape=[1],
dtype='float32',
value=float(lr_values[-1]))
with switch.default():
fluid.layers.tensor.assign(last_value_var, lr)
return lr
\ No newline at end of file
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或