Skip to content

P
PaddleSlim

PaddlePaddle / PaddleSlim
大约 2 年 前同步成功

通知 51
Star 1434
Fork 344
P
PaddleSlim
“7815ddf8c3b7a23f4bcbefa8b7e97ed4bd6d1cc4”上不存在“mobile/test/net/test_nlp.cpp”
未验证 提交 99f7d05b 编写于 作者: Y yukavio 提交者： GitHub
浏览文件
操作

Migrate prune (#492) 

* remove sensitive_pruner module

* migrate prune module
上级 bed39157
显示空白变更内容
内联 并排
Showing with 118 addition and 1647 deletion
demo/auto_prune/README.md 已删除 100644 → 0
浏览文件 @ bed39157
该示例介绍如何使用自动裁剪。
该示例需要使用IMAGENET数据，以及预训练模型。支持以下模型：
- MobileNetV1
- MobileNetV2
- ResNet50
## 1. 接口介绍
该示例涉及以下接口：
- [paddleslim.prune.AutoPruner])
- [paddleslim.prune.Pruner])
## 2. 运行示例
提供两种自动裁剪模式，直接以裁剪目标进行一次自动裁剪，和多次迭代的方式进行裁剪。
###2.1一次裁剪
在路径`PaddleSlim/demo/auto_prune`下执行以下代码运行示例：
```
export CUDA_VISIBLE_DEVICES=0
python train.py --model "MobileNet"
从log中获取搜索的最佳裁剪率列表，加入到train_finetune.py的ratiolist中，如下命令finetune得到最终结果
python train_finetune.py --model "MobileNet" --lr 0.1 --num_epochs 120 --step_epochs 30 60 90
```
通过`python train.py --help`查看更多选项。
###2.2多次迭代裁剪
在路径`PaddleSlim/demo/auto_prune`下执行以下代码运行示例：
```
export CUDA_VISIBLE_DEVICES=0
python train_iterator.py --model "MobileNet"
从log中获取本次迭代搜索的最佳裁剪率列表，加入到train_finetune.py的ratiolist中,如下命令开始finetune本次搜索到的结果
python train_finetune.py --model "MobileNet"
将第一次迭代的最佳裁剪率列表，加入到train_iterator.py 的ratiolist中,如下命令进行第二次迭代
python train_iterator.py --model "MobileNet" --pretrained_model "checkpoint/Mobilenet/19"
finetune第二次迭代搜索结果，并继续重复迭代，直到获得目标裁剪率的结果
...
如下命令finetune最终结果
python train_finetune.py --model "MobileNet" --pretrained_model "checkpoint/Mobilenet/19" --num_epochs 70 --step_epochs 10 40
```
## 3. 注意
### 3.1 一次裁剪
在`paddleslim.prune.AutoPruner`接口的参数中，pruned_flops表示期望的最低flops剪切率。
### 3.2 多次迭代裁剪
单次迭代的裁剪目标，建议不高于10%。
在load前次迭代结果时，需要删除checkpoint下learning_rate、@LR_DECAY_COUNTER@等文件，避免继承之前的learning_rate，影响finetune效果。
demo/auto_prune/train.py 已删除 100644 → 0
浏览文件 @ bed39157
import os
import sys
import logging
import paddle
import argparse
import functools
import math
import time
import numpy as np
import paddle.fluid as fluid
sys.path[0] = os.path.join(os.path.dirname("__file__"), os.path.pardir)
from paddleslim.prune import AutoPruner
from paddleslim.common import get_logger
from paddleslim.analysis import flops
import models
from utility import add_arguments, print_arguments
_logger = get_logger(__name__, level=logging.INFO)
parser = argparse.ArgumentParser(description=__doc__)
add_arg = functools.partial(add_arguments, argparser=parser)
# yapf: disable
add_arg('batch_size', int, 64 * 4, "Minibatch size.")
add_arg('use_gpu', bool, True, "Whether to use GPU or not.")
add_arg('model', str, "MobileNet", "The target model.")
add_arg('pretrained_model', str, "../pretrained_model/MobileNetV1_pretrained", "Whether to use pretrained model.")
add_arg('lr', float, 0.1, "The learning rate used to fine-tune pruned model.")
add_arg('lr_strategy', str, "piecewise_decay", "The learning rate decay strategy.")
add_arg('l2_decay', float, 3e-5, "The l2_decay parameter.")
add_arg('momentum_rate', float, 0.9, "The value of momentum_rate.")
add_arg('num_epochs', int, 120, "The number of total epochs.")
add_arg('total_images', int, 1281167, "The number of total training images.")
parser.add_argument('--step_epochs', nargs='+', type=int, default=[30, 60, 90], help="piecewise decay step")
add_arg('config_file', str, None, "The config file for compression with yaml format.")
add_arg('data', str, "imagenet", "Which data to use. 'mnist' or 'imagenet'")
add_arg('log_period', int, 10, "Log period in batches.")
add_arg('test_period', int, 10, "Test period in epoches.")
# yapf: enable
model_list = [m for m in dir(models) if "__" not in m]
def piecewise_decay(args):
step = int(math.ceil(float(args.total_images) / args.batch_size))
bd = [step * e for e in args.step_epochs]
lr = [args.lr * (0.1**i) for i in range(len(bd) + 1)]
learning_rate = fluid.layers.piecewise_decay(boundaries=bd, values=lr)
optimizer = fluid.optimizer.Momentum(
learning_rate=learning_rate,
momentum=args.momentum_rate,
regularization=fluid.regularizer.L2Decay(args.l2_decay))
return optimizer
def cosine_decay(args):
step = int(math.ceil(float(args.total_images) / args.batch_size))
learning_rate = fluid.layers.cosine_decay(
learning_rate=args.lr, step_each_epoch=step, epochs=args.num_epochs)
optimizer = fluid.optimizer.Momentum(
learning_rate=learning_rate,
momentum=args.momentum_rate,
regularization=fluid.regularizer.L2Decay(args.l2_decay))
return optimizer
def create_optimizer(args):
if args.lr_strategy == "piecewise_decay":
return piecewise_decay(args)
elif args.lr_strategy == "cosine_decay":
return cosine_decay(args)
def compress(args):
train_reader = None
test_reader = None
if args.data == "mnist":
import paddle.dataset.mnist as reader
train_reader = reader.train()
val_reader = reader.test()
class_dim = 10
image_shape = "1,28,28"
elif args.data == "imagenet":
import imagenet_reader as reader
train_reader = reader.train()
val_reader = reader.val()
class_dim = 1000
image_shape = "3,224,224"
else:
raise ValueError("{} is not supported.".format(args.data))
image_shape = [int(m) for m in image_shape.split(",")]
assert args.model in model_list, "{} is not in lists: {}".format(args.model,
model_list)
image = fluid.layers.data(name='image', shape=image_shape, dtype='float32')
label = fluid.layers.data(name='label', shape=[1], dtype='int64')
# model definition
model = models.__dict__[args.model]()
out = model.net(input=image, class_dim=class_dim)
cost = fluid.layers.cross_entropy(input=out, label=label)
avg_cost = fluid.layers.mean(x=cost)
acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1)
acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5)
val_program = fluid.default_main_program().clone(for_test=True)
opt = create_optimizer(args)
opt.minimize(avg_cost)
place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
if args.pretrained_model:
def if_exist(var):
return os.path.exists(os.path.join(args.pretrained_model, var.name))
fluid.io.load_vars(exe, args.pretrained_model, predicate=if_exist)
val_reader = paddle.fluid.io.batch(val_reader, batch_size=args.batch_size)
train_reader = paddle.fluid.io.batch(
train_reader, batch_size=args.batch_size, drop_last=True)
train_feeder = feeder = fluid.DataFeeder([image, label], place)
val_feeder = feeder = fluid.DataFeeder(
[image, label], place, program=val_program)
def test(epoch, program):
batch_id = 0
acc_top1_ns = []
acc_top5_ns = []
for data in val_reader():
start_time = time.time()
acc_top1_n, acc_top5_n = exe.run(
program,
feed=train_feeder.feed(data),
fetch_list=[acc_top1.name, acc_top5.name])
end_time = time.time()
if batch_id % args.log_period == 0:
_logger.info(
"Eval epoch[{}] batch[{}] - acc_top1: {}; acc_top5: {}; time: {}".
format(epoch, batch_id,
np.mean(acc_top1_n),
np.mean(acc_top5_n), end_time - start_time))
acc_top1_ns.append(np.mean(acc_top1_n))
acc_top5_ns.append(np.mean(acc_top5_n))
batch_id += 1
_logger.info("Final eval epoch[{}] - acc_top1: {}; acc_top5: {}".format(
epoch,
np.mean(np.array(acc_top1_ns)), np.mean(np.array(acc_top5_ns))))
return np.mean(np.array(acc_top1_ns))
def train(epoch, program):
build_strategy = fluid.BuildStrategy()
exec_strategy = fluid.ExecutionStrategy()
train_program = fluid.compiler.CompiledProgram(
program).with_data_parallel(
loss_name=avg_cost.name,
build_strategy=build_strategy,
exec_strategy=exec_strategy)
batch_id = 0
for data in train_reader():
start_time = time.time()
loss_n, acc_top1_n, acc_top5_n = exe.run(
train_program,
feed=train_feeder.feed(data),
fetch_list=[avg_cost.name, acc_top1.name, acc_top5.name])
end_time = time.time()
loss_n = np.mean(loss_n)
acc_top1_n = np.mean(acc_top1_n)
acc_top5_n = np.mean(acc_top5_n)
if batch_id % args.log_period == 0:
_logger.info(
"epoch[{}]-batch[{}] - loss: {}; acc_top1: {}; acc_top5: {}; time: {}".
format(epoch, batch_id, loss_n, acc_top1_n, acc_top5_n,
end_time - start_time))
batch_id += 1
params = []
for param in fluid.default_main_program().global_block().all_parameters():
if "_sep_weights" in param.name:
params.append(param.name)
pruner = AutoPruner(
val_program,
fluid.global_scope(),
place,
params=params,
init_ratios=[0.33] * len(params),
pruned_flops=0.5,
pruned_latency=None,
server_addr=("", 0),
init_temperature=100,
reduce_rate=0.85,
max_try_times=300,
max_client_num=10,
search_steps=100,
max_ratios=0.9,
min_ratios=0.,
is_server=True,
key="auto_pruner")
while True:
pruned_program, pruned_val_program = pruner.prune(
fluid.default_main_program(), val_program)
for i in range(1):
train(i, pruned_program)
score = test(0, pruned_val_program)
pruner.reward(score)
def main():
args = parser.parse_args()
print_arguments(args)
compress(args)
if __name__ == '__main__':
main()
demo/auto_prune/train_finetune.py 已删除 100644 → 0
浏览文件 @ bed39157
import os
import sys
import logging
import paddle
import argparse
import functools
import math
import paddle.fluid as fluid
import imagenet_reader as reader
import models
from utility import add_arguments, print_arguments
import numpy as np
import time
from paddleslim.prune import Pruner
from paddleslim.analysis import flops
parser = argparse.ArgumentParser(description=__doc__)
add_arg = functools.partial(add_arguments, argparser=parser)
# yapf: disable
add_arg('batch_size', int, 64 * 4, "Minibatch size.")
add_arg('use_gpu', bool, True, "Whether to use GPU or not.")
add_arg('model', str, "MobileNet", "The target model.")
add_arg('model_save_dir', str, "./", "checkpoint model.")
add_arg('pretrained_model', str, "../pretrained_model/MobileNetV1_pretrained", "Whether to use pretrained model.")
add_arg('lr', float, 0.01, "The learning rate used to fine-tune pruned model.")
add_arg('lr_strategy', str, "piecewise_decay", "The learning rate decay strategy.")
add_arg('l2_decay', float, 3e-5, "The l2_decay parameter.")
add_arg('momentum_rate', float, 0.9, "The value of momentum_rate.")
add_arg('num_epochs', int, 20, "The number of total epochs.")
add_arg('total_images', int, 1281167, "The number of total training images.")
parser.add_argument('--step_epochs', nargs='+', type=int, default=[5, 15], help="piecewise decay step")
add_arg('config_file', str, None, "The config file for compression with yaml format.")
# yapf: enable
model_list = [m for m in dir(models) if "__" not in m]
ratiolist = [
# [0.06, 0.0, 0.09, 0.03, 0.09, 0.02, 0.05, 0.03, 0.0, 0.07, 0.07, 0.05, 0.08],
# [0.08, 0.02, 0.03, 0.13, 0.1, 0.06, 0.03, 0.04, 0.14, 0.02, 0.03, 0.02, 0.01],
]
def save_model(args, exe, train_prog, eval_prog, info):
model_path = os.path.join(args.model_save_dir, args.model, str(info))
if not os.path.isdir(model_path):
os.makedirs(model_path)
fluid.io.save_persistables(exe, model_path, main_program=train_prog)
print("Already save model in %s" % (model_path))
def piecewise_decay(args):
step = int(math.ceil(float(args.total_images) / args.batch_size))
bd = [step * e for e in args.step_epochs]
lr = [args.lr * (0.1**i) for i in range(len(bd) + 1)]
learning_rate = fluid.layers.piecewise_decay(boundaries=bd, values=lr)
optimizer = fluid.optimizer.Momentum(
learning_rate=learning_rate,
momentum=args.momentum_rate,
regularization=fluid.regularizer.L2Decay(args.l2_decay))
return optimizer
def cosine_decay(args):
step = int(math.ceil(float(args.total_images) / args.batch_size))
learning_rate = fluid.layers.cosine_decay(
learning_rate=args.lr, step_each_epoch=step, epochs=args.num_epochs)
optimizer = fluid.optimizer.Momentum(
learning_rate=learning_rate,
momentum=args.momentum_rate,
regularization=fluid.regularizer.L2Decay(args.l2_decay))
return optimizer
def create_optimizer(args):
if args.lr_strategy == "piecewise_decay":
return piecewise_decay(args)
elif args.lr_strategy == "cosine_decay":
return cosine_decay(args)
def compress(args):
class_dim = 1000
image_shape = "3,224,224"
image_shape = [int(m) for m in image_shape.split(",")]
assert args.model in model_list, "{} is not in lists: {}".format(args.model,
model_list)
image = fluid.layers.data(name='image', shape=image_shape, dtype='float32')
label = fluid.layers.data(name='label', shape=[1], dtype='int64')
# model definition
model = models.__dict__[args.model]()
out = model.net(input=image, class_dim=class_dim)
cost = fluid.layers.cross_entropy(input=out, label=label)
avg_cost = fluid.layers.mean(x=cost)
acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1)
acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5)
val_program = fluid.default_main_program().clone(for_test=True)
opt = create_optimizer(args)
opt.minimize(avg_cost)
place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
if args.pretrained_model:
def if_exist(var):
exist = os.path.exists(
os.path.join(args.pretrained_model, var.name))
print("exist", exist)
return exist
#fluid.io.load_vars(exe, args.pretrained_model, predicate=if_exist)
val_reader = paddle.fluid.io.batch(reader.val(), batch_size=args.batch_size)
train_reader = paddle.fluid.io.batch(
reader.train(), batch_size=args.batch_size, drop_last=True)
train_feeder = feeder = fluid.DataFeeder([image, label], place)
val_feeder = feeder = fluid.DataFeeder(
[image, label], place, program=val_program)
def test(epoch, program):
batch_id = 0
acc_top1_ns = []
acc_top5_ns = []
for data in val_reader():
start_time = time.time()
acc_top1_n, acc_top5_n = exe.run(
program,
feed=train_feeder.feed(data),
fetch_list=[acc_top1.name, acc_top5.name])
end_time = time.time()
print(
"Eval epoch[{}] batch[{}] - acc_top1: {}; acc_top5: {}; time: {}".
format(epoch, batch_id,
np.mean(acc_top1_n),
np.mean(acc_top5_n), end_time - start_time))
acc_top1_ns.append(np.mean(acc_top1_n))
acc_top5_ns.append(np.mean(acc_top5_n))
batch_id += 1
print("Final eval epoch[{}] - acc_top1: {}; acc_top5: {}".format(
epoch,
np.mean(np.array(acc_top1_ns)), np.mean(np.array(acc_top5_ns))))
def train(epoch, program):
build_strategy = fluid.BuildStrategy()
exec_strategy = fluid.ExecutionStrategy()
train_program = fluid.compiler.CompiledProgram(
program).with_data_parallel(
loss_name=avg_cost.name,
build_strategy=build_strategy,
exec_strategy=exec_strategy)
batch_id = 0
for data in train_reader():
start_time = time.time()
loss_n, acc_top1_n, acc_top5_n, lr_n = exe.run(
train_program,
feed=train_feeder.feed(data),
fetch_list=[
avg_cost.name, acc_top1.name, acc_top5.name, "learning_rate"
])
end_time = time.time()
loss_n = np.mean(loss_n)
acc_top1_n = np.mean(acc_top1_n)
acc_top5_n = np.mean(acc_top5_n)
lr_n = np.mean(lr_n)
print(
"epoch[{}]-batch[{}] - loss: {}; acc_top1: {}; acc_top5: {};lrn: {}; time: {}".
format(epoch, batch_id, loss_n, acc_top1_n, acc_top5_n, lr_n,
end_time - start_time))
batch_id += 1
params = []
for param in fluid.default_main_program().global_block().all_parameters():
#if "_weights" in param.name and "conv1_weights" not in param.name:
if "_sep_weights" in param.name:
params.append(param.name)
print("fops before pruning: {}".format(flops(fluid.default_main_program())))
pruned_program_iter = fluid.default_main_program()
pruned_val_program_iter = val_program
for ratios in ratiolist:
pruner = Pruner()
pruned_val_program_iter = pruner.prune(
pruned_val_program_iter,
fluid.global_scope(),
params=params,
ratios=ratios,
place=place,
only_graph=True)
pruned_program_iter = pruner.prune(
pruned_program_iter,
fluid.global_scope(),
params=params,
ratios=ratios,
place=place)
print("fops after pruning: {}".format(flops(pruned_program_iter)))
""" do not inherit learning rate """
if (os.path.exists(args.pretrained_model + "/learning_rate")):
os.remove(args.pretrained_model + "/learning_rate")
if (os.path.exists(args.pretrained_model + "/@LR_DECAY_COUNTER@")):
os.remove(args.pretrained_model + "/@LR_DECAY_COUNTER@")
fluid.io.load_vars(
exe,
args.pretrained_model,
main_program=pruned_program_iter,
predicate=if_exist)
pruned_program = pruned_program_iter
pruned_val_program = pruned_val_program_iter
for i in range(args.num_epochs):
train(i, pruned_program)
test(i, pruned_val_program)
save_model(args, exe, pruned_program, pruned_val_program, i)
def main():
args = parser.parse_args()
print_arguments(args)
compress(args)
if __name__ == '__main__':
main()
demo/auto_prune/train_iterator.py 已删除 100644 → 0
浏览文件 @ bed39157
import os
import sys
import logging
import paddle
import argparse
import functools
import math
import time
import numpy as np
import paddle.fluid as fluid
from paddleslim.prune import AutoPruner
from paddleslim.common import get_logger
from paddleslim.analysis import flops
from paddleslim.prune import Pruner
sys.path.append(sys.path[0] + "/../")
import models
from utility import add_arguments, print_arguments
_logger = get_logger(__name__, level=logging.INFO)
parser = argparse.ArgumentParser(description=__doc__)
add_arg = functools.partial(add_arguments, argparser=parser)
# yapf: disable
add_arg('batch_size', int, 64 * 4, "Minibatch size.")
add_arg('use_gpu', bool, True, "Whether to use GPU or not.")
add_arg('model', str, "MobileNet", "The target model.")
add_arg('pretrained_model', str, "../pretrained_model/MobileNetV1_pretrained", "Whether to use pretrained model.")
add_arg('model_save_dir', str, "./", "checkpoint model.")
add_arg('lr', float, 0.1, "The learning rate used to fine-tune pruned model.")
add_arg('lr_strategy', str, "piecewise_decay", "The learning rate decay strategy.")
add_arg('l2_decay', float, 3e-5, "The l2_decay parameter.")
add_arg('momentum_rate', float, 0.9, "The value of momentum_rate.")
add_arg('num_epochs', int, 120, "The number of total epochs.")
add_arg('total_images', int, 1281167, "The number of total training images.")
parser.add_argument('--step_epochs', nargs='+', type=int, default=[30, 60, 90], help="piecewise decay step")
add_arg('config_file', str, None, "The config file for compression with yaml format.")
add_arg('data', str, "imagenet", "Which data to use. 'mnist' or 'imagenet'")
add_arg('log_period', int, 10, "Log period in batches.")
add_arg('test_period', int, 10, "Test period in epoches.")
# yapf: enable
model_list = [m for m in dir(models) if "__" not in m]
ratiolist = [
# [0.06, 0.0, 0.09, 0.03, 0.09, 0.02, 0.05, 0.03, 0.0, 0.07, 0.07, 0.05, 0.08],
# [0.08, 0.02, 0.03, 0.13, 0.1, 0.06, 0.03, 0.04, 0.14, 0.02, 0.03, 0.02, 0.01],
]
def piecewise_decay(args):
step = int(math.ceil(float(args.total_images) / args.batch_size))
bd = [step * e for e in args.step_epochs]
lr = [args.lr * (0.1**i) for i in range(len(bd) + 1)]
learning_rate = fluid.layers.piecewise_decay(boundaries=bd, values=lr)
optimizer = fluid.optimizer.Momentum(
learning_rate=learning_rate,
momentum=args.momentum_rate,
regularization=fluid.regularizer.L2Decay(args.l2_decay))
return optimizer
def cosine_decay(args):
step = int(math.ceil(float(args.total_images) / args.batch_size))
learning_rate = fluid.layers.cosine_decay(
learning_rate=args.lr, step_each_epoch=step, epochs=args.num_epochs)
optimizer = fluid.optimizer.Momentum(
learning_rate=learning_rate,
momentum=args.momentum_rate,
regularization=fluid.regularizer.L2Decay(args.l2_decay))
return optimizer
def create_optimizer(args):
if args.lr_strategy == "piecewise_decay":
return piecewise_decay(args)
elif args.lr_strategy == "cosine_decay":
return cosine_decay(args)
def compress(args):
train_reader = None
test_reader = None
if args.data == "mnist":
import paddle.dataset.mnist as reader
train_reader = reader.train()
val_reader = reader.test()
class_dim = 10
image_shape = "1,28,28"
elif args.data == "imagenet":
import imagenet_reader as reader
train_reader = reader.train()
val_reader = reader.val()
class_dim = 1000
image_shape = "3,224,224"
else:
raise ValueError("{} is not supported.".format(args.data))
image_shape = [int(m) for m in image_shape.split(",")]
assert args.model in model_list, "{} is not in lists: {}".format(args.model,
model_list)
image = fluid.layers.data(name='image', shape=image_shape, dtype='float32')
label = fluid.layers.data(name='label', shape=[1], dtype='int64')
# model definition
model = models.__dict__[args.model]()
out = model.net(input=image, class_dim=class_dim)
cost = fluid.layers.cross_entropy(input=out, label=label)
avg_cost = fluid.layers.mean(x=cost)
acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1)
acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5)
val_program = fluid.default_main_program().clone(for_test=True)
opt = create_optimizer(args)
opt.minimize(avg_cost)
place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
if args.pretrained_model:
def if_exist(var):
return os.path.exists(os.path.join(args.pretrained_model, var.name))
# fluid.io.load_vars(exe, args.pretrained_model, predicate=if_exist)
val_reader = paddle.fluid.io.batch(val_reader, batch_size=args.batch_size)
train_reader = paddle.fluid.io.batch(
train_reader, batch_size=args.batch_size, drop_last=True)
train_feeder = feeder = fluid.DataFeeder([image, label], place)
val_feeder = feeder = fluid.DataFeeder(
[image, label], place, program=val_program)
def test(epoch, program):
batch_id = 0
acc_top1_ns = []
acc_top5_ns = []
for data in val_reader():
start_time = time.time()
acc_top1_n, acc_top5_n = exe.run(
program,
feed=train_feeder.feed(data),
fetch_list=[acc_top1.name, acc_top5.name])
end_time = time.time()
if batch_id % args.log_period == 0:
_logger.info(
"Eval epoch[{}] batch[{}] - acc_top1: {}; acc_top5: {}; time: {}".
format(epoch, batch_id,
np.mean(acc_top1_n),
np.mean(acc_top5_n), end_time - start_time))
acc_top1_ns.append(np.mean(acc_top1_n))
acc_top5_ns.append(np.mean(acc_top5_n))
batch_id += 1
_logger.info("Final eval epoch[{}] - acc_top1: {}; acc_top5: {}".format(
epoch,
np.mean(np.array(acc_top1_ns)), np.mean(np.array(acc_top5_ns))))
return np.mean(np.array(acc_top1_ns))
def train(epoch, program):
build_strategy = fluid.BuildStrategy()
exec_strategy = fluid.ExecutionStrategy()
train_program = fluid.compiler.CompiledProgram(
program).with_data_parallel(
loss_name=avg_cost.name,
build_strategy=build_strategy,
exec_strategy=exec_strategy)
batch_id = 0
for data in train_reader():
start_time = time.time()
loss_n, acc_top1_n, acc_top5_n = exe.run(
train_program,
feed=train_feeder.feed(data),
fetch_list=[avg_cost.name, acc_top1.name, acc_top5.name])
end_time = time.time()
loss_n = np.mean(loss_n)
acc_top1_n = np.mean(acc_top1_n)
acc_top5_n = np.mean(acc_top5_n)
if batch_id % args.log_period == 0:
_logger.info(
"epoch[{}]-batch[{}] - loss: {}; acc_top1: {}; acc_top5: {}; time: {}".
format(epoch, batch_id, loss_n, acc_top1_n, acc_top5_n,
end_time - start_time))
batch_id += 1
params = []
for param in fluid.default_main_program().global_block().all_parameters():
if "_sep_weights" in param.name:
params.append(param.name)
pruned_program_iter = fluid.default_main_program()
pruned_val_program_iter = val_program
for ratios in ratiolist:
pruner = Pruner()
pruned_val_program_iter = pruner.prune(
pruned_val_program_iter,
fluid.global_scope(),
params=params,
ratios=ratios,
place=place,
only_graph=True)
pruned_program_iter = pruner.prune(
pruned_program_iter,
fluid.global_scope(),
params=params,
ratios=ratios,
place=place)
print("fops after pruning: {}".format(flops(pruned_program_iter)))
fluid.io.load_vars(
exe,
args.pretrained_model,
main_program=pruned_program_iter,
predicate=if_exist)
pruner = AutoPruner(
pruned_val_program_iter,
fluid.global_scope(),
place,
params=params,
init_ratios=[0.1] * len(params),
pruned_flops=0.1,
pruned_latency=None,
server_addr=("", 0),
init_temperature=100,
reduce_rate=0.85,
max_try_times=300,
max_client_num=10,
search_steps=100,
max_ratios=0.2,
min_ratios=0.,
is_server=True,
key="auto_pruner")
while True:
pruned_program, pruned_val_program = pruner.prune(
pruned_program_iter, pruned_val_program_iter)
for i in range(0):
train(i, pruned_program)
score = test(0, pruned_val_program)
pruner.reward(score)
def main():
args = parser.parse_args()
print_arguments(args)
compress(args)
if __name__ == '__main__':
main()
demo/prune/eval.py
浏览文件 @ 99f7d05b
...@@ -7,7 +7,6 @@ import functools ...@@ -7,7 +7,6 @@ import functools
import math import math
import time import time
import numpy as np import numpy as np
import paddle.fluid as fluid
from paddleslim.prune import load_model from paddleslim.prune import load_model
from paddleslim.common import get_logger from paddleslim.common import get_logger
from paddleslim.analysis import flops from paddleslim.analysis import flops
...@@ -35,9 +34,7 @@ def eval(args): ...@@ -35,9 +34,7 @@ def eval(args):
train_reader = None train_reader = None
test_reader = None test_reader = None
if args.data == "mnist": if args.data == "mnist":
import paddle.dataset.mnist as reader val_reader = paddle.dataset.mnist.test()
train_reader = reader.train()
val_reader = reader.test()
class_dim = 10 class_dim = 10
image_shape = "1,28,28" image_shape = "1,28,28"
elif args.data == "imagenet": elif args.data == "imagenet":
...@@ -51,34 +48,36 @@ def eval(args): ...@@ -51,34 +48,36 @@ def eval(args):
image_shape = [int(m) for m in image_shape.split(",")] image_shape = [int(m) for m in image_shape.split(",")]
assert args.model in model_list, "{} is not in lists: {}".format(args.model, assert args.model in model_list, "{} is not in lists: {}".format(args.model,
model_list) model_list)
image = fluid.layers.data(name='image', shape=image_shape, dtype='float32') image = paddle.static.data(
label = fluid.layers.data(name='label', shape=[1], dtype='int64') name='image', shape=[None] + image_shape, dtype='float32')
label = paddle.static.data(name='label', shape=[None, 1], dtype='int64')
# model definition # model definition
model = models.__dict__[args.model]() model = models.__dict__[args.model]()
out = model.net(input=image, class_dim=class_dim) out = model.net(input=image, class_dim=class_dim)
acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1) acc_top1 = paddle.metric.accuracy(input=out, label=label, k=1)
acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5) acc_top5 = paddle.metric.accuracy(input=out, label=label, k=5)
val_program = fluid.default_main_program().clone(for_test=True) val_program = paddle.static.default_main_program().clone(for_test=True)
place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() place = paddle.CUDAPlace(0) if args.use_gpu else paddle.CPUPlace()
exe = fluid.Executor(place) exe = paddle.static.Executor(place)
exe.run(fluid.default_startup_program()) exe.run(paddle.static.default_startup_program())
val_reader = paddle.fluid.io.batch(val_reader, batch_size=args.batch_size) val_reader = paddle.batch(val_reader, batch_size=args.batch_size)
val_feeder = feeder = fluid.DataFeeder( valid_loader = paddle.io.DataLoader.from_generator(
[image, label], place, program=val_program) feed_list=[image, label],
capacity=64,
use_double_buffer=True,
iterable=True)
valid_loader.set_sample_list_generator(val_reader, place)
load_model(exe, val_program, args.model_path) load_model(exe, val_program, args.model_path)
batch_id = 0
acc_top1_ns = [] acc_top1_ns = []
acc_top5_ns = [] acc_top5_ns = []
for data in val_reader(): for batch_id, data in enumerate(valid_loader):
start_time = time.time() start_time = time.time()
acc_top1_n, acc_top5_n = exe.run( acc_top1_n, acc_top5_n = exe.run(
val_program, val_program, feed=data, fetch_list=[acc_top1.name, acc_top5.name])
feed=val_feeder.feed(data),
fetch_list=[acc_top1.name, acc_top5.name])
end_time = time.time() end_time = time.time()
if batch_id % args.log_period == 0: if batch_id % args.log_period == 0:
_logger.info( _logger.info(
...@@ -88,7 +87,6 @@ def eval(args): ...@@ -88,7 +87,6 @@ def eval(args):
np.mean(acc_top5_n), end_time - start_time)) np.mean(acc_top5_n), end_time - start_time))
acc_top1_ns.append(np.mean(acc_top1_n)) acc_top1_ns.append(np.mean(acc_top1_n))
acc_top5_ns.append(np.mean(acc_top5_n)) acc_top5_ns.append(np.mean(acc_top5_n))
batch_id += 1
_logger.info("Final eval - acc_top1: {}; acc_top5: {}".format( _logger.info("Final eval - acc_top1: {}; acc_top5: {}".format(
np.mean(np.array(acc_top1_ns)), np.mean(np.array(acc_top5_ns)))) np.mean(np.array(acc_top1_ns)), np.mean(np.array(acc_top5_ns))))
......
demo/prune/train.py
浏览文件 @ 99f7d05b
...@@ -7,7 +7,6 @@ import functools ...@@ -7,7 +7,6 @@ import functools
import math import math
import time import time
import numpy as np import numpy as np
import paddle.fluid as fluid
sys.path[0] = os.path.join(os.path.dirname("__file__"), os.path.pardir) sys.path[0] = os.path.join(os.path.dirname("__file__"), os.path.pardir)
from paddleslim.prune import Pruner, save_model from paddleslim.prune import Pruner, save_model
from paddleslim.common import get_logger from paddleslim.common import get_logger
...@@ -69,23 +68,23 @@ def piecewise_decay(args): ...@@ -69,23 +68,23 @@ def piecewise_decay(args):
step = int(math.ceil(float(args.total_images) / args.batch_size)) step = int(math.ceil(float(args.total_images) / args.batch_size))
bd = [step * e for e in args.step_epochs] bd = [step * e for e in args.step_epochs]
lr = [args.lr * (0.1**i) for i in range(len(bd) + 1)] lr = [args.lr * (0.1**i) for i in range(len(bd) + 1)]
learning_rate = fluid.layers.piecewise_decay(boundaries=bd, values=lr) learning_rate = paddle.optimizer.lr.PiecewiseDecay(boundaries=bd, values=lr)
optimizer = fluid.optimizer.Momentum( optimizer = paddle.optimizer.Momentum(
learning_rate=learning_rate, learning_rate=learning_rate,
momentum=args.momentum_rate, momentum=args.momentum_rate,
regularization=fluid.regularizer.L2Decay(args.l2_decay)) weight_decay=paddle.regularizer.L2Decay(args.l2_decay))
return optimizer return optimizer
def cosine_decay(args): def cosine_decay(args):
step = int(math.ceil(float(args.total_images) / args.batch_size)) step = int(math.ceil(float(args.total_images) / args.batch_size))
learning_rate = fluid.layers.cosine_decay( learning_rate = paddle.optimizer.lr.cosine_decay(
learning_rate=args.lr, step_each_epoch=step, epochs=args.num_epochs) learning_rate=args.lr, step_each_epoch=step, epochs=args.num_epochs)
optimizer = fluid.optimizer.Momentum( optimizer = paddle.optimizer.Momentum(
learning_rate=learning_rate, learning_rate=learning_rate,
momentum=args.momentum_rate, momentum=args.momentum_rate,
regularization=fluid.regularizer.L2Decay(args.l2_decay)) weight_decay=paddle.regularizer.L2Decay(args.l2_decay))
return optimizer return optimizer
...@@ -100,9 +99,8 @@ def compress(args): ...@@ -100,9 +99,8 @@ def compress(args):
train_reader = None train_reader = None
test_reader = None test_reader = None
if args.data == "mnist": if args.data == "mnist":
import paddle.dataset.mnist as reader train_reader = paddle.dataset.mnist.train()
train_reader = reader.train() val_reader = paddle.dataset.mnist.test()
val_reader = reader.test()
class_dim = 10 class_dim = 10
image_shape = "1,28,28" image_shape = "1,28,28"
elif args.data == "imagenet": elif args.data == "imagenet":
...@@ -116,21 +114,24 @@ def compress(args): ...@@ -116,21 +114,24 @@ def compress(args):
image_shape = [int(m) for m in image_shape.split(",")] image_shape = [int(m) for m in image_shape.split(",")]
assert args.model in model_list, "{} is not in lists: {}".format(args.model, assert args.model in model_list, "{} is not in lists: {}".format(args.model,
model_list) model_list)
image = fluid.layers.data(name='image', shape=image_shape, dtype='float32') image = paddle.static.data(
label = fluid.layers.data(name='label', shape=[1], dtype='int64') name='image', shape=[None] + image_shape, dtype='float32')
label = paddle.static.data(name='label', shape=[None, 1], dtype='int64')
# model definition # model definition
model = models.__dict__[args.model]() model = models.__dict__[args.model]()
out = model.net(input=image, class_dim=class_dim) out = model.net(input=image, class_dim=class_dim)
cost = fluid.layers.cross_entropy(input=out, label=label) cost = paddle.nn.functional.loss.cross_entropy(input=out, label=label)
avg_cost = fluid.layers.mean(x=cost) avg_cost = paddle.mean(x=cost)
acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1) acc_top1 = paddle.metric.accuracy(input=out, label=label, k=1)
acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5) acc_top5 = paddle.metric.accuracy(input=out, label=label, k=5)
val_program = fluid.default_main_program().clone(for_test=True) val_program = paddle.static.default_main_program().clone(for_test=True)
opt = create_optimizer(args) opt = create_optimizer(args)
opt.minimize(avg_cost) opt.minimize(avg_cost)
place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() places = paddle.static.cuda_places(
exe = fluid.Executor(place) ) if args.use_gpu else paddle.static.cpu_places()
exe.run(fluid.default_startup_program()) place = places[0]
exe = paddle.static.Executor(place)
exe.run(paddle.static.default_startup_program())
if args.pretrained_model: if args.pretrained_model:
...@@ -139,26 +140,33 @@ def compress(args): ...@@ -139,26 +140,33 @@ def compress(args):
_logger.info("Load pretrained model from {}".format( _logger.info("Load pretrained model from {}".format(
args.pretrained_model)) args.pretrained_model))
fluid.io.load_vars(exe, args.pretrained_model, predicate=if_exist) paddle.fluid.io.load_vars(
exe, args.pretrained_model, predicate=if_exist)
val_reader = paddle.fluid.io.batch(val_reader, batch_size=args.batch_size) val_reader = paddle.batch(val_reader, batch_size=args.batch_size)
train_reader = paddle.fluid.io.batch( train_reader = paddle.batch(
train_reader, batch_size=args.batch_size, drop_last=True) train_reader, batch_size=args.batch_size, drop_last=True)
train_feeder = feeder = fluid.DataFeeder([image, label], place) train_loader = paddle.io.DataLoader.from_generator(
val_feeder = feeder = fluid.DataFeeder( feed_list=[image, label],
[image, label], place, program=val_program) capacity=64,
use_double_buffer=True,
iterable=True)
valid_loader = paddle.io.DataLoader.from_generator(
feed_list=[image, label],
capacity=64,
use_double_buffer=True,
iterable=True)
train_loader.set_sample_list_generator(train_reader, places)
valid_loader.set_sample_list_generator(val_reader, place)
def test(epoch, program): def test(epoch, program):
batch_id = 0
acc_top1_ns = [] acc_top1_ns = []
acc_top5_ns = [] acc_top5_ns = []
for data in val_reader(): for batch_id, data in enumerate(valid_loader):
start_time = time.time() start_time = time.time()
acc_top1_n, acc_top5_n = exe.run( acc_top1_n, acc_top5_n = exe.run(
program, program, feed=data, fetch_list=[acc_top1.name, acc_top5.name])
feed=train_feeder.feed(data),
fetch_list=[acc_top1.name, acc_top5.name])
end_time = time.time() end_time = time.time()
if batch_id % args.log_period == 0: if batch_id % args.log_period == 0:
_logger.info( _logger.info(
...@@ -168,7 +176,6 @@ def compress(args): ...@@ -168,7 +176,6 @@ def compress(args):
np.mean(acc_top5_n), end_time - start_time)) np.mean(acc_top5_n), end_time - start_time))
acc_top1_ns.append(np.mean(acc_top1_n)) acc_top1_ns.append(np.mean(acc_top1_n))
acc_top5_ns.append(np.mean(acc_top5_n)) acc_top5_ns.append(np.mean(acc_top5_n))
batch_id += 1
_logger.info("Final eval epoch[{}] - acc_top1: {}; acc_top5: {}".format( _logger.info("Final eval epoch[{}] - acc_top1: {}; acc_top5: {}".format(
epoch, epoch,
...@@ -176,20 +183,19 @@ def compress(args): ...@@ -176,20 +183,19 @@ def compress(args):
def train(epoch, program): def train(epoch, program):
build_strategy = fluid.BuildStrategy() build_strategy = paddle.static.BuildStrategy()
exec_strategy = fluid.ExecutionStrategy() exec_strategy = paddle.static.ExecutionStrategy()
train_program = fluid.compiler.CompiledProgram( train_program = paddle.static.CompiledProgram(
program).with_data_parallel( program).with_data_parallel(
loss_name=avg_cost.name, loss_name=avg_cost.name,
build_strategy=build_strategy, build_strategy=build_strategy,
exec_strategy=exec_strategy) exec_strategy=exec_strategy)
batch_id = 0 for batch_id, data in enumerate(train_loader):
for data in train_reader():
start_time = time.time() start_time = time.time()
loss_n, acc_top1_n, acc_top5_n = exe.run( loss_n, acc_top1_n, acc_top5_n = exe.run(
train_program, train_program,
feed=train_feeder.feed(data), feed=data,
fetch_list=[avg_cost.name, acc_top1.name, acc_top5.name]) fetch_list=[avg_cost.name, acc_top1.name, acc_top5.name])
end_time = time.time() end_time = time.time()
loss_n = np.mean(loss_n) loss_n = np.mean(loss_n)
...@@ -203,22 +209,21 @@ def compress(args): ...@@ -203,22 +209,21 @@ def compress(args):
batch_id += 1 batch_id += 1
test(0, val_program) test(0, val_program)
params = get_pruned_params(args, paddle.static.default_main_program())
params = get_pruned_params(args, fluid.default_main_program())
_logger.info("FLOPs before pruning: {}".format( _logger.info("FLOPs before pruning: {}".format(
flops(fluid.default_main_program()))) flops(paddle.static.default_main_program())))
pruner = Pruner(args.criterion) pruner = Pruner(args.criterion)
pruned_val_program, _, _ = pruner.prune( pruned_val_program, _, _ = pruner.prune(
val_program, val_program,
fluid.global_scope(), paddle.static.global_scope(),
params=params, params=params,
ratios=[args.pruned_ratio] * len(params), ratios=[args.pruned_ratio] * len(params),
place=place, place=place,
only_graph=True) only_graph=True)
pruned_program, _, _ = pruner.prune( pruned_program, _, _ = pruner.prune(
fluid.default_main_program(), paddle.static.default_main_program(),
fluid.global_scope(), paddle.static.global_scope(),
params=params, params=params,
ratios=[args.pruned_ratio] * len(params), ratios=[args.pruned_ratio] * len(params),
place=place) place=place)
...@@ -232,8 +237,8 @@ def compress(args): ...@@ -232,8 +237,8 @@ def compress(args):
if args.save_inference: if args.save_inference:
infer_model_path = os.path.join(args.model_path, "infer_models", infer_model_path = os.path.join(args.model_path, "infer_models",
str(i)) str(i))
fluid.io.save_inference_model(infer_model_path, ["image"], [out], paddle.fluid.io.save_inference_model(infer_model_path, ["image"],
exe, pruned_val_program) [out], exe, pruned_val_program)
_logger.info("Saved inference model into [{}]".format( _logger.info("Saved inference model into [{}]".format(
infer_model_path)) infer_model_path))
......
demo/sensitive/train.py
浏览文件 @ 99f7d05b
...@@ -7,7 +7,6 @@ import functools ...@@ -7,7 +7,6 @@ import functools
import math import math
import time import time
import numpy as np import numpy as np
import paddle.fluid as fluid
from paddleslim.prune import merge_sensitive, get_ratios_by_loss from paddleslim.prune import merge_sensitive, get_ratios_by_loss
from paddleslim.prune import sensitivity from paddleslim.prune import sensitivity
from paddleslim.common import get_logger from paddleslim.common import get_logger
...@@ -46,43 +45,47 @@ def compress(args): ...@@ -46,43 +45,47 @@ def compress(args):
else: else:
raise ValueError("{} is not supported.".format(args.data)) raise ValueError("{} is not supported.".format(args.data))
image_shape = [int(m) for m in image_shape.split(",")] image_shape = [int(m) for m in image_shape.split(",")]
assert args.model in model_list, "{} is not in lists: {}".format( assert args.model in model_list, "{} is not in lists: {}".format(args.model,
args.model, model_list) model_list)
image = fluid.layers.data(name='image', shape=image_shape, dtype='float32') image = paddle.static.data(
label = fluid.layers.data(name='label', shape=[1], dtype='int64') name='image', shape=[None] + image_shape, dtype='float32')
label = paddle.static.data(name='label', shape=[None, 1], dtype='int64')
# model definition # model definition
model = models.__dict__[args.model]() model = models.__dict__[args.model]()
out = model.net(input=image, class_dim=class_dim) out = model.net(input=image, class_dim=class_dim)
acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1) acc_top1 = paddle.metric.accuracy(input=out, label=label, k=1)
acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5) acc_top5 = paddle.metric.accuracy(input=out, label=label, k=5)
val_program = fluid.default_main_program().clone(for_test=True) val_program = paddle.static.default_main_program().clone(for_test=True)
place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() places = paddle.static.cuda_places(
exe = fluid.Executor(place) ) if args.use_gpu else paddle.static.cpu_places()
exe.run(fluid.default_startup_program()) place = places[0]
exe = paddle.static.Executor(place)
exe.run(paddle.static.default_startup_program())
if args.pretrained_model: if args.pretrained_model:
def if_exist(var): def if_exist(var):
return os.path.exists( return os.path.exists(os.path.join(args.pretrained_model, var.name))
os.path.join(args.pretrained_model, var.name))
fluid.io.load_vars(exe, args.pretrained_model, predicate=if_exist) paddle.fluid.io.load_vars(
exe, args.pretrained_model, predicate=if_exist)
val_reader = paddle.fluid.io.batch(val_reader, batch_size=args.batch_size) val_reader = paddle.batch(val_reader, batch_size=args.batch_size)
valid_loader = paddle.io.DataLoader.from_generator(
feed_list=[image, label],
capacity=64,
use_double_buffer=True,
iterable=True)
val_feeder = feeder = fluid.DataFeeder( valid_loader.set_sample_list_generator(val_reader, place)
[image, label], place, program=val_program)
def test(program): def test(program):
batch_id = 0
acc_top1_ns = [] acc_top1_ns = []
acc_top5_ns = [] acc_top5_ns = []
for data in val_reader(): for batch_id, data in enumerate(valid_loader):
start_time = time.time() start_time = time.time()
acc_top1_n, acc_top5_n = exe.run( acc_top1_n, acc_top5_n = exe.run(
program, program, feed=data, fetch_list=[acc_top1.name, acc_top5.name])
feed=val_feeder.feed(data),
fetch_list=[acc_top1.name, acc_top5.name])
end_time = time.time() end_time = time.time()
if batch_id % args.log_period == 0: if batch_id % args.log_period == 0:
_logger.info( _logger.info(
...@@ -99,8 +102,10 @@ def compress(args): ...@@ -99,8 +102,10 @@ def compress(args):
return np.mean(np.array(acc_top1_ns)) return np.mean(np.array(acc_top1_ns))
params = [] params = []
for param in fluid.default_main_program().global_block().all_parameters(): for param in paddle.static.default_main_program().global_block(
if "_sep_weights" in param.name: ).all_parameters():
if "weights" in param.name:
print(param.name)
params.append(param.name) params.append(param.name)
sensitivity( sensitivity(
...@@ -119,15 +124,15 @@ def compress(args): ...@@ -119,15 +124,15 @@ def compress(args):
sensitivities_file="sensitivities_1.data", sensitivities_file="sensitivities_1.data",
pruned_ratios=[0.5, 0.6, 0.7]) pruned_ratios=[0.5, 0.6, 0.7])
sens = merge_sensitive( sens = merge_sensitive(["./sensitivities_0.data", "./sensitivities_1.data"])
["./sensitivities_0.data", "./sensitivities_1.data"])
ratios = get_ratios_by_loss(sens, 0.01) ratios = get_ratios_by_loss(sens, 0.01)
print ratios print(sens)
def main(): def main():
paddle.enable_static()
args = parser.parse_args() args = parser.parse_args()
print_arguments(args) print_arguments(args)
compress(args) compress(args)
......
demo/sensitive_prune/greedy_prune.py 已删除 100644 → 0
浏览文件 @ bed39157
import os
import sys
import logging
import paddle
import argparse
import functools
import math
import time
import numpy as np
import paddle.fluid as fluid
from paddleslim.prune import SensitivePruner
from paddleslim.common import get_logger
from paddleslim.analysis import flops
sys.path.append(sys.path[0] + "/../")
import models
from utility import add_arguments, print_arguments
_logger = get_logger(__name__, level=logging.INFO)
parser = argparse.ArgumentParser(description=__doc__)
add_arg = functools.partial(add_arguments, argparser=parser)
# yapf: disable
add_arg('batch_size', int, 64 * 4, "Minibatch size.")
add_arg('use_gpu', bool, True, "Whether to use GPU or not.")
add_arg('model', str, "MobileNet", "The target model.")
add_arg('pretrained_model', str, "../pretrained_model/MobileNetV1_pretained", "Whether to use pretrained model.")
add_arg('lr', float, 0.1, "The learning rate used to fine-tune pruned model.")
add_arg('lr_strategy', str, "piecewise_decay", "The learning rate decay strategy.")
add_arg('l2_decay', float, 3e-5, "The l2_decay parameter.")
add_arg('momentum_rate', float, 0.9, "The value of momentum_rate.")
add_arg('num_epochs', int, 120, "The number of total epochs.")
add_arg('total_images', int, 1281167, "The number of total training images.")
parser.add_argument('--step_epochs', nargs='+', type=int, default=[30, 60, 90], help="piecewise decay step")
add_arg('config_file', str, None, "The config file for compression with yaml format.")
add_arg('data', str, "mnist", "Which data to use. 'mnist' or 'imagenet'")
add_arg('log_period', int, 10, "Log period in batches.")
add_arg('test_period', int, 10, "Test period in epoches.")
add_arg('checkpoints', str, "./checkpoints", "Checkpoints path.")
add_arg('prune_steps', int, 1000, "prune steps.")
add_arg('retrain_epoch', int, 5, "Retrain epoch.")
# yapf: enable
model_list = [m for m in dir(models) if "__" not in m]
def piecewise_decay(args):
step = int(math.ceil(float(args.total_images) / args.batch_size))
bd = [step * e for e in args.step_epochs]
lr = [args.lr * (0.1**i) for i in range(len(bd) + 1)]
learning_rate = fluid.layers.piecewise_decay(boundaries=bd, values=lr)
optimizer = fluid.optimizer.Momentum(
learning_rate=learning_rate,
momentum=args.momentum_rate,
regularization=fluid.regularizer.L2Decay(args.l2_decay))
return optimizer
def cosine_decay(args):
step = int(math.ceil(float(args.total_images) / args.batch_size))
learning_rate = fluid.layers.cosine_decay(
learning_rate=args.lr, step_each_epoch=step, epochs=args.num_epochs)
optimizer = fluid.optimizer.Momentum(
learning_rate=learning_rate,
momentum=args.momentum_rate,
regularization=fluid.regularizer.L2Decay(args.l2_decay))
return optimizer
def create_optimizer(args):
if args.lr_strategy == "piecewise_decay":
return piecewise_decay(args)
elif args.lr_strategy == "cosine_decay":
return cosine_decay(args)
def compress(args):
train_reader = None
test_reader = None
if args.data == "mnist":
import paddle.dataset.mnist as reader
train_reader = reader.train()
val_reader = reader.test()
class_dim = 10
image_shape = "1,28,28"
elif args.data == "imagenet":
import imagenet_reader as reader
train_reader = reader.train()
val_reader = reader.val()
class_dim = 1000
image_shape = "3,224,224"
else:
raise ValueError("{} is not supported.".format(args.data))
image_shape = [int(m) for m in image_shape.split(",")]
assert args.model in model_list, "{} is not in lists: {}".format(
args.model, model_list)
image = fluid.layers.data(name='image', shape=image_shape, dtype='float32')
label = fluid.layers.data(name='label', shape=[1], dtype='int64')
# model definition
model = models.__dict__[args.model]()
out = model.net(input=image, class_dim=class_dim)
cost = fluid.layers.cross_entropy(input=out, label=label)
avg_cost = fluid.layers.mean(x=cost)
acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1)
acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5)
val_program = fluid.default_main_program().clone(for_test=True)
opt = create_optimizer(args)
opt.minimize(avg_cost)
place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
if args.pretrained_model:
def if_exist(var):
return os.path.exists(
os.path.join(args.pretrained_model, var.name))
fluid.io.load_vars(exe, args.pretrained_model, predicate=if_exist)
val_reader = paddle.fluid.io.batch(val_reader, batch_size=args.batch_size)
train_reader = paddle.fluid.io.batch(
train_reader, batch_size=args.batch_size, drop_last=True)
train_feeder = feeder = fluid.DataFeeder([image, label], place)
val_feeder = feeder = fluid.DataFeeder(
[image, label], place, program=val_program)
def test(epoch, program):
batch_id = 0
acc_top1_ns = []
acc_top5_ns = []
for data in val_reader():
start_time = time.time()
acc_top1_n, acc_top5_n = exe.run(
program,
feed=train_feeder.feed(data),
fetch_list=[acc_top1.name, acc_top5.name])
end_time = time.time()
if batch_id % args.log_period == 0:
_logger.info(
"Eval epoch[{}] batch[{}] - acc_top1: {:.3f}; acc_top5: {:.3f}; time: {:.3f}".
format(epoch, batch_id,
np.mean(acc_top1_n),
np.mean(acc_top5_n), end_time - start_time))
acc_top1_ns.append(np.mean(acc_top1_n))
acc_top5_ns.append(np.mean(acc_top5_n))
batch_id += 1
_logger.info(
"Final eval epoch[{}] - acc_top1: {:.3f}; acc_top5: {:.3f}".format(
epoch,
np.mean(np.array(acc_top1_ns)), np.mean(
np.array(acc_top5_ns))))
return np.mean(np.array(acc_top1_ns))
def train(epoch, program):
build_strategy = fluid.BuildStrategy()
exec_strategy = fluid.ExecutionStrategy()
train_program = fluid.compiler.CompiledProgram(
program).with_data_parallel(
loss_name=avg_cost.name,
build_strategy=build_strategy,
exec_strategy=exec_strategy)
batch_id = 0
for data in train_reader():
start_time = time.time()
loss_n, acc_top1_n, acc_top5_n = exe.run(
train_program,
feed=train_feeder.feed(data),
fetch_list=[avg_cost.name, acc_top1.name, acc_top5.name])
end_time = time.time()
loss_n = np.mean(loss_n)
acc_top1_n = np.mean(acc_top1_n)
acc_top5_n = np.mean(acc_top5_n)
if batch_id % args.log_period == 0:
_logger.info(
"epoch[{}]-batch[{}] - loss: {:.3f}; acc_top1: {:.3f}; acc_top5: {:.3f}; time: {:.3f}".
format(epoch, batch_id, loss_n, acc_top1_n, acc_top5_n,
end_time - start_time))
batch_id += 1
params = []
for param in fluid.default_main_program().global_block().all_parameters():
if "_sep_weights" in param.name:
params.append(param.name)
def eval_func(program):
return test(0, program)
if args.data == "mnist":
train(0, fluid.default_main_program())
pruner = SensitivePruner(place, eval_func, checkpoints=args.checkpoints)
pruned_program, pruned_val_program, iter = pruner.restore()
if pruned_program is None:
pruned_program = fluid.default_main_program()
if pruned_val_program is None:
pruned_val_program = val_program
base_flops = flops(val_program)
start = iter
end = args.prune_steps
for iter in range(start, end):
pruned_program, pruned_val_program = pruner.greedy_prune(
pruned_program, pruned_val_program, params, 0.03, topk=1)
current_flops = flops(pruned_val_program)
print("iter:{}; pruned FLOPS: {}".format(
iter, float(base_flops - current_flops) / base_flops))
acc = None
for i in range(args.retrain_epoch):
train(i, pruned_program)
acc = test(i, pruned_val_program)
print("iter:{}; pruned FLOPS: {}; acc: {}".format(
iter, float(base_flops - current_flops) / base_flops, acc))
pruner.save_checkpoint(pruned_program, pruned_val_program)
def main():
args = parser.parse_args()
print_arguments(args)
compress(args)
if __name__ == '__main__':
main()
demo/sensitive_prune/prune.py 已删除 100644 → 0
浏览文件 @ bed39157
import os
import sys
import logging
import paddle
import argparse
import functools
import math
import time
import numpy as np
import paddle.fluid as fluid
from paddleslim.prune import SensitivePruner
from paddleslim.common import get_logger
from paddleslim.analysis import flops
sys.path.append(sys.path[0] + "/../")
import models
from utility import add_arguments, print_arguments
_logger = get_logger(__name__, level=logging.INFO)
parser = argparse.ArgumentParser(description=__doc__)
add_arg = functools.partial(add_arguments, argparser=parser)
# yapf: disable
add_arg('batch_size', int, 64 * 4, "Minibatch size.")
add_arg('use_gpu', bool, True, "Whether to use GPU or not.")
add_arg('model', str, "MobileNet", "The target model.")
add_arg('pretrained_model', str, "../pretrained_model/MobileNetV1_pretained", "Whether to use pretrained model.")
add_arg('lr', float, 0.1, "The learning rate used to fine-tune pruned model.")
add_arg('lr_strategy', str, "piecewise_decay", "The learning rate decay strategy.")
add_arg('l2_decay', float, 3e-5, "The l2_decay parameter.")
add_arg('momentum_rate', float, 0.9, "The value of momentum_rate.")
add_arg('num_epochs', int, 120, "The number of total epochs.")
add_arg('total_images', int, 1281167, "The number of total training images.")
parser.add_argument('--step_epochs', nargs='+', type=int, default=[30, 60, 90], help="piecewise decay step")
add_arg('config_file', str, None, "The config file for compression with yaml format.")
add_arg('data', str, "mnist", "Which data to use. 'mnist' or 'imagenet'")
add_arg('log_period', int, 10, "Log period in batches.")
add_arg('test_period', int, 10, "Test period in epoches.")
add_arg('checkpoints', str, "./checkpoints", "Checkpoints path.")
# yapf: enable
model_list = [m for m in dir(models) if "__" not in m]
def piecewise_decay(args):
step = int(math.ceil(float(args.total_images) / args.batch_size))
bd = [step * e for e in args.step_epochs]
lr = [args.lr * (0.1**i) for i in range(len(bd) + 1)]
learning_rate = fluid.layers.piecewise_decay(boundaries=bd, values=lr)
optimizer = fluid.optimizer.Momentum(
learning_rate=learning_rate,
momentum=args.momentum_rate,
regularization=fluid.regularizer.L2Decay(args.l2_decay))
return optimizer
def cosine_decay(args):
step = int(math.ceil(float(args.total_images) / args.batch_size))
learning_rate = fluid.layers.cosine_decay(
learning_rate=args.lr, step_each_epoch=step, epochs=args.num_epochs)
optimizer = fluid.optimizer.Momentum(
learning_rate=learning_rate,
momentum=args.momentum_rate,
regularization=fluid.regularizer.L2Decay(args.l2_decay))
return optimizer
def create_optimizer(args):
if args.lr_strategy == "piecewise_decay":
return piecewise_decay(args)
elif args.lr_strategy == "cosine_decay":
return cosine_decay(args)
def compress(args):
train_reader = None
test_reader = None
if args.data == "mnist":
import paddle.dataset.mnist as reader
train_reader = reader.train()
val_reader = reader.test()
class_dim = 10
image_shape = "1,28,28"
elif args.data == "imagenet":
import imagenet_reader as reader
train_reader = reader.train()
val_reader = reader.val()
class_dim = 1000
image_shape = "3,224,224"
else:
raise ValueError("{} is not supported.".format(args.data))
image_shape = [int(m) for m in image_shape.split(",")]
assert args.model in model_list, "{} is not in lists: {}".format(
args.model, model_list)
image = fluid.layers.data(name='image', shape=image_shape, dtype='float32')
label = fluid.layers.data(name='label', shape=[1], dtype='int64')
# model definition
model = models.__dict__[args.model]()
out = model.net(input=image, class_dim=class_dim)
cost = fluid.layers.cross_entropy(input=out, label=label)
avg_cost = fluid.layers.mean(x=cost)
acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1)
acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5)
val_program = fluid.default_main_program().clone(for_test=True)
opt = create_optimizer(args)
opt.minimize(avg_cost)
place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
if args.pretrained_model:
def if_exist(var):
return os.path.exists(
os.path.join(args.pretrained_model, var.name))
fluid.io.load_vars(exe, args.pretrained_model, predicate=if_exist)
val_reader = paddle.fluid.io.batch(val_reader, batch_size=args.batch_size)
train_reader = paddle.fluid.io.batch(
train_reader, batch_size=args.batch_size, drop_last=True)
train_feeder = feeder = fluid.DataFeeder([image, label], place)
val_feeder = feeder = fluid.DataFeeder(
[image, label], place, program=val_program)
def test(epoch, program):
batch_id = 0
acc_top1_ns = []
acc_top5_ns = []
for data in val_reader():
start_time = time.time()
acc_top1_n, acc_top5_n = exe.run(
program,
feed=train_feeder.feed(data),
fetch_list=[acc_top1.name, acc_top5.name])
end_time = time.time()
if batch_id % args.log_period == 0:
_logger.info(
"Eval epoch[{}] batch[{}] - acc_top1: {:.3f}; acc_top5: {:.3f}; time: {:.3f}".
format(epoch, batch_id,
np.mean(acc_top1_n),
np.mean(acc_top5_n), end_time - start_time))
acc_top1_ns.append(np.mean(acc_top1_n))
acc_top5_ns.append(np.mean(acc_top5_n))
batch_id += 1
_logger.info(
"Final eval epoch[{}] - acc_top1: {:.3f}; acc_top5: {:.3f}".format(
epoch,
np.mean(np.array(acc_top1_ns)), np.mean(
np.array(acc_top5_ns))))
return np.mean(np.array(acc_top1_ns))
def train(epoch, program):
build_strategy = fluid.BuildStrategy()
exec_strategy = fluid.ExecutionStrategy()
train_program = fluid.compiler.CompiledProgram(
program).with_data_parallel(
loss_name=avg_cost.name,
build_strategy=build_strategy,
exec_strategy=exec_strategy)
batch_id = 0
for data in train_reader():
start_time = time.time()
loss_n, acc_top1_n, acc_top5_n = exe.run(
train_program,
feed=train_feeder.feed(data),
fetch_list=[avg_cost.name, acc_top1.name, acc_top5.name])
end_time = time.time()
loss_n = np.mean(loss_n)
acc_top1_n = np.mean(acc_top1_n)
acc_top5_n = np.mean(acc_top5_n)
if batch_id % args.log_period == 0:
_logger.info(
"epoch[{}]-batch[{}] - loss: {:.3f}; acc_top1: {:.3f}; acc_top5: {:.3f}; time: {:.3f}".
format(epoch, batch_id, loss_n, acc_top1_n, acc_top5_n,
end_time - start_time))
batch_id += 1
params = []
for param in fluid.default_main_program().global_block().all_parameters():
if "_sep_weights" in param.name:
params.append(param.name)
def eval_func(program):
return test(0, program)
if args.data == "mnist":
train(0, fluid.default_main_program())
pruner = SensitivePruner(place, eval_func, checkpoints=args.checkpoints)
pruned_program, pruned_val_program, iter = pruner.restore()
if pruned_program is None:
pruned_program = fluid.default_main_program()
if pruned_val_program is None:
pruned_val_program = val_program
start = iter
end = 6
for iter in range(start, end):
pruned_program, pruned_val_program = pruner.prune(
pruned_program, pruned_val_program, params, 0.1)
train(iter, pruned_program)
test(iter, pruned_val_program)
pruner.save_checkpoint(pruned_program, pruned_val_program)
print("before flops: {}".format(flops(fluid.default_main_program())))
print("after flops: {}".format(flops(pruned_val_program)))
def main():
args = parser.parse_args()
print_arguments(args)
compress(args)
if __name__ == '__main__':
main()
paddleslim/prune/__init__.py
浏览文件 @ 99f7d05b
...@@ -17,8 +17,6 @@ from .pruner import * ...@@ -17,8 +17,6 @@ from .pruner import *
from ..prune import pruner from ..prune import pruner
from .auto_pruner import * from .auto_pruner import *
from ..prune import auto_pruner from ..prune import auto_pruner
from .sensitive_pruner import *
from ..prune import sensitive_pruner
from .sensitive import * from .sensitive import *
from ..prune import sensitive from ..prune import sensitive
from .prune_walker import * from .prune_walker import *
...@@ -36,7 +34,6 @@ __all__ = [] ...@@ -36,7 +34,6 @@ __all__ = []
__all__ += pruner.__all__ __all__ += pruner.__all__
__all__ += auto_pruner.__all__ __all__ += auto_pruner.__all__
__all__ += sensitive_pruner.__all__
__all__ += sensitive.__all__ __all__ += sensitive.__all__
__all__ += prune_walker.__all__ __all__ += prune_walker.__all__
__all__ += prune_io.__all__ __all__ += prune_io.__all__
......
paddleslim/prune/auto_pruner.py
浏览文件 @ 99f7d05b
...@@ -15,7 +15,6 @@ ...@@ -15,7 +15,6 @@
import socket import socket
import logging import logging
import numpy as np import numpy as np
import paddle.fluid as fluid
from .pruner import Pruner from .pruner import Pruner
from ..core import VarWrapper, OpWrapper, GraphWrapper from ..core import VarWrapper, OpWrapper, GraphWrapper
from ..common import SAController from ..common import SAController
......
paddleslim/prune/prune_io.py
浏览文件 @ 99f7d05b
import os import os
import paddle.fluid as fluid import paddle
from paddle.fluid import Program
from ..core import GraphWrapper from ..core import GraphWrapper
from ..common import get_logger from ..common import get_logger
import json import json
...@@ -23,9 +22,10 @@ def save_model(exe, graph, dirname): ...@@ -23,9 +22,10 @@ def save_model(exe, graph, dirname):
dirname(str): The directory that the model saved into. dirname(str): The directory that the model saved into.
""" """
assert graph is not None and dirname is not None assert graph is not None and dirname is not None
graph = GraphWrapper(graph) if isinstance(graph, Program) else graph graph = GraphWrapper(graph) if isinstance(graph,
paddle.static.Program) else graph
fluid.io.save_persistables( paddle.fluid.io.save_persistables(
executor=exe, executor=exe,
dirname=dirname, dirname=dirname,
main_program=graph.program, main_program=graph.program,
...@@ -33,7 +33,7 @@ def save_model(exe, graph, dirname): ...@@ -33,7 +33,7 @@ def save_model(exe, graph, dirname):
weights_file = dirname weights_file = dirname
_logger.info("Save model weights into {}".format(weights_file)) _logger.info("Save model weights into {}".format(weights_file))
shapes = {} shapes = {}
for var in fluid.io.get_program_persistable_vars(graph.program): for var in paddle.fluid.io.get_program_persistable_vars(graph.program):
shapes[var.name] = var.shape shapes[var.name] = var.shape
SHAPES_FILE = os.path.join(dirname, _SHAPES_FILE) SHAPES_FILE = os.path.join(dirname, _SHAPES_FILE)
with open(SHAPES_FILE, "w") as f: with open(SHAPES_FILE, "w") as f:
...@@ -50,7 +50,8 @@ def load_model(exe, graph, dirname): ...@@ -50,7 +50,8 @@ def load_model(exe, graph, dirname):
dirname(str): The directory that the model will be loaded. dirname(str): The directory that the model will be loaded.
""" """
assert graph is not None and dirname is not None assert graph is not None and dirname is not None
graph = GraphWrapper(graph) if isinstance(graph, Program) else graph graph = GraphWrapper(graph) if isinstance(graph,
paddle.static.Program) else graph
SHAPES_FILE = os.path.join(dirname, _SHAPES_FILE) SHAPES_FILE = os.path.join(dirname, _SHAPES_FILE)
_logger.info("Load shapes of weights from {}".format(SHAPES_FILE)) _logger.info("Load shapes of weights from {}".format(SHAPES_FILE))
...@@ -64,7 +65,7 @@ def load_model(exe, graph, dirname): ...@@ -64,7 +65,7 @@ def load_model(exe, graph, dirname):
_logger.info('{} is not loaded'.format(param_name)) _logger.info('{} is not loaded'.format(param_name))
_logger.info("Load shapes of weights from {}".format(SHAPES_FILE)) _logger.info("Load shapes of weights from {}".format(SHAPES_FILE))
fluid.io.load_persistables( paddle.fluid.io.load_persistables(
executor=exe, executor=exe,
dirname=dirname, dirname=dirname,
main_program=graph.program, main_program=graph.program,
......
paddleslim/prune/pruner.py
浏览文件 @ 99f7d05b
...@@ -14,10 +14,9 @@ ...@@ -14,10 +14,9 @@
import logging import logging
import sys import sys
import copy
import numpy as np import numpy as np
from functools import reduce from functools import reduce
import paddle.fluid as fluid
import copy
from ..core import VarWrapper, OpWrapper, GraphWrapper from ..core import VarWrapper, OpWrapper, GraphWrapper
from .group_param import collect_convs from .group_param import collect_convs
from .criterion import CRITERION from .criterion import CRITERION
...@@ -38,8 +37,7 @@ class Pruner(): ...@@ -38,8 +37,7 @@ class Pruner():
""" """
def __init__(self, def __init__(self, criterion="l1_norm",
criterion="l1_norm",
idx_selector="default_idx_selector"): idx_selector="default_idx_selector"):
if isinstance(criterion, str): if isinstance(criterion, str):
self.criterion = CRITERION.get(criterion) self.criterion = CRITERION.get(criterion)
...@@ -93,8 +91,8 @@ class Pruner(): ...@@ -93,8 +91,8 @@ class Pruner():
_logger.info("pruning: {}".format(param)) _logger.info("pruning: {}".format(param))
if graph.var(param) is None: if graph.var(param) is None:
_logger.warn( _logger.warn(
"Variable[{}] to be pruned is not in current graph.". "Variable[{}] to be pruned is not in current graph.".format(
format(param)) param))
continue continue
group = collect_convs([param], graph, group = collect_convs([param], graph,
visited)[0] # [(name, axis, pruned_idx)] visited)[0] # [(name, axis, pruned_idx)]
......
paddleslim/prune/sensitive.py
浏览文件 @ 99f7d05b
...@@ -17,7 +17,7 @@ import os ...@@ -17,7 +17,7 @@ import os
import logging import logging
import pickle import pickle
import numpy as np import numpy as np
import paddle.fluid as fluid import paddle
from ..core import GraphWrapper from ..core import GraphWrapper
from ..common import get_logger from ..common import get_logger
from ..analysis import flops from ..analysis import flops
...@@ -26,8 +26,7 @@ from ..prune import Pruner ...@@ -26,8 +26,7 @@ from ..prune import Pruner
_logger = get_logger(__name__, level=logging.INFO) _logger = get_logger(__name__, level=logging.INFO)
__all__ = [ __all__ = [
"sensitivity", "flops_sensitivity", "load_sensitivities", "merge_sensitive", "sensitivity", "load_sensitivities", "merge_sensitive", "get_ratios_by_loss"
"get_ratios_by_loss"
] ]
...@@ -68,7 +67,7 @@ def sensitivity(program, ...@@ -68,7 +67,7 @@ def sensitivity(program,
Returns: Returns:
dict: A dict storing sensitivities. dict: A dict storing sensitivities.
""" """
scope = fluid.global_scope() scope = paddle.static.global_scope()
graph = GraphWrapper(program) graph = GraphWrapper(program)
sensitivities = load_sensitivities(sensitivities_file) sensitivities = load_sensitivities(sensitivities_file)
...@@ -121,80 +120,6 @@ def sensitivity(program, ...@@ -121,80 +120,6 @@ def sensitivity(program,
return sensitivities return sensitivities
def flops_sensitivity(program,
place,
param_names,
eval_func,
sensitivities_file=None,
pruned_flops_rate=0.1):
assert (1.0 / len(param_names) > pruned_flops_rate)
scope = fluid.global_scope()
graph = GraphWrapper(program)
sensitivities = load_sensitivities(sensitivities_file)
for name in param_names:
if name not in sensitivities:
sensitivities[name] = {}
base_flops = flops(program)
target_pruned_flops = base_flops * pruned_flops_rate
pruner = Pruner()
baseline = None
for name in sensitivities:
pruned_program, _, _ = pruner.prune(
program=graph.program,
scope=None,
params=[name],
ratios=[0.5],
place=None,
lazy=False,
only_graph=True)
param_flops = (base_flops - flops(pruned_program)) * 2
channel_size = graph.var(name).shape()[0]
pruned_ratio = target_pruned_flops / float(param_flops)
pruned_ratio = round(pruned_ratio, 3)
pruned_size = round(pruned_ratio * channel_size)
pruned_ratio = 1 if pruned_size >= channel_size else pruned_ratio
if len(sensitivities[name].keys()) > 0:
_logger.debug(
'{} exist; pruned ratio: {}; excepted ratio: {}'.format(
name, sensitivities[name].keys(), pruned_ratio))
continue
if baseline is None:
baseline = eval_func(graph.program)
param_backup = {}
pruner = Pruner()
_logger.info("sensitive - param: {}; ratios: {}".format(name,
pruned_ratio))
loss = 1
if pruned_ratio < 1:
pruned_program = pruner.prune(
program=graph.program,
scope=scope,
params=[name],
ratios=[pruned_ratio],
place=place,
lazy=True,
only_graph=False,
param_backup=param_backup)
pruned_metric = eval_func(pruned_program)
loss = (baseline - pruned_metric) / baseline
_logger.info("pruned param: {}; {}; loss={}".format(name, pruned_ratio,
loss))
sensitivities[name][pruned_ratio] = loss
_save_sensitivities(sensitivities, sensitivities_file)
# restore pruned parameters
for param_name in param_backup.keys():
param_t = scope.find_var(param_name).get_tensor()
param_t.set(param_backup[param_name], place)
return sensitivities
def merge_sensitive(sensitivities): def merge_sensitive(sensitivities):
"""Merge sensitivities. """Merge sensitivities.
...@@ -206,7 +131,7 @@ def merge_sensitive(sensitivities): ...@@ -206,7 +131,7 @@ def merge_sensitive(sensitivities):
""" """
assert len(sensitivities) > 0 assert len(sensitivities) > 0
if not isinstance(sensitivities[0], dict): if not isinstance(sensitivities[0], dict):
sensitivities = [pickle.load(open(sen, 'r')) for sen in sensitivities] sensitivities = [load_sensitivities(sen) for sen in sensitivities]
new_sensitivities = {} new_sensitivities = {}
for sen in sensitivities: for sen in sensitivities:
......
paddleslim/prune/sensitive_pruner.py 已删除 100644 → 0
浏览文件 @ bed39157
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import logging
import copy
from scipy.optimize import leastsq
import numpy as np
import paddle.fluid as fluid
from ..common import get_logger
from .sensitive import sensitivity
from .sensitive import flops_sensitivity, get_ratios_by_loss
from ..analysis import flops
from .pruner import Pruner
__all__ = ["SensitivePruner"]
_logger = get_logger(__name__, level=logging.INFO)
class SensitivePruner(object):
"""
Pruner used to prune parameters iteratively according to sensitivities
of parameters in each step.
Args:
place(fluid.CUDAPlace | fluid.CPUPlace): The device place where
program execute.
eval_func(function): A callback function used to evaluate pruned
program. The argument of this function is pruned program.
And it return a score of given program.
scope(fluid.scope): The scope used to execute program.
"""
def __init__(self, place, eval_func, scope=None, checkpoints=None):
self._eval_func = eval_func
self._iter = 0
self._place = place
self._scope = fluid.global_scope() if scope is None else scope
self._pruner = Pruner()
self._checkpoints = checkpoints
def save_checkpoint(self, train_program, eval_program):
checkpoint = os.path.join(self._checkpoints, str(self._iter - 1))
exe = fluid.Executor(self._place)
fluid.io.save_persistables(
exe, checkpoint, main_program=train_program, filename="__params__")
with open(checkpoint + "/main_program", "wb") as f:
f.write(train_program.desc.serialize_to_string())
with open(checkpoint + "/eval_program", "wb") as f:
f.write(eval_program.desc.serialize_to_string())
def restore(self, checkpoints=None):
exe = fluid.Executor(self._place)
checkpoints = self._checkpoints if checkpoints is None else checkpoints
_logger.info("check points: {}".format(checkpoints))
main_program = None
eval_program = None
if checkpoints is not None:
cks = [dir for dir in os.listdir(checkpoints)]
if len(cks) > 0:
latest = max([int(ck) for ck in cks])
latest_ck_path = os.path.join(checkpoints, str(latest))
self._iter += 1
with open(latest_ck_path + "/main_program", "rb") as f:
program_desc_str = f.read()
main_program = fluid.Program.parse_from_string(
program_desc_str)
with open(latest_ck_path + "/eval_program", "rb") as f:
program_desc_str = f.read()
eval_program = fluid.Program.parse_from_string(
program_desc_str)
with fluid.scope_guard(self._scope):
fluid.io.load_persistables(exe, latest_ck_path,
main_program, "__params__")
_logger.info("load checkpoint from: {}".format(latest_ck_path))
_logger.info("flops of eval program: {}".format(
flops(eval_program)))
return main_program, eval_program, self._iter
def greedy_prune(self,
train_program,
eval_program,
params,
pruned_flops_rate,
topk=1):
sensitivities_file = "greedy_sensitivities_iter{}.data".format(
self._iter)
with fluid.scope_guard(self._scope):
sensitivities = flops_sensitivity(
eval_program,
self._place,
params,
self._eval_func,
sensitivities_file=sensitivities_file,
pruned_flops_rate=pruned_flops_rate)
_logger.info(sensitivities)
params, ratios = self._greedy_ratio_by_sensitive(sensitivities, topk)
_logger.info("Pruning: {} by {}".format(params, ratios))
pruned_program = self._pruner.prune(
train_program,
self._scope,
params,
ratios,
place=self._place,
only_graph=False)
pruned_val_program = None
if eval_program is not None:
pruned_val_program = self._pruner.prune(
eval_program,
self._scope,
params,
ratios,
place=self._place,
only_graph=True)
self._iter += 1
return pruned_program, pruned_val_program
def prune(self, train_program, eval_program, params, pruned_flops):
"""
Pruning parameters of training and evaluation network by sensitivities in current step.
Args:
train_program(fluid.Program): The training program to be pruned.
eval_program(fluid.Program): The evaluation program to be pruned. And it is also used to calculate sensitivities of parameters.
params(list<str>): The parameters to be pruned.
pruned_flops(float): The ratio of FLOPS to be pruned in current step.
Returns:
tuple: A tuple of pruned training program and pruned evaluation program.
"""
_logger.info("Pruning: {}".format(params))
sensitivities_file = "sensitivities_iter{}.data".format(self._iter)
with fluid.scope_guard(self._scope):
sensitivities = sensitivity(
eval_program,
self._place,
params,
self._eval_func,
sensitivities_file=sensitivities_file,
step_size=0.1)
_logger.info(sensitivities)
_, ratios = self.get_ratios_by_sensitive(sensitivities, pruned_flops,
eval_program)
pruned_program = self._pruner.prune(
train_program,
self._scope,
params,
ratios,
place=self._place,
only_graph=False)
pruned_val_program = None
if eval_program is not None:
pruned_val_program = self._pruner.prune(
eval_program,
self._scope,
params,
ratios,
place=self._place,
only_graph=True)
self._iter += 1
return pruned_program, pruned_val_program
def _greedy_ratio_by_sensitive(self, sensitivities, topk=1):
losses = {}
percents = {}
for param in sensitivities:
losses[param] = sensitivities[param]['loss'][0]
percents[param] = sensitivities[param]['pruned_percent'][0]
topk_parms = sorted(losses, key=losses.__getitem__)[:topk]
topk_percents = [percents[param] for param in topk_parms]
return topk_parms, topk_percents
def get_ratios_by_sensitive(self, sensitivities, pruned_flops,
eval_program):
"""
Search a group of ratios for pruning target flops.
Args:
sensitivities(dict): The sensitivities used to generate a group of pruning ratios. The key of dict
is name of parameters to be pruned. The value of dict is a list of tuple with
format `(pruned_ratio, accuracy_loss)`.
pruned_flops(float): The percent of FLOPS to be pruned.
eval_program(Program): The program whose FLOPS is considered.
Returns:
dict: A group of ratios. The key of dict is name of parameters while the value is the ratio to be pruned.
"""
min_loss = 0.
max_loss = 0.
# step 2: Find a group of ratios by binary searching.
base_flops = flops(eval_program)
ratios = None
max_times = 20
while min_loss < max_loss and max_times > 0:
loss = (max_loss + min_loss) / 2
_logger.info(
'-----------Try pruned ratios while acc loss={}-----------'.
format(loss))
ratios = self.get_ratios_by_loss(sensitivities, loss)
_logger.info('Pruned ratios={}'.format(
[round(ratio, 3) for ratio in ratios.values()]))
pruned_program = self._pruner.prune(
eval_program,
None, # scope
ratios.keys(),
ratios.values(),
None, # place
only_graph=True)
pruned_ratio = 1 - (float(flops(pruned_program)) / base_flops)
_logger.info('Pruned flops: {:.4f}'.format(pruned_ratio))
# Check whether current ratios is enough
if abs(pruned_ratio - pruned_flops) < 0.015:
break
if pruned_ratio > pruned_flops:
max_loss = loss
else:
min_loss = loss
max_times -= 1
return ratios
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或