From 99f7d05b93b113ff6574c4b28e13aec9843f71f7 Mon Sep 17 00:00:00 2001 From: yukavio <67678385+yukavio@users.noreply.github.com> Date: Tue, 3 Nov 2020 13:13:00 +0800 Subject: [PATCH] Migrate prune (#492) * remove sensitive_pruner module * migrate prune module --- demo/auto_prune/README.md | 63 ------- demo/auto_prune/train.py | 220 ----------------------- demo/auto_prune/train_finetune.py | 225 ------------------------ demo/auto_prune/train_iterator.py | 252 --------------------------- demo/prune/eval.py | 40 ++--- demo/prune/train.py | 97 ++++++----- demo/sensitive/train.py | 59 ++++--- demo/sensitive_prune/greedy_prune.py | 231 ------------------------ demo/sensitive_prune/prune.py | 223 ------------------------ paddleslim/prune/__init__.py | 3 - paddleslim/prune/auto_pruner.py | 1 - paddleslim/prune/prune_io.py | 15 +- paddleslim/prune/pruner.py | 10 +- paddleslim/prune/sensitive.py | 83 +-------- paddleslim/prune/sensitive_pruner.py | 243 -------------------------- 15 files changed, 118 insertions(+), 1647 deletions(-) delete mode 100644 demo/auto_prune/README.md delete mode 100644 demo/auto_prune/train.py delete mode 100644 demo/auto_prune/train_finetune.py delete mode 100644 demo/auto_prune/train_iterator.py delete mode 100644 demo/sensitive_prune/greedy_prune.py delete mode 100644 demo/sensitive_prune/prune.py delete mode 100644 paddleslim/prune/sensitive_pruner.py diff --git a/demo/auto_prune/README.md b/demo/auto_prune/README.md deleted file mode 100644 index 0f6b5bb8..00000000 --- a/demo/auto_prune/README.md +++ /dev/null @@ -1,63 +0,0 @@ -该示例介绍如何使用自动裁剪。 -该示例需要使用IMAGENET数据，以及预训练模型。支持以下模型： - -- MobileNetV1 -- MobileNetV2 -- ResNet50 - -## 1. 接口介绍 - -该示例涉及以下接口： - -- [paddleslim.prune.AutoPruner]) -- [paddleslim.prune.Pruner]) - -## 2. 运行示例 - - -提供两种自动裁剪模式，直接以裁剪目标进行一次自动裁剪，和多次迭代的方式进行裁剪。 - -###2.1一次裁剪 - -在路径`PaddleSlim/demo/auto_prune`下执行以下代码运行示例： - -``` -export CUDA_VISIBLE_DEVICES=0 -python train.py --model "MobileNet" -从log中获取搜索的最佳裁剪率列表，加入到train_finetune.py的ratiolist中，如下命令finetune得到最终结果 -python train_finetune.py --model "MobileNet" --lr 0.1 --num_epochs 120 --step_epochs 30 60 90 - -``` - -通过`python train.py --help`查看更多选项。 - - -###2.2多次迭代裁剪 - -在路径`PaddleSlim/demo/auto_prune`下执行以下代码运行示例： - -``` -export CUDA_VISIBLE_DEVICES=0 -python train_iterator.py --model "MobileNet" -从log中获取本次迭代搜索的最佳裁剪率列表，加入到train_finetune.py的ratiolist中,如下命令开始finetune本次搜索到的结果 -python train_finetune.py --model "MobileNet" -将第一次迭代的最佳裁剪率列表，加入到train_iterator.py 的ratiolist中,如下命令进行第二次迭代 -python train_iterator.py --model "MobileNet" --pretrained_model "checkpoint/Mobilenet/19" -finetune第二次迭代搜索结果，并继续重复迭代，直到获得目标裁剪率的结果 -... -如下命令finetune最终结果 -python train_finetune.py --model "MobileNet" --pretrained_model "checkpoint/Mobilenet/19" --num_epochs 70 --step_epochs 10 40 -``` - - -## 3. 注意 - -### 3.1 一次裁剪 - -在`paddleslim.prune.AutoPruner`接口的参数中，pruned_flops表示期望的最低flops剪切率。 - - -### 3.2 多次迭代裁剪 - -单次迭代的裁剪目标，建议不高于10%。 -在load前次迭代结果时，需要删除checkpoint下learning_rate、@LR_DECAY_COUNTER@等文件，避免继承之前的learning_rate，影响finetune效果。 diff --git a/demo/auto_prune/train.py b/demo/auto_prune/train.py deleted file mode 100644 index f0b2a2b0..00000000 --- a/demo/auto_prune/train.py +++ /dev/null @@ -1,220 +0,0 @@ -import os -import sys -import logging -import paddle -import argparse -import functools -import math -import time -import numpy as np -import paddle.fluid as fluid -sys.path[0] = os.path.join(os.path.dirname("__file__"), os.path.pardir) -from paddleslim.prune import AutoPruner -from paddleslim.common import get_logger -from paddleslim.analysis import flops -import models -from utility import add_arguments, print_arguments - -_logger = get_logger(__name__, level=logging.INFO) - -parser = argparse.ArgumentParser(description=__doc__) -add_arg = functools.partial(add_arguments, argparser=parser) -# yapf: disable -add_arg('batch_size', int, 64 * 4, "Minibatch size.") -add_arg('use_gpu', bool, True, "Whether to use GPU or not.") -add_arg('model', str, "MobileNet", "The target model.") -add_arg('pretrained_model', str, "../pretrained_model/MobileNetV1_pretrained", "Whether to use pretrained model.") -add_arg('lr', float, 0.1, "The learning rate used to fine-tune pruned model.") -add_arg('lr_strategy', str, "piecewise_decay", "The learning rate decay strategy.") -add_arg('l2_decay', float, 3e-5, "The l2_decay parameter.") -add_arg('momentum_rate', float, 0.9, "The value of momentum_rate.") -add_arg('num_epochs', int, 120, "The number of total epochs.") -add_arg('total_images', int, 1281167, "The number of total training images.") -parser.add_argument('--step_epochs', nargs='+', type=int, default=[30, 60, 90], help="piecewise decay step") -add_arg('config_file', str, None, "The config file for compression with yaml format.") -add_arg('data', str, "imagenet", "Which data to use. 'mnist' or 'imagenet'") -add_arg('log_period', int, 10, "Log period in batches.") -add_arg('test_period', int, 10, "Test period in epoches.") -# yapf: enable - -model_list = [m for m in dir(models) if "__" not in m] - - -def piecewise_decay(args): - step = int(math.ceil(float(args.total_images) / args.batch_size)) - bd = [step * e for e in args.step_epochs] - lr = [args.lr * (0.1**i) for i in range(len(bd) + 1)] - learning_rate = fluid.layers.piecewise_decay(boundaries=bd, values=lr) - optimizer = fluid.optimizer.Momentum( - learning_rate=learning_rate, - momentum=args.momentum_rate, - regularization=fluid.regularizer.L2Decay(args.l2_decay)) - return optimizer - - -def cosine_decay(args): - step = int(math.ceil(float(args.total_images) / args.batch_size)) - learning_rate = fluid.layers.cosine_decay( - learning_rate=args.lr, step_each_epoch=step, epochs=args.num_epochs) - optimizer = fluid.optimizer.Momentum( - learning_rate=learning_rate, - momentum=args.momentum_rate, - regularization=fluid.regularizer.L2Decay(args.l2_decay)) - return optimizer - - -def create_optimizer(args): - if args.lr_strategy == "piecewise_decay": - return piecewise_decay(args) - elif args.lr_strategy == "cosine_decay": - return cosine_decay(args) - - -def compress(args): - - train_reader = None - test_reader = None - if args.data == "mnist": - import paddle.dataset.mnist as reader - train_reader = reader.train() - val_reader = reader.test() - class_dim = 10 - image_shape = "1,28,28" - elif args.data == "imagenet": - import imagenet_reader as reader - train_reader = reader.train() - val_reader = reader.val() - class_dim = 1000 - image_shape = "3,224,224" - else: - raise ValueError("{} is not supported.".format(args.data)) - - image_shape = [int(m) for m in image_shape.split(",")] - assert args.model in model_list, "{} is not in lists: {}".format(args.model, - model_list) - image = fluid.layers.data(name='image', shape=image_shape, dtype='float32') - label = fluid.layers.data(name='label', shape=[1], dtype='int64') - # model definition - model = models.__dict__[args.model]() - out = model.net(input=image, class_dim=class_dim) - cost = fluid.layers.cross_entropy(input=out, label=label) - avg_cost = fluid.layers.mean(x=cost) - acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1) - acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5) - val_program = fluid.default_main_program().clone(for_test=True) - opt = create_optimizer(args) - opt.minimize(avg_cost) - place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - - if args.pretrained_model: - - def if_exist(var): - return os.path.exists(os.path.join(args.pretrained_model, var.name)) - - fluid.io.load_vars(exe, args.pretrained_model, predicate=if_exist) - - val_reader = paddle.fluid.io.batch(val_reader, batch_size=args.batch_size) - train_reader = paddle.fluid.io.batch( - train_reader, batch_size=args.batch_size, drop_last=True) - - train_feeder = feeder = fluid.DataFeeder([image, label], place) - val_feeder = feeder = fluid.DataFeeder( - [image, label], place, program=val_program) - - def test(epoch, program): - batch_id = 0 - acc_top1_ns = [] - acc_top5_ns = [] - for data in val_reader(): - start_time = time.time() - acc_top1_n, acc_top5_n = exe.run( - program, - feed=train_feeder.feed(data), - fetch_list=[acc_top1.name, acc_top5.name]) - end_time = time.time() - if batch_id % args.log_period == 0: - _logger.info( - "Eval epoch[{}] batch[{}] - acc_top1: {}; acc_top5: {}; time: {}". - format(epoch, batch_id, - np.mean(acc_top1_n), - np.mean(acc_top5_n), end_time - start_time)) - acc_top1_ns.append(np.mean(acc_top1_n)) - acc_top5_ns.append(np.mean(acc_top5_n)) - batch_id += 1 - - _logger.info("Final eval epoch[{}] - acc_top1: {}; acc_top5: {}".format( - epoch, - np.mean(np.array(acc_top1_ns)), np.mean(np.array(acc_top5_ns)))) - return np.mean(np.array(acc_top1_ns)) - - def train(epoch, program): - - build_strategy = fluid.BuildStrategy() - exec_strategy = fluid.ExecutionStrategy() - train_program = fluid.compiler.CompiledProgram( - program).with_data_parallel( - loss_name=avg_cost.name, - build_strategy=build_strategy, - exec_strategy=exec_strategy) - - batch_id = 0 - for data in train_reader(): - start_time = time.time() - loss_n, acc_top1_n, acc_top5_n = exe.run( - train_program, - feed=train_feeder.feed(data), - fetch_list=[avg_cost.name, acc_top1.name, acc_top5.name]) - end_time = time.time() - loss_n = np.mean(loss_n) - acc_top1_n = np.mean(acc_top1_n) - acc_top5_n = np.mean(acc_top5_n) - if batch_id % args.log_period == 0: - _logger.info( - "epoch[{}]-batch[{}] - loss: {}; acc_top1: {}; acc_top5: {}; time: {}". - format(epoch, batch_id, loss_n, acc_top1_n, acc_top5_n, - end_time - start_time)) - batch_id += 1 - - params = [] - for param in fluid.default_main_program().global_block().all_parameters(): - if "_sep_weights" in param.name: - params.append(param.name) - - pruner = AutoPruner( - val_program, - fluid.global_scope(), - place, - params=params, - init_ratios=[0.33] * len(params), - pruned_flops=0.5, - pruned_latency=None, - server_addr=("", 0), - init_temperature=100, - reduce_rate=0.85, - max_try_times=300, - max_client_num=10, - search_steps=100, - max_ratios=0.9, - min_ratios=0., - is_server=True, - key="auto_pruner") - - while True: - pruned_program, pruned_val_program = pruner.prune( - fluid.default_main_program(), val_program) - for i in range(1): - train(i, pruned_program) - score = test(0, pruned_val_program) - pruner.reward(score) - - -def main(): - args = parser.parse_args() - print_arguments(args) - compress(args) - - -if __name__ == '__main__': - main() diff --git a/demo/auto_prune/train_finetune.py b/demo/auto_prune/train_finetune.py deleted file mode 100644 index bc56ffa0..00000000 --- a/demo/auto_prune/train_finetune.py +++ /dev/null @@ -1,225 +0,0 @@ -import os -import sys -import logging -import paddle -import argparse -import functools -import math -import paddle.fluid as fluid -import imagenet_reader as reader -import models -from utility import add_arguments, print_arguments -import numpy as np -import time -from paddleslim.prune import Pruner -from paddleslim.analysis import flops - -parser = argparse.ArgumentParser(description=__doc__) -add_arg = functools.partial(add_arguments, argparser=parser) -# yapf: disable -add_arg('batch_size', int, 64 * 4, "Minibatch size.") -add_arg('use_gpu', bool, True, "Whether to use GPU or not.") -add_arg('model', str, "MobileNet", "The target model.") -add_arg('model_save_dir', str, "./", "checkpoint model.") -add_arg('pretrained_model', str, "../pretrained_model/MobileNetV1_pretrained", "Whether to use pretrained model.") -add_arg('lr', float, 0.01, "The learning rate used to fine-tune pruned model.") -add_arg('lr_strategy', str, "piecewise_decay", "The learning rate decay strategy.") -add_arg('l2_decay', float, 3e-5, "The l2_decay parameter.") -add_arg('momentum_rate', float, 0.9, "The value of momentum_rate.") -add_arg('num_epochs', int, 20, "The number of total epochs.") -add_arg('total_images', int, 1281167, "The number of total training images.") -parser.add_argument('--step_epochs', nargs='+', type=int, default=[5, 15], help="piecewise decay step") -add_arg('config_file', str, None, "The config file for compression with yaml format.") -# yapf: enable - -model_list = [m for m in dir(models) if "__" not in m] -ratiolist = [ - # [0.06, 0.0, 0.09, 0.03, 0.09, 0.02, 0.05, 0.03, 0.0, 0.07, 0.07, 0.05, 0.08], - # [0.08, 0.02, 0.03, 0.13, 0.1, 0.06, 0.03, 0.04, 0.14, 0.02, 0.03, 0.02, 0.01], -] - - -def save_model(args, exe, train_prog, eval_prog, info): - model_path = os.path.join(args.model_save_dir, args.model, str(info)) - if not os.path.isdir(model_path): - os.makedirs(model_path) - fluid.io.save_persistables(exe, model_path, main_program=train_prog) - print("Already save model in %s" % (model_path)) - - -def piecewise_decay(args): - step = int(math.ceil(float(args.total_images) / args.batch_size)) - bd = [step * e for e in args.step_epochs] - lr = [args.lr * (0.1**i) for i in range(len(bd) + 1)] - learning_rate = fluid.layers.piecewise_decay(boundaries=bd, values=lr) - optimizer = fluid.optimizer.Momentum( - learning_rate=learning_rate, - momentum=args.momentum_rate, - regularization=fluid.regularizer.L2Decay(args.l2_decay)) - return optimizer - - -def cosine_decay(args): - step = int(math.ceil(float(args.total_images) / args.batch_size)) - learning_rate = fluid.layers.cosine_decay( - learning_rate=args.lr, step_each_epoch=step, epochs=args.num_epochs) - optimizer = fluid.optimizer.Momentum( - learning_rate=learning_rate, - momentum=args.momentum_rate, - regularization=fluid.regularizer.L2Decay(args.l2_decay)) - return optimizer - - -def create_optimizer(args): - if args.lr_strategy == "piecewise_decay": - return piecewise_decay(args) - elif args.lr_strategy == "cosine_decay": - return cosine_decay(args) - - -def compress(args): - class_dim = 1000 - image_shape = "3,224,224" - image_shape = [int(m) for m in image_shape.split(",")] - assert args.model in model_list, "{} is not in lists: {}".format(args.model, - model_list) - image = fluid.layers.data(name='image', shape=image_shape, dtype='float32') - label = fluid.layers.data(name='label', shape=[1], dtype='int64') - # model definition - model = models.__dict__[args.model]() - out = model.net(input=image, class_dim=class_dim) - cost = fluid.layers.cross_entropy(input=out, label=label) - avg_cost = fluid.layers.mean(x=cost) - acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1) - acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5) - val_program = fluid.default_main_program().clone(for_test=True) - opt = create_optimizer(args) - opt.minimize(avg_cost) - place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - - if args.pretrained_model: - - def if_exist(var): - exist = os.path.exists( - os.path.join(args.pretrained_model, var.name)) - print("exist", exist) - return exist - - #fluid.io.load_vars(exe, args.pretrained_model, predicate=if_exist) - - val_reader = paddle.fluid.io.batch(reader.val(), batch_size=args.batch_size) - train_reader = paddle.fluid.io.batch( - reader.train(), batch_size=args.batch_size, drop_last=True) - - train_feeder = feeder = fluid.DataFeeder([image, label], place) - val_feeder = feeder = fluid.DataFeeder( - [image, label], place, program=val_program) - - def test(epoch, program): - batch_id = 0 - acc_top1_ns = [] - acc_top5_ns = [] - for data in val_reader(): - start_time = time.time() - acc_top1_n, acc_top5_n = exe.run( - program, - feed=train_feeder.feed(data), - fetch_list=[acc_top1.name, acc_top5.name]) - end_time = time.time() - print( - "Eval epoch[{}] batch[{}] - acc_top1: {}; acc_top5: {}; time: {}". - format(epoch, batch_id, - np.mean(acc_top1_n), - np.mean(acc_top5_n), end_time - start_time)) - acc_top1_ns.append(np.mean(acc_top1_n)) - acc_top5_ns.append(np.mean(acc_top5_n)) - batch_id += 1 - - print("Final eval epoch[{}] - acc_top1: {}; acc_top5: {}".format( - epoch, - np.mean(np.array(acc_top1_ns)), np.mean(np.array(acc_top5_ns)))) - - def train(epoch, program): - - build_strategy = fluid.BuildStrategy() - exec_strategy = fluid.ExecutionStrategy() - train_program = fluid.compiler.CompiledProgram( - program).with_data_parallel( - loss_name=avg_cost.name, - build_strategy=build_strategy, - exec_strategy=exec_strategy) - - batch_id = 0 - for data in train_reader(): - start_time = time.time() - loss_n, acc_top1_n, acc_top5_n, lr_n = exe.run( - train_program, - feed=train_feeder.feed(data), - fetch_list=[ - avg_cost.name, acc_top1.name, acc_top5.name, "learning_rate" - ]) - end_time = time.time() - loss_n = np.mean(loss_n) - acc_top1_n = np.mean(acc_top1_n) - acc_top5_n = np.mean(acc_top5_n) - lr_n = np.mean(lr_n) - print( - "epoch[{}]-batch[{}] - loss: {}; acc_top1: {}; acc_top5: {};lrn: {}; time: {}". - format(epoch, batch_id, loss_n, acc_top1_n, acc_top5_n, lr_n, - end_time - start_time)) - batch_id += 1 - - params = [] - for param in fluid.default_main_program().global_block().all_parameters(): - #if "_weights" in param.name and "conv1_weights" not in param.name: - if "_sep_weights" in param.name: - params.append(param.name) - print("fops before pruning: {}".format(flops(fluid.default_main_program()))) - pruned_program_iter = fluid.default_main_program() - pruned_val_program_iter = val_program - for ratios in ratiolist: - pruner = Pruner() - pruned_val_program_iter = pruner.prune( - pruned_val_program_iter, - fluid.global_scope(), - params=params, - ratios=ratios, - place=place, - only_graph=True) - - pruned_program_iter = pruner.prune( - pruned_program_iter, - fluid.global_scope(), - params=params, - ratios=ratios, - place=place) - print("fops after pruning: {}".format(flops(pruned_program_iter))) - """ do not inherit learning rate """ - if (os.path.exists(args.pretrained_model + "/learning_rate")): - os.remove(args.pretrained_model + "/learning_rate") - if (os.path.exists(args.pretrained_model + "/@LR_DECAY_COUNTER@")): - os.remove(args.pretrained_model + "/@LR_DECAY_COUNTER@") - fluid.io.load_vars( - exe, - args.pretrained_model, - main_program=pruned_program_iter, - predicate=if_exist) - - pruned_program = pruned_program_iter - pruned_val_program = pruned_val_program_iter - for i in range(args.num_epochs): - train(i, pruned_program) - test(i, pruned_val_program) - save_model(args, exe, pruned_program, pruned_val_program, i) - - -def main(): - args = parser.parse_args() - print_arguments(args) - compress(args) - - -if __name__ == '__main__': - main() diff --git a/demo/auto_prune/train_iterator.py b/demo/auto_prune/train_iterator.py deleted file mode 100644 index be80d892..00000000 --- a/demo/auto_prune/train_iterator.py +++ /dev/null @@ -1,252 +0,0 @@ -import os -import sys -import logging -import paddle -import argparse -import functools -import math -import time -import numpy as np -import paddle.fluid as fluid -from paddleslim.prune import AutoPruner -from paddleslim.common import get_logger -from paddleslim.analysis import flops -from paddleslim.prune import Pruner -sys.path.append(sys.path[0] + "/../") -import models -from utility import add_arguments, print_arguments - -_logger = get_logger(__name__, level=logging.INFO) - -parser = argparse.ArgumentParser(description=__doc__) -add_arg = functools.partial(add_arguments, argparser=parser) -# yapf: disable -add_arg('batch_size', int, 64 * 4, "Minibatch size.") -add_arg('use_gpu', bool, True, "Whether to use GPU or not.") -add_arg('model', str, "MobileNet", "The target model.") -add_arg('pretrained_model', str, "../pretrained_model/MobileNetV1_pretrained", "Whether to use pretrained model.") -add_arg('model_save_dir', str, "./", "checkpoint model.") -add_arg('lr', float, 0.1, "The learning rate used to fine-tune pruned model.") -add_arg('lr_strategy', str, "piecewise_decay", "The learning rate decay strategy.") -add_arg('l2_decay', float, 3e-5, "The l2_decay parameter.") -add_arg('momentum_rate', float, 0.9, "The value of momentum_rate.") -add_arg('num_epochs', int, 120, "The number of total epochs.") -add_arg('total_images', int, 1281167, "The number of total training images.") -parser.add_argument('--step_epochs', nargs='+', type=int, default=[30, 60, 90], help="piecewise decay step") -add_arg('config_file', str, None, "The config file for compression with yaml format.") -add_arg('data', str, "imagenet", "Which data to use. 'mnist' or 'imagenet'") -add_arg('log_period', int, 10, "Log period in batches.") -add_arg('test_period', int, 10, "Test period in epoches.") -# yapf: enable - -model_list = [m for m in dir(models) if "__" not in m] -ratiolist = [ - # [0.06, 0.0, 0.09, 0.03, 0.09, 0.02, 0.05, 0.03, 0.0, 0.07, 0.07, 0.05, 0.08], - # [0.08, 0.02, 0.03, 0.13, 0.1, 0.06, 0.03, 0.04, 0.14, 0.02, 0.03, 0.02, 0.01], -] - - -def piecewise_decay(args): - step = int(math.ceil(float(args.total_images) / args.batch_size)) - bd = [step * e for e in args.step_epochs] - lr = [args.lr * (0.1**i) for i in range(len(bd) + 1)] - learning_rate = fluid.layers.piecewise_decay(boundaries=bd, values=lr) - optimizer = fluid.optimizer.Momentum( - learning_rate=learning_rate, - momentum=args.momentum_rate, - regularization=fluid.regularizer.L2Decay(args.l2_decay)) - return optimizer - - -def cosine_decay(args): - step = int(math.ceil(float(args.total_images) / args.batch_size)) - learning_rate = fluid.layers.cosine_decay( - learning_rate=args.lr, step_each_epoch=step, epochs=args.num_epochs) - optimizer = fluid.optimizer.Momentum( - learning_rate=learning_rate, - momentum=args.momentum_rate, - regularization=fluid.regularizer.L2Decay(args.l2_decay)) - return optimizer - - -def create_optimizer(args): - if args.lr_strategy == "piecewise_decay": - return piecewise_decay(args) - elif args.lr_strategy == "cosine_decay": - return cosine_decay(args) - - -def compress(args): - - train_reader = None - test_reader = None - if args.data == "mnist": - import paddle.dataset.mnist as reader - train_reader = reader.train() - val_reader = reader.test() - class_dim = 10 - image_shape = "1,28,28" - elif args.data == "imagenet": - import imagenet_reader as reader - train_reader = reader.train() - val_reader = reader.val() - class_dim = 1000 - image_shape = "3,224,224" - else: - raise ValueError("{} is not supported.".format(args.data)) - - image_shape = [int(m) for m in image_shape.split(",")] - assert args.model in model_list, "{} is not in lists: {}".format(args.model, - model_list) - image = fluid.layers.data(name='image', shape=image_shape, dtype='float32') - label = fluid.layers.data(name='label', shape=[1], dtype='int64') - # model definition - model = models.__dict__[args.model]() - out = model.net(input=image, class_dim=class_dim) - cost = fluid.layers.cross_entropy(input=out, label=label) - avg_cost = fluid.layers.mean(x=cost) - acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1) - acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5) - val_program = fluid.default_main_program().clone(for_test=True) - opt = create_optimizer(args) - opt.minimize(avg_cost) - place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - - if args.pretrained_model: - - def if_exist(var): - return os.path.exists(os.path.join(args.pretrained_model, var.name)) - -# fluid.io.load_vars(exe, args.pretrained_model, predicate=if_exist) - - val_reader = paddle.fluid.io.batch(val_reader, batch_size=args.batch_size) - train_reader = paddle.fluid.io.batch( - train_reader, batch_size=args.batch_size, drop_last=True) - - train_feeder = feeder = fluid.DataFeeder([image, label], place) - val_feeder = feeder = fluid.DataFeeder( - [image, label], place, program=val_program) - - def test(epoch, program): - batch_id = 0 - acc_top1_ns = [] - acc_top5_ns = [] - for data in val_reader(): - start_time = time.time() - acc_top1_n, acc_top5_n = exe.run( - program, - feed=train_feeder.feed(data), - fetch_list=[acc_top1.name, acc_top5.name]) - end_time = time.time() - if batch_id % args.log_period == 0: - _logger.info( - "Eval epoch[{}] batch[{}] - acc_top1: {}; acc_top5: {}; time: {}". - format(epoch, batch_id, - np.mean(acc_top1_n), - np.mean(acc_top5_n), end_time - start_time)) - acc_top1_ns.append(np.mean(acc_top1_n)) - acc_top5_ns.append(np.mean(acc_top5_n)) - batch_id += 1 - - _logger.info("Final eval epoch[{}] - acc_top1: {}; acc_top5: {}".format( - epoch, - np.mean(np.array(acc_top1_ns)), np.mean(np.array(acc_top5_ns)))) - return np.mean(np.array(acc_top1_ns)) - - def train(epoch, program): - - build_strategy = fluid.BuildStrategy() - exec_strategy = fluid.ExecutionStrategy() - train_program = fluid.compiler.CompiledProgram( - program).with_data_parallel( - loss_name=avg_cost.name, - build_strategy=build_strategy, - exec_strategy=exec_strategy) - - batch_id = 0 - for data in train_reader(): - start_time = time.time() - loss_n, acc_top1_n, acc_top5_n = exe.run( - train_program, - feed=train_feeder.feed(data), - fetch_list=[avg_cost.name, acc_top1.name, acc_top5.name]) - end_time = time.time() - loss_n = np.mean(loss_n) - acc_top1_n = np.mean(acc_top1_n) - acc_top5_n = np.mean(acc_top5_n) - if batch_id % args.log_period == 0: - _logger.info( - "epoch[{}]-batch[{}] - loss: {}; acc_top1: {}; acc_top5: {}; time: {}". - format(epoch, batch_id, loss_n, acc_top1_n, acc_top5_n, - end_time - start_time)) - batch_id += 1 - - params = [] - for param in fluid.default_main_program().global_block().all_parameters(): - if "_sep_weights" in param.name: - params.append(param.name) - - pruned_program_iter = fluid.default_main_program() - pruned_val_program_iter = val_program - - for ratios in ratiolist: - pruner = Pruner() - pruned_val_program_iter = pruner.prune( - pruned_val_program_iter, - fluid.global_scope(), - params=params, - ratios=ratios, - place=place, - only_graph=True) - - pruned_program_iter = pruner.prune( - pruned_program_iter, - fluid.global_scope(), - params=params, - ratios=ratios, - place=place) - print("fops after pruning: {}".format(flops(pruned_program_iter))) - fluid.io.load_vars( - exe, - args.pretrained_model, - main_program=pruned_program_iter, - predicate=if_exist) - - pruner = AutoPruner( - pruned_val_program_iter, - fluid.global_scope(), - place, - params=params, - init_ratios=[0.1] * len(params), - pruned_flops=0.1, - pruned_latency=None, - server_addr=("", 0), - init_temperature=100, - reduce_rate=0.85, - max_try_times=300, - max_client_num=10, - search_steps=100, - max_ratios=0.2, - min_ratios=0., - is_server=True, - key="auto_pruner") - - while True: - pruned_program, pruned_val_program = pruner.prune( - pruned_program_iter, pruned_val_program_iter) - for i in range(0): - train(i, pruned_program) - score = test(0, pruned_val_program) - pruner.reward(score) - - -def main(): - args = parser.parse_args() - print_arguments(args) - compress(args) - - -if __name__ == '__main__': - main() diff --git a/demo/prune/eval.py b/demo/prune/eval.py index a93a056a..6cbc2540 100644 --- a/demo/prune/eval.py +++ b/demo/prune/eval.py @@ -7,7 +7,6 @@ import functools import math import time import numpy as np -import paddle.fluid as fluid from paddleslim.prune import load_model from paddleslim.common import get_logger from paddleslim.analysis import flops @@ -35,9 +34,7 @@ def eval(args): train_reader = None test_reader = None if args.data == "mnist": - import paddle.dataset.mnist as reader - train_reader = reader.train() - val_reader = reader.test() + val_reader = paddle.dataset.mnist.test() class_dim = 10 image_shape = "1,28,28" elif args.data == "imagenet": @@ -51,34 +48,36 @@ def eval(args): image_shape = [int(m) for m in image_shape.split(",")] assert args.model in model_list, "{} is not in lists: {}".format(args.model, model_list) - image = fluid.layers.data(name='image', shape=image_shape, dtype='float32') - label = fluid.layers.data(name='label', shape=[1], dtype='int64') + image = paddle.static.data( + name='image', shape=[None] + image_shape, dtype='float32') + label = paddle.static.data(name='label', shape=[None, 1], dtype='int64') # model definition model = models.__dict__[args.model]() out = model.net(input=image, class_dim=class_dim) - acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1) - acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5) - val_program = fluid.default_main_program().clone(for_test=True) - place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) + acc_top1 = paddle.metric.accuracy(input=out, label=label, k=1) + acc_top5 = paddle.metric.accuracy(input=out, label=label, k=5) + val_program = paddle.static.default_main_program().clone(for_test=True) + place = paddle.CUDAPlace(0) if args.use_gpu else paddle.CPUPlace() + exe = paddle.static.Executor(place) + exe.run(paddle.static.default_startup_program()) - val_reader = paddle.fluid.io.batch(val_reader, batch_size=args.batch_size) + val_reader = paddle.batch(val_reader, batch_size=args.batch_size) - val_feeder = feeder = fluid.DataFeeder( - [image, label], place, program=val_program) + valid_loader = paddle.io.DataLoader.from_generator( + feed_list=[image, label], + capacity=64, + use_double_buffer=True, + iterable=True) + valid_loader.set_sample_list_generator(val_reader, place) load_model(exe, val_program, args.model_path) - batch_id = 0 acc_top1_ns = [] acc_top5_ns = [] - for data in val_reader(): + for batch_id, data in enumerate(valid_loader): start_time = time.time() acc_top1_n, acc_top5_n = exe.run( - val_program, - feed=val_feeder.feed(data), - fetch_list=[acc_top1.name, acc_top5.name]) + val_program, feed=data, fetch_list=[acc_top1.name, acc_top5.name]) end_time = time.time() if batch_id % args.log_period == 0: _logger.info( @@ -88,7 +87,6 @@ def eval(args): np.mean(acc_top5_n), end_time - start_time)) acc_top1_ns.append(np.mean(acc_top1_n)) acc_top5_ns.append(np.mean(acc_top5_n)) - batch_id += 1 _logger.info("Final eval - acc_top1: {}; acc_top5: {}".format( np.mean(np.array(acc_top1_ns)), np.mean(np.array(acc_top5_ns)))) diff --git a/demo/prune/train.py b/demo/prune/train.py index 197e2998..9748ca4e 100644 --- a/demo/prune/train.py +++ b/demo/prune/train.py @@ -7,7 +7,6 @@ import functools import math import time import numpy as np -import paddle.fluid as fluid sys.path[0] = os.path.join(os.path.dirname("__file__"), os.path.pardir) from paddleslim.prune import Pruner, save_model from paddleslim.common import get_logger @@ -69,23 +68,23 @@ def piecewise_decay(args): step = int(math.ceil(float(args.total_images) / args.batch_size)) bd = [step * e for e in args.step_epochs] lr = [args.lr * (0.1**i) for i in range(len(bd) + 1)] - learning_rate = fluid.layers.piecewise_decay(boundaries=bd, values=lr) + learning_rate = paddle.optimizer.lr.PiecewiseDecay(boundaries=bd, values=lr) - optimizer = fluid.optimizer.Momentum( + optimizer = paddle.optimizer.Momentum( learning_rate=learning_rate, momentum=args.momentum_rate, - regularization=fluid.regularizer.L2Decay(args.l2_decay)) + weight_decay=paddle.regularizer.L2Decay(args.l2_decay)) return optimizer def cosine_decay(args): step = int(math.ceil(float(args.total_images) / args.batch_size)) - learning_rate = fluid.layers.cosine_decay( + learning_rate = paddle.optimizer.lr.cosine_decay( learning_rate=args.lr, step_each_epoch=step, epochs=args.num_epochs) - optimizer = fluid.optimizer.Momentum( + optimizer = paddle.optimizer.Momentum( learning_rate=learning_rate, momentum=args.momentum_rate, - regularization=fluid.regularizer.L2Decay(args.l2_decay)) + weight_decay=paddle.regularizer.L2Decay(args.l2_decay)) return optimizer @@ -100,9 +99,8 @@ def compress(args): train_reader = None test_reader = None if args.data == "mnist": - import paddle.dataset.mnist as reader - train_reader = reader.train() - val_reader = reader.test() + train_reader = paddle.dataset.mnist.train() + val_reader = paddle.dataset.mnist.test() class_dim = 10 image_shape = "1,28,28" elif args.data == "imagenet": @@ -116,21 +114,24 @@ def compress(args): image_shape = [int(m) for m in image_shape.split(",")] assert args.model in model_list, "{} is not in lists: {}".format(args.model, model_list) - image = fluid.layers.data(name='image', shape=image_shape, dtype='float32') - label = fluid.layers.data(name='label', shape=[1], dtype='int64') + image = paddle.static.data( + name='image', shape=[None] + image_shape, dtype='float32') + label = paddle.static.data(name='label', shape=[None, 1], dtype='int64') # model definition model = models.__dict__[args.model]() out = model.net(input=image, class_dim=class_dim) - cost = fluid.layers.cross_entropy(input=out, label=label) - avg_cost = fluid.layers.mean(x=cost) - acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1) - acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5) - val_program = fluid.default_main_program().clone(for_test=True) + cost = paddle.nn.functional.loss.cross_entropy(input=out, label=label) + avg_cost = paddle.mean(x=cost) + acc_top1 = paddle.metric.accuracy(input=out, label=label, k=1) + acc_top5 = paddle.metric.accuracy(input=out, label=label, k=5) + val_program = paddle.static.default_main_program().clone(for_test=True) opt = create_optimizer(args) opt.minimize(avg_cost) - place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) + places = paddle.static.cuda_places( + ) if args.use_gpu else paddle.static.cpu_places() + place = places[0] + exe = paddle.static.Executor(place) + exe.run(paddle.static.default_startup_program()) if args.pretrained_model: @@ -139,26 +140,33 @@ def compress(args): _logger.info("Load pretrained model from {}".format( args.pretrained_model)) - fluid.io.load_vars(exe, args.pretrained_model, predicate=if_exist) + paddle.fluid.io.load_vars( + exe, args.pretrained_model, predicate=if_exist) - val_reader = paddle.fluid.io.batch(val_reader, batch_size=args.batch_size) - train_reader = paddle.fluid.io.batch( + val_reader = paddle.batch(val_reader, batch_size=args.batch_size) + train_reader = paddle.batch( train_reader, batch_size=args.batch_size, drop_last=True) - train_feeder = feeder = fluid.DataFeeder([image, label], place) - val_feeder = feeder = fluid.DataFeeder( - [image, label], place, program=val_program) + train_loader = paddle.io.DataLoader.from_generator( + feed_list=[image, label], + capacity=64, + use_double_buffer=True, + iterable=True) + valid_loader = paddle.io.DataLoader.from_generator( + feed_list=[image, label], + capacity=64, + use_double_buffer=True, + iterable=True) + train_loader.set_sample_list_generator(train_reader, places) + valid_loader.set_sample_list_generator(val_reader, place) def test(epoch, program): - batch_id = 0 acc_top1_ns = [] acc_top5_ns = [] - for data in val_reader(): + for batch_id, data in enumerate(valid_loader): start_time = time.time() acc_top1_n, acc_top5_n = exe.run( - program, - feed=train_feeder.feed(data), - fetch_list=[acc_top1.name, acc_top5.name]) + program, feed=data, fetch_list=[acc_top1.name, acc_top5.name]) end_time = time.time() if batch_id % args.log_period == 0: _logger.info( @@ -168,7 +176,6 @@ def compress(args): np.mean(acc_top5_n), end_time - start_time)) acc_top1_ns.append(np.mean(acc_top1_n)) acc_top5_ns.append(np.mean(acc_top5_n)) - batch_id += 1 _logger.info("Final eval epoch[{}] - acc_top1: {}; acc_top5: {}".format( epoch, @@ -176,20 +183,19 @@ def compress(args): def train(epoch, program): - build_strategy = fluid.BuildStrategy() - exec_strategy = fluid.ExecutionStrategy() - train_program = fluid.compiler.CompiledProgram( + build_strategy = paddle.static.BuildStrategy() + exec_strategy = paddle.static.ExecutionStrategy() + train_program = paddle.static.CompiledProgram( program).with_data_parallel( loss_name=avg_cost.name, build_strategy=build_strategy, exec_strategy=exec_strategy) - batch_id = 0 - for data in train_reader(): + for batch_id, data in enumerate(train_loader): start_time = time.time() loss_n, acc_top1_n, acc_top5_n = exe.run( train_program, - feed=train_feeder.feed(data), + feed=data, fetch_list=[avg_cost.name, acc_top1.name, acc_top5.name]) end_time = time.time() loss_n = np.mean(loss_n) @@ -203,22 +209,21 @@ def compress(args): batch_id += 1 test(0, val_program) - - params = get_pruned_params(args, fluid.default_main_program()) + params = get_pruned_params(args, paddle.static.default_main_program()) _logger.info("FLOPs before pruning: {}".format( - flops(fluid.default_main_program()))) + flops(paddle.static.default_main_program()))) pruner = Pruner(args.criterion) pruned_val_program, _, _ = pruner.prune( val_program, - fluid.global_scope(), + paddle.static.global_scope(), params=params, ratios=[args.pruned_ratio] * len(params), place=place, only_graph=True) pruned_program, _, _ = pruner.prune( - fluid.default_main_program(), - fluid.global_scope(), + paddle.static.default_main_program(), + paddle.static.global_scope(), params=params, ratios=[args.pruned_ratio] * len(params), place=place) @@ -232,8 +237,8 @@ def compress(args): if args.save_inference: infer_model_path = os.path.join(args.model_path, "infer_models", str(i)) - fluid.io.save_inference_model(infer_model_path, ["image"], [out], - exe, pruned_val_program) + paddle.fluid.io.save_inference_model(infer_model_path, ["image"], + [out], exe, pruned_val_program) _logger.info("Saved inference model into [{}]".format( infer_model_path)) diff --git a/demo/sensitive/train.py b/demo/sensitive/train.py index 693d6bf4..78c4c1e5 100644 --- a/demo/sensitive/train.py +++ b/demo/sensitive/train.py @@ -7,7 +7,6 @@ import functools import math import time import numpy as np -import paddle.fluid as fluid from paddleslim.prune import merge_sensitive, get_ratios_by_loss from paddleslim.prune import sensitivity from paddleslim.common import get_logger @@ -46,43 +45,47 @@ def compress(args): else: raise ValueError("{} is not supported.".format(args.data)) image_shape = [int(m) for m in image_shape.split(",")] - assert args.model in model_list, "{} is not in lists: {}".format( - args.model, model_list) - image = fluid.layers.data(name='image', shape=image_shape, dtype='float32') - label = fluid.layers.data(name='label', shape=[1], dtype='int64') + assert args.model in model_list, "{} is not in lists: {}".format(args.model, + model_list) + image = paddle.static.data( + name='image', shape=[None] + image_shape, dtype='float32') + label = paddle.static.data(name='label', shape=[None, 1], dtype='int64') # model definition model = models.__dict__[args.model]() out = model.net(input=image, class_dim=class_dim) - acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1) - acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5) - val_program = fluid.default_main_program().clone(for_test=True) - place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) + acc_top1 = paddle.metric.accuracy(input=out, label=label, k=1) + acc_top5 = paddle.metric.accuracy(input=out, label=label, k=5) + val_program = paddle.static.default_main_program().clone(for_test=True) + places = paddle.static.cuda_places( + ) if args.use_gpu else paddle.static.cpu_places() + place = places[0] + exe = paddle.static.Executor(place) + exe.run(paddle.static.default_startup_program()) if args.pretrained_model: def if_exist(var): - return os.path.exists( - os.path.join(args.pretrained_model, var.name)) + return os.path.exists(os.path.join(args.pretrained_model, var.name)) - fluid.io.load_vars(exe, args.pretrained_model, predicate=if_exist) + paddle.fluid.io.load_vars( + exe, args.pretrained_model, predicate=if_exist) - val_reader = paddle.fluid.io.batch(val_reader, batch_size=args.batch_size) + val_reader = paddle.batch(val_reader, batch_size=args.batch_size) + valid_loader = paddle.io.DataLoader.from_generator( + feed_list=[image, label], + capacity=64, + use_double_buffer=True, + iterable=True) - val_feeder = feeder = fluid.DataFeeder( - [image, label], place, program=val_program) + valid_loader.set_sample_list_generator(val_reader, place) def test(program): - batch_id = 0 acc_top1_ns = [] acc_top5_ns = [] - for data in val_reader(): + for batch_id, data in enumerate(valid_loader): start_time = time.time() acc_top1_n, acc_top5_n = exe.run( - program, - feed=val_feeder.feed(data), - fetch_list=[acc_top1.name, acc_top5.name]) + program, feed=data, fetch_list=[acc_top1.name, acc_top5.name]) end_time = time.time() if batch_id % args.log_period == 0: _logger.info( @@ -99,8 +102,10 @@ def compress(args): return np.mean(np.array(acc_top1_ns)) params = [] - for param in fluid.default_main_program().global_block().all_parameters(): - if "_sep_weights" in param.name: + for param in paddle.static.default_main_program().global_block( + ).all_parameters(): + if "weights" in param.name: + print(param.name) params.append(param.name) sensitivity( @@ -119,15 +124,15 @@ def compress(args): sensitivities_file="sensitivities_1.data", pruned_ratios=[0.5, 0.6, 0.7]) - sens = merge_sensitive( - ["./sensitivities_0.data", "./sensitivities_1.data"]) + sens = merge_sensitive(["./sensitivities_0.data", "./sensitivities_1.data"]) ratios = get_ratios_by_loss(sens, 0.01) - print ratios + print(sens) def main(): + paddle.enable_static() args = parser.parse_args() print_arguments(args) compress(args) diff --git a/demo/sensitive_prune/greedy_prune.py b/demo/sensitive_prune/greedy_prune.py deleted file mode 100644 index f59b7eaa..00000000 --- a/demo/sensitive_prune/greedy_prune.py +++ /dev/null @@ -1,231 +0,0 @@ -import os -import sys -import logging -import paddle -import argparse -import functools -import math -import time -import numpy as np -import paddle.fluid as fluid -from paddleslim.prune import SensitivePruner -from paddleslim.common import get_logger -from paddleslim.analysis import flops -sys.path.append(sys.path[0] + "/../") -import models -from utility import add_arguments, print_arguments - -_logger = get_logger(__name__, level=logging.INFO) - -parser = argparse.ArgumentParser(description=__doc__) -add_arg = functools.partial(add_arguments, argparser=parser) -# yapf: disable -add_arg('batch_size', int, 64 * 4, "Minibatch size.") -add_arg('use_gpu', bool, True, "Whether to use GPU or not.") -add_arg('model', str, "MobileNet", "The target model.") -add_arg('pretrained_model', str, "../pretrained_model/MobileNetV1_pretained", "Whether to use pretrained model.") -add_arg('lr', float, 0.1, "The learning rate used to fine-tune pruned model.") -add_arg('lr_strategy', str, "piecewise_decay", "The learning rate decay strategy.") -add_arg('l2_decay', float, 3e-5, "The l2_decay parameter.") -add_arg('momentum_rate', float, 0.9, "The value of momentum_rate.") -add_arg('num_epochs', int, 120, "The number of total epochs.") -add_arg('total_images', int, 1281167, "The number of total training images.") -parser.add_argument('--step_epochs', nargs='+', type=int, default=[30, 60, 90], help="piecewise decay step") -add_arg('config_file', str, None, "The config file for compression with yaml format.") -add_arg('data', str, "mnist", "Which data to use. 'mnist' or 'imagenet'") -add_arg('log_period', int, 10, "Log period in batches.") -add_arg('test_period', int, 10, "Test period in epoches.") -add_arg('checkpoints', str, "./checkpoints", "Checkpoints path.") -add_arg('prune_steps', int, 1000, "prune steps.") -add_arg('retrain_epoch', int, 5, "Retrain epoch.") -# yapf: enable - -model_list = [m for m in dir(models) if "__" not in m] - - -def piecewise_decay(args): - step = int(math.ceil(float(args.total_images) / args.batch_size)) - bd = [step * e for e in args.step_epochs] - lr = [args.lr * (0.1**i) for i in range(len(bd) + 1)] - learning_rate = fluid.layers.piecewise_decay(boundaries=bd, values=lr) - optimizer = fluid.optimizer.Momentum( - learning_rate=learning_rate, - momentum=args.momentum_rate, - regularization=fluid.regularizer.L2Decay(args.l2_decay)) - return optimizer - - -def cosine_decay(args): - step = int(math.ceil(float(args.total_images) / args.batch_size)) - learning_rate = fluid.layers.cosine_decay( - learning_rate=args.lr, step_each_epoch=step, epochs=args.num_epochs) - optimizer = fluid.optimizer.Momentum( - learning_rate=learning_rate, - momentum=args.momentum_rate, - regularization=fluid.regularizer.L2Decay(args.l2_decay)) - return optimizer - - -def create_optimizer(args): - if args.lr_strategy == "piecewise_decay": - return piecewise_decay(args) - elif args.lr_strategy == "cosine_decay": - return cosine_decay(args) - - -def compress(args): - - train_reader = None - test_reader = None - if args.data == "mnist": - import paddle.dataset.mnist as reader - train_reader = reader.train() - val_reader = reader.test() - class_dim = 10 - image_shape = "1,28,28" - elif args.data == "imagenet": - import imagenet_reader as reader - train_reader = reader.train() - val_reader = reader.val() - class_dim = 1000 - image_shape = "3,224,224" - else: - raise ValueError("{} is not supported.".format(args.data)) - - image_shape = [int(m) for m in image_shape.split(",")] - assert args.model in model_list, "{} is not in lists: {}".format( - args.model, model_list) - image = fluid.layers.data(name='image', shape=image_shape, dtype='float32') - label = fluid.layers.data(name='label', shape=[1], dtype='int64') - # model definition - model = models.__dict__[args.model]() - out = model.net(input=image, class_dim=class_dim) - cost = fluid.layers.cross_entropy(input=out, label=label) - avg_cost = fluid.layers.mean(x=cost) - acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1) - acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5) - val_program = fluid.default_main_program().clone(for_test=True) - opt = create_optimizer(args) - opt.minimize(avg_cost) - place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - - if args.pretrained_model: - - def if_exist(var): - return os.path.exists( - os.path.join(args.pretrained_model, var.name)) - - fluid.io.load_vars(exe, args.pretrained_model, predicate=if_exist) - - val_reader = paddle.fluid.io.batch(val_reader, batch_size=args.batch_size) - train_reader = paddle.fluid.io.batch( - train_reader, batch_size=args.batch_size, drop_last=True) - - train_feeder = feeder = fluid.DataFeeder([image, label], place) - val_feeder = feeder = fluid.DataFeeder( - [image, label], place, program=val_program) - - def test(epoch, program): - batch_id = 0 - acc_top1_ns = [] - acc_top5_ns = [] - for data in val_reader(): - start_time = time.time() - acc_top1_n, acc_top5_n = exe.run( - program, - feed=train_feeder.feed(data), - fetch_list=[acc_top1.name, acc_top5.name]) - end_time = time.time() - if batch_id % args.log_period == 0: - _logger.info( - "Eval epoch[{}] batch[{}] - acc_top1: {:.3f}; acc_top5: {:.3f}; time: {:.3f}". - format(epoch, batch_id, - np.mean(acc_top1_n), - np.mean(acc_top5_n), end_time - start_time)) - acc_top1_ns.append(np.mean(acc_top1_n)) - acc_top5_ns.append(np.mean(acc_top5_n)) - batch_id += 1 - - _logger.info( - "Final eval epoch[{}] - acc_top1: {:.3f}; acc_top5: {:.3f}".format( - epoch, - np.mean(np.array(acc_top1_ns)), np.mean( - np.array(acc_top5_ns)))) - return np.mean(np.array(acc_top1_ns)) - - def train(epoch, program): - - build_strategy = fluid.BuildStrategy() - exec_strategy = fluid.ExecutionStrategy() - train_program = fluid.compiler.CompiledProgram( - program).with_data_parallel( - loss_name=avg_cost.name, - build_strategy=build_strategy, - exec_strategy=exec_strategy) - - batch_id = 0 - for data in train_reader(): - start_time = time.time() - loss_n, acc_top1_n, acc_top5_n = exe.run( - train_program, - feed=train_feeder.feed(data), - fetch_list=[avg_cost.name, acc_top1.name, acc_top5.name]) - end_time = time.time() - loss_n = np.mean(loss_n) - acc_top1_n = np.mean(acc_top1_n) - acc_top5_n = np.mean(acc_top5_n) - if batch_id % args.log_period == 0: - _logger.info( - "epoch[{}]-batch[{}] - loss: {:.3f}; acc_top1: {:.3f}; acc_top5: {:.3f}; time: {:.3f}". - format(epoch, batch_id, loss_n, acc_top1_n, acc_top5_n, - end_time - start_time)) - batch_id += 1 - - params = [] - for param in fluid.default_main_program().global_block().all_parameters(): - if "_sep_weights" in param.name: - params.append(param.name) - - def eval_func(program): - return test(0, program) - - if args.data == "mnist": - train(0, fluid.default_main_program()) - - pruner = SensitivePruner(place, eval_func, checkpoints=args.checkpoints) - pruned_program, pruned_val_program, iter = pruner.restore() - - if pruned_program is None: - pruned_program = fluid.default_main_program() - if pruned_val_program is None: - pruned_val_program = val_program - - base_flops = flops(val_program) - - start = iter - end = args.prune_steps - for iter in range(start, end): - pruned_program, pruned_val_program = pruner.greedy_prune( - pruned_program, pruned_val_program, params, 0.03, topk=1) - current_flops = flops(pruned_val_program) - print("iter:{}; pruned FLOPS: {}".format( - iter, float(base_flops - current_flops) / base_flops)) - acc = None - for i in range(args.retrain_epoch): - train(i, pruned_program) - acc = test(i, pruned_val_program) - print("iter:{}; pruned FLOPS: {}; acc: {}".format( - iter, float(base_flops - current_flops) / base_flops, acc)) - pruner.save_checkpoint(pruned_program, pruned_val_program) - - -def main(): - args = parser.parse_args() - print_arguments(args) - compress(args) - - -if __name__ == '__main__': - main() diff --git a/demo/sensitive_prune/prune.py b/demo/sensitive_prune/prune.py deleted file mode 100644 index a4cb8ac2..00000000 --- a/demo/sensitive_prune/prune.py +++ /dev/null @@ -1,223 +0,0 @@ -import os -import sys -import logging -import paddle -import argparse -import functools -import math -import time -import numpy as np -import paddle.fluid as fluid -from paddleslim.prune import SensitivePruner -from paddleslim.common import get_logger -from paddleslim.analysis import flops -sys.path.append(sys.path[0] + "/../") -import models -from utility import add_arguments, print_arguments - -_logger = get_logger(__name__, level=logging.INFO) - -parser = argparse.ArgumentParser(description=__doc__) -add_arg = functools.partial(add_arguments, argparser=parser) -# yapf: disable -add_arg('batch_size', int, 64 * 4, "Minibatch size.") -add_arg('use_gpu', bool, True, "Whether to use GPU or not.") -add_arg('model', str, "MobileNet", "The target model.") -add_arg('pretrained_model', str, "../pretrained_model/MobileNetV1_pretained", "Whether to use pretrained model.") -add_arg('lr', float, 0.1, "The learning rate used to fine-tune pruned model.") -add_arg('lr_strategy', str, "piecewise_decay", "The learning rate decay strategy.") -add_arg('l2_decay', float, 3e-5, "The l2_decay parameter.") -add_arg('momentum_rate', float, 0.9, "The value of momentum_rate.") -add_arg('num_epochs', int, 120, "The number of total epochs.") -add_arg('total_images', int, 1281167, "The number of total training images.") -parser.add_argument('--step_epochs', nargs='+', type=int, default=[30, 60, 90], help="piecewise decay step") -add_arg('config_file', str, None, "The config file for compression with yaml format.") -add_arg('data', str, "mnist", "Which data to use. 'mnist' or 'imagenet'") -add_arg('log_period', int, 10, "Log period in batches.") -add_arg('test_period', int, 10, "Test period in epoches.") -add_arg('checkpoints', str, "./checkpoints", "Checkpoints path.") -# yapf: enable - -model_list = [m for m in dir(models) if "__" not in m] - - -def piecewise_decay(args): - step = int(math.ceil(float(args.total_images) / args.batch_size)) - bd = [step * e for e in args.step_epochs] - lr = [args.lr * (0.1**i) for i in range(len(bd) + 1)] - learning_rate = fluid.layers.piecewise_decay(boundaries=bd, values=lr) - optimizer = fluid.optimizer.Momentum( - learning_rate=learning_rate, - momentum=args.momentum_rate, - regularization=fluid.regularizer.L2Decay(args.l2_decay)) - return optimizer - - -def cosine_decay(args): - step = int(math.ceil(float(args.total_images) / args.batch_size)) - learning_rate = fluid.layers.cosine_decay( - learning_rate=args.lr, step_each_epoch=step, epochs=args.num_epochs) - optimizer = fluid.optimizer.Momentum( - learning_rate=learning_rate, - momentum=args.momentum_rate, - regularization=fluid.regularizer.L2Decay(args.l2_decay)) - return optimizer - - -def create_optimizer(args): - if args.lr_strategy == "piecewise_decay": - return piecewise_decay(args) - elif args.lr_strategy == "cosine_decay": - return cosine_decay(args) - - -def compress(args): - - train_reader = None - test_reader = None - if args.data == "mnist": - import paddle.dataset.mnist as reader - train_reader = reader.train() - val_reader = reader.test() - class_dim = 10 - image_shape = "1,28,28" - elif args.data == "imagenet": - import imagenet_reader as reader - train_reader = reader.train() - val_reader = reader.val() - class_dim = 1000 - image_shape = "3,224,224" - else: - raise ValueError("{} is not supported.".format(args.data)) - - image_shape = [int(m) for m in image_shape.split(",")] - assert args.model in model_list, "{} is not in lists: {}".format( - args.model, model_list) - image = fluid.layers.data(name='image', shape=image_shape, dtype='float32') - label = fluid.layers.data(name='label', shape=[1], dtype='int64') - # model definition - model = models.__dict__[args.model]() - out = model.net(input=image, class_dim=class_dim) - cost = fluid.layers.cross_entropy(input=out, label=label) - avg_cost = fluid.layers.mean(x=cost) - acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1) - acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5) - val_program = fluid.default_main_program().clone(for_test=True) - opt = create_optimizer(args) - opt.minimize(avg_cost) - place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - - if args.pretrained_model: - - def if_exist(var): - return os.path.exists( - os.path.join(args.pretrained_model, var.name)) - - fluid.io.load_vars(exe, args.pretrained_model, predicate=if_exist) - - val_reader = paddle.fluid.io.batch(val_reader, batch_size=args.batch_size) - train_reader = paddle.fluid.io.batch( - train_reader, batch_size=args.batch_size, drop_last=True) - - train_feeder = feeder = fluid.DataFeeder([image, label], place) - val_feeder = feeder = fluid.DataFeeder( - [image, label], place, program=val_program) - - def test(epoch, program): - batch_id = 0 - acc_top1_ns = [] - acc_top5_ns = [] - for data in val_reader(): - start_time = time.time() - acc_top1_n, acc_top5_n = exe.run( - program, - feed=train_feeder.feed(data), - fetch_list=[acc_top1.name, acc_top5.name]) - end_time = time.time() - if batch_id % args.log_period == 0: - _logger.info( - "Eval epoch[{}] batch[{}] - acc_top1: {:.3f}; acc_top5: {:.3f}; time: {:.3f}". - format(epoch, batch_id, - np.mean(acc_top1_n), - np.mean(acc_top5_n), end_time - start_time)) - acc_top1_ns.append(np.mean(acc_top1_n)) - acc_top5_ns.append(np.mean(acc_top5_n)) - batch_id += 1 - - _logger.info( - "Final eval epoch[{}] - acc_top1: {:.3f}; acc_top5: {:.3f}".format( - epoch, - np.mean(np.array(acc_top1_ns)), np.mean( - np.array(acc_top5_ns)))) - return np.mean(np.array(acc_top1_ns)) - - def train(epoch, program): - - build_strategy = fluid.BuildStrategy() - exec_strategy = fluid.ExecutionStrategy() - train_program = fluid.compiler.CompiledProgram( - program).with_data_parallel( - loss_name=avg_cost.name, - build_strategy=build_strategy, - exec_strategy=exec_strategy) - - batch_id = 0 - for data in train_reader(): - start_time = time.time() - loss_n, acc_top1_n, acc_top5_n = exe.run( - train_program, - feed=train_feeder.feed(data), - fetch_list=[avg_cost.name, acc_top1.name, acc_top5.name]) - end_time = time.time() - loss_n = np.mean(loss_n) - acc_top1_n = np.mean(acc_top1_n) - acc_top5_n = np.mean(acc_top5_n) - if batch_id % args.log_period == 0: - _logger.info( - "epoch[{}]-batch[{}] - loss: {:.3f}; acc_top1: {:.3f}; acc_top5: {:.3f}; time: {:.3f}". - format(epoch, batch_id, loss_n, acc_top1_n, acc_top5_n, - end_time - start_time)) - batch_id += 1 - - params = [] - for param in fluid.default_main_program().global_block().all_parameters(): - if "_sep_weights" in param.name: - params.append(param.name) - - def eval_func(program): - return test(0, program) - - if args.data == "mnist": - train(0, fluid.default_main_program()) - - pruner = SensitivePruner(place, eval_func, checkpoints=args.checkpoints) - pruned_program, pruned_val_program, iter = pruner.restore() - - if pruned_program is None: - pruned_program = fluid.default_main_program() - if pruned_val_program is None: - pruned_val_program = val_program - - start = iter - end = 6 - for iter in range(start, end): - pruned_program, pruned_val_program = pruner.prune( - pruned_program, pruned_val_program, params, 0.1) - train(iter, pruned_program) - test(iter, pruned_val_program) - pruner.save_checkpoint(pruned_program, pruned_val_program) - - print("before flops: {}".format(flops(fluid.default_main_program()))) - print("after flops: {}".format(flops(pruned_val_program))) - - -def main(): - args = parser.parse_args() - print_arguments(args) - compress(args) - - -if __name__ == '__main__': - main() diff --git a/paddleslim/prune/__init__.py b/paddleslim/prune/__init__.py index 361a3af1..01062078 100644 --- a/paddleslim/prune/__init__.py +++ b/paddleslim/prune/__init__.py @@ -17,8 +17,6 @@ from .pruner import * from ..prune import pruner from .auto_pruner import * from ..prune import auto_pruner -from .sensitive_pruner import * -from ..prune import sensitive_pruner from .sensitive import * from ..prune import sensitive from .prune_walker import * @@ -36,7 +34,6 @@ __all__ = [] __all__ += pruner.__all__ __all__ += auto_pruner.__all__ -__all__ += sensitive_pruner.__all__ __all__ += sensitive.__all__ __all__ += prune_walker.__all__ __all__ += prune_io.__all__ diff --git a/paddleslim/prune/auto_pruner.py b/paddleslim/prune/auto_pruner.py index 6ace8546..4e4e56c6 100644 --- a/paddleslim/prune/auto_pruner.py +++ b/paddleslim/prune/auto_pruner.py @@ -15,7 +15,6 @@ import socket import logging import numpy as np -import paddle.fluid as fluid from .pruner import Pruner from ..core import VarWrapper, OpWrapper, GraphWrapper from ..common import SAController diff --git a/paddleslim/prune/prune_io.py b/paddleslim/prune/prune_io.py index c0e2c975..901b22a1 100644 --- a/paddleslim/prune/prune_io.py +++ b/paddleslim/prune/prune_io.py @@ -1,6 +1,5 @@ import os -import paddle.fluid as fluid -from paddle.fluid import Program +import paddle from ..core import GraphWrapper from ..common import get_logger import json @@ -23,9 +22,10 @@ def save_model(exe, graph, dirname): dirname(str): The directory that the model saved into. """ assert graph is not None and dirname is not None - graph = GraphWrapper(graph) if isinstance(graph, Program) else graph + graph = GraphWrapper(graph) if isinstance(graph, + paddle.static.Program) else graph - fluid.io.save_persistables( + paddle.fluid.io.save_persistables( executor=exe, dirname=dirname, main_program=graph.program, @@ -33,7 +33,7 @@ def save_model(exe, graph, dirname): weights_file = dirname _logger.info("Save model weights into {}".format(weights_file)) shapes = {} - for var in fluid.io.get_program_persistable_vars(graph.program): + for var in paddle.fluid.io.get_program_persistable_vars(graph.program): shapes[var.name] = var.shape SHAPES_FILE = os.path.join(dirname, _SHAPES_FILE) with open(SHAPES_FILE, "w") as f: @@ -50,7 +50,8 @@ def load_model(exe, graph, dirname): dirname(str): The directory that the model will be loaded. """ assert graph is not None and dirname is not None - graph = GraphWrapper(graph) if isinstance(graph, Program) else graph + graph = GraphWrapper(graph) if isinstance(graph, + paddle.static.Program) else graph SHAPES_FILE = os.path.join(dirname, _SHAPES_FILE) _logger.info("Load shapes of weights from {}".format(SHAPES_FILE)) @@ -64,7 +65,7 @@ def load_model(exe, graph, dirname): _logger.info('{} is not loaded'.format(param_name)) _logger.info("Load shapes of weights from {}".format(SHAPES_FILE)) - fluid.io.load_persistables( + paddle.fluid.io.load_persistables( executor=exe, dirname=dirname, main_program=graph.program, diff --git a/paddleslim/prune/pruner.py b/paddleslim/prune/pruner.py index 7fbac719..dc8c83e2 100644 --- a/paddleslim/prune/pruner.py +++ b/paddleslim/prune/pruner.py @@ -14,10 +14,9 @@ import logging import sys +import copy import numpy as np from functools import reduce -import paddle.fluid as fluid -import copy from ..core import VarWrapper, OpWrapper, GraphWrapper from .group_param import collect_convs from .criterion import CRITERION @@ -38,8 +37,7 @@ class Pruner(): """ - def __init__(self, - criterion="l1_norm", + def __init__(self, criterion="l1_norm", idx_selector="default_idx_selector"): if isinstance(criterion, str): self.criterion = CRITERION.get(criterion) @@ -93,8 +91,8 @@ class Pruner(): _logger.info("pruning: {}".format(param)) if graph.var(param) is None: _logger.warn( - "Variable[{}] to be pruned is not in current graph.". - format(param)) + "Variable[{}] to be pruned is not in current graph.".format( + param)) continue group = collect_convs([param], graph, visited)[0] # [(name, axis, pruned_idx)] diff --git a/paddleslim/prune/sensitive.py b/paddleslim/prune/sensitive.py index a5a6e360..20a19be2 100644 --- a/paddleslim/prune/sensitive.py +++ b/paddleslim/prune/sensitive.py @@ -17,7 +17,7 @@ import os import logging import pickle import numpy as np -import paddle.fluid as fluid +import paddle from ..core import GraphWrapper from ..common import get_logger from ..analysis import flops @@ -26,8 +26,7 @@ from ..prune import Pruner _logger = get_logger(__name__, level=logging.INFO) __all__ = [ - "sensitivity", "flops_sensitivity", "load_sensitivities", "merge_sensitive", - "get_ratios_by_loss" + "sensitivity", "load_sensitivities", "merge_sensitive", "get_ratios_by_loss" ] @@ -68,7 +67,7 @@ def sensitivity(program, Returns: dict: A dict storing sensitivities. """ - scope = fluid.global_scope() + scope = paddle.static.global_scope() graph = GraphWrapper(program) sensitivities = load_sensitivities(sensitivities_file) @@ -121,80 +120,6 @@ def sensitivity(program, return sensitivities -def flops_sensitivity(program, - place, - param_names, - eval_func, - sensitivities_file=None, - pruned_flops_rate=0.1): - - assert (1.0 / len(param_names) > pruned_flops_rate) - - scope = fluid.global_scope() - graph = GraphWrapper(program) - sensitivities = load_sensitivities(sensitivities_file) - - for name in param_names: - if name not in sensitivities: - sensitivities[name] = {} - base_flops = flops(program) - target_pruned_flops = base_flops * pruned_flops_rate - - pruner = Pruner() - baseline = None - for name in sensitivities: - - pruned_program, _, _ = pruner.prune( - program=graph.program, - scope=None, - params=[name], - ratios=[0.5], - place=None, - lazy=False, - only_graph=True) - param_flops = (base_flops - flops(pruned_program)) * 2 - channel_size = graph.var(name).shape()[0] - pruned_ratio = target_pruned_flops / float(param_flops) - pruned_ratio = round(pruned_ratio, 3) - pruned_size = round(pruned_ratio * channel_size) - pruned_ratio = 1 if pruned_size >= channel_size else pruned_ratio - - if len(sensitivities[name].keys()) > 0: - _logger.debug( - '{} exist; pruned ratio: {}; excepted ratio: {}'.format( - name, sensitivities[name].keys(), pruned_ratio)) - continue - if baseline is None: - baseline = eval_func(graph.program) - param_backup = {} - pruner = Pruner() - _logger.info("sensitive - param: {}; ratios: {}".format(name, - pruned_ratio)) - loss = 1 - if pruned_ratio < 1: - pruned_program = pruner.prune( - program=graph.program, - scope=scope, - params=[name], - ratios=[pruned_ratio], - place=place, - lazy=True, - only_graph=False, - param_backup=param_backup) - pruned_metric = eval_func(pruned_program) - loss = (baseline - pruned_metric) / baseline - _logger.info("pruned param: {}; {}; loss={}".format(name, pruned_ratio, - loss)) - sensitivities[name][pruned_ratio] = loss - _save_sensitivities(sensitivities, sensitivities_file) - - # restore pruned parameters - for param_name in param_backup.keys(): - param_t = scope.find_var(param_name).get_tensor() - param_t.set(param_backup[param_name], place) - return sensitivities - - def merge_sensitive(sensitivities): """Merge sensitivities. @@ -206,7 +131,7 @@ def merge_sensitive(sensitivities): """ assert len(sensitivities) > 0 if not isinstance(sensitivities[0], dict): - sensitivities = [pickle.load(open(sen, 'r')) for sen in sensitivities] + sensitivities = [load_sensitivities(sen) for sen in sensitivities] new_sensitivities = {} for sen in sensitivities: diff --git a/paddleslim/prune/sensitive_pruner.py b/paddleslim/prune/sensitive_pruner.py deleted file mode 100644 index c2164822..00000000 --- a/paddleslim/prune/sensitive_pruner.py +++ /dev/null @@ -1,243 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License" -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import logging -import copy -from scipy.optimize import leastsq -import numpy as np -import paddle.fluid as fluid -from ..common import get_logger -from .sensitive import sensitivity -from .sensitive import flops_sensitivity, get_ratios_by_loss -from ..analysis import flops -from .pruner import Pruner - -__all__ = ["SensitivePruner"] - -_logger = get_logger(__name__, level=logging.INFO) - - -class SensitivePruner(object): - """ - Pruner used to prune parameters iteratively according to sensitivities - of parameters in each step. - - Args: - place(fluid.CUDAPlace | fluid.CPUPlace): The device place where - program execute. - eval_func(function): A callback function used to evaluate pruned - program. The argument of this function is pruned program. - And it return a score of given program. - scope(fluid.scope): The scope used to execute program. - """ - - def __init__(self, place, eval_func, scope=None, checkpoints=None): - self._eval_func = eval_func - self._iter = 0 - self._place = place - self._scope = fluid.global_scope() if scope is None else scope - self._pruner = Pruner() - self._checkpoints = checkpoints - - def save_checkpoint(self, train_program, eval_program): - checkpoint = os.path.join(self._checkpoints, str(self._iter - 1)) - exe = fluid.Executor(self._place) - fluid.io.save_persistables( - exe, checkpoint, main_program=train_program, filename="__params__") - - with open(checkpoint + "/main_program", "wb") as f: - f.write(train_program.desc.serialize_to_string()) - with open(checkpoint + "/eval_program", "wb") as f: - f.write(eval_program.desc.serialize_to_string()) - - def restore(self, checkpoints=None): - - exe = fluid.Executor(self._place) - checkpoints = self._checkpoints if checkpoints is None else checkpoints - _logger.info("check points: {}".format(checkpoints)) - main_program = None - eval_program = None - if checkpoints is not None: - cks = [dir for dir in os.listdir(checkpoints)] - if len(cks) > 0: - latest = max([int(ck) for ck in cks]) - latest_ck_path = os.path.join(checkpoints, str(latest)) - self._iter += 1 - - with open(latest_ck_path + "/main_program", "rb") as f: - program_desc_str = f.read() - main_program = fluid.Program.parse_from_string( - program_desc_str) - - with open(latest_ck_path + "/eval_program", "rb") as f: - program_desc_str = f.read() - eval_program = fluid.Program.parse_from_string( - program_desc_str) - - with fluid.scope_guard(self._scope): - fluid.io.load_persistables(exe, latest_ck_path, - main_program, "__params__") - _logger.info("load checkpoint from: {}".format(latest_ck_path)) - _logger.info("flops of eval program: {}".format( - flops(eval_program))) - return main_program, eval_program, self._iter - - def greedy_prune(self, - train_program, - eval_program, - params, - pruned_flops_rate, - topk=1): - - sensitivities_file = "greedy_sensitivities_iter{}.data".format( - self._iter) - with fluid.scope_guard(self._scope): - sensitivities = flops_sensitivity( - eval_program, - self._place, - params, - self._eval_func, - sensitivities_file=sensitivities_file, - pruned_flops_rate=pruned_flops_rate) - _logger.info(sensitivities) - params, ratios = self._greedy_ratio_by_sensitive(sensitivities, topk) - - _logger.info("Pruning: {} by {}".format(params, ratios)) - pruned_program = self._pruner.prune( - train_program, - self._scope, - params, - ratios, - place=self._place, - only_graph=False) - pruned_val_program = None - if eval_program is not None: - pruned_val_program = self._pruner.prune( - eval_program, - self._scope, - params, - ratios, - place=self._place, - only_graph=True) - self._iter += 1 - return pruned_program, pruned_val_program - - def prune(self, train_program, eval_program, params, pruned_flops): - """ - Pruning parameters of training and evaluation network by sensitivities in current step. - - Args: - train_program(fluid.Program): The training program to be pruned. - eval_program(fluid.Program): The evaluation program to be pruned. And it is also used to calculate sensitivities of parameters. - params(list): The parameters to be pruned. - pruned_flops(float): The ratio of FLOPS to be pruned in current step. - - Returns: - tuple: A tuple of pruned training program and pruned evaluation program. - """ - _logger.info("Pruning: {}".format(params)) - sensitivities_file = "sensitivities_iter{}.data".format(self._iter) - with fluid.scope_guard(self._scope): - sensitivities = sensitivity( - eval_program, - self._place, - params, - self._eval_func, - sensitivities_file=sensitivities_file, - step_size=0.1) - _logger.info(sensitivities) - _, ratios = self.get_ratios_by_sensitive(sensitivities, pruned_flops, - eval_program) - - pruned_program = self._pruner.prune( - train_program, - self._scope, - params, - ratios, - place=self._place, - only_graph=False) - pruned_val_program = None - if eval_program is not None: - pruned_val_program = self._pruner.prune( - eval_program, - self._scope, - params, - ratios, - place=self._place, - only_graph=True) - self._iter += 1 - return pruned_program, pruned_val_program - - def _greedy_ratio_by_sensitive(self, sensitivities, topk=1): - losses = {} - percents = {} - for param in sensitivities: - losses[param] = sensitivities[param]['loss'][0] - percents[param] = sensitivities[param]['pruned_percent'][0] - topk_parms = sorted(losses, key=losses.__getitem__)[:topk] - topk_percents = [percents[param] for param in topk_parms] - return topk_parms, topk_percents - - def get_ratios_by_sensitive(self, sensitivities, pruned_flops, - eval_program): - """ - Search a group of ratios for pruning target flops. - - Args: - - sensitivities(dict): The sensitivities used to generate a group of pruning ratios. The key of dict - is name of parameters to be pruned. The value of dict is a list of tuple with - format `(pruned_ratio, accuracy_loss)`. - pruned_flops(float): The percent of FLOPS to be pruned. - eval_program(Program): The program whose FLOPS is considered. - - Returns: - - dict: A group of ratios. The key of dict is name of parameters while the value is the ratio to be pruned. - """ - - min_loss = 0. - max_loss = 0. - # step 2: Find a group of ratios by binary searching. - base_flops = flops(eval_program) - ratios = None - max_times = 20 - while min_loss < max_loss and max_times > 0: - loss = (max_loss + min_loss) / 2 - _logger.info( - '-----------Try pruned ratios while acc loss={}-----------'. - format(loss)) - ratios = self.get_ratios_by_loss(sensitivities, loss) - _logger.info('Pruned ratios={}'.format( - [round(ratio, 3) for ratio in ratios.values()])) - pruned_program = self._pruner.prune( - eval_program, - None, # scope - ratios.keys(), - ratios.values(), - None, # place - only_graph=True) - pruned_ratio = 1 - (float(flops(pruned_program)) / base_flops) - _logger.info('Pruned flops: {:.4f}'.format(pruned_ratio)) - - # Check whether current ratios is enough - if abs(pruned_ratio - pruned_flops) < 0.015: - break - if pruned_ratio > pruned_flops: - max_loss = loss - else: - min_loss = loss - max_times -= 1 - return ratios -- GitLab