From 5df65e506d1f4ff7f588437cc35f01fdb69717eb Mon Sep 17 00:00:00 2001 From: whs Date: Fri, 7 Jun 2019 17:10:17 +0800 Subject: [PATCH] Add Ligth-NAS for PaddleSlim (#17679) * Add auto pruning strategy. 1. Fix compressor. 2. Enhence graph executor. 3. Add SAController 4. Add auto pruning strategy. 5. Add unitest for auto pruning strategy. test=develop * Init light-nas * Add light nas. * Some fix. test=develop * Fix sa controller. test=develop * Fix unitest of light nas. test=develop * Fix setup.py.in and API.spec. test=develop * Fix unitest. 1. Fix unitest on windows. 2. Fix package importing in tests directory. * 1. Remove unused comments. 2. Expose eval_epoch option. 3. Remove unused function in search_agent. 4. Expose max_client_num to yaml file. 5. Move flops constraint to on_epoch_begin function test=develop * Fix light nas strategy. test=develop * Make controller server stable. test=develop * 1. Add try exception to compressor. 2. Remove unitest of light-nas for windows. test=develop * Add comments Enhence controller test=develop * Fix comments. test=develop --- paddle/fluid/API.spec | 2 +- python/paddle/fluid/compiler.py | 1 - .../fluid/contrib/slim/core/compressor.py | 138 ++++--- .../paddle/fluid/contrib/slim/core/config.py | 14 +- .../fluid/contrib/slim/graph/executor.py | 1 + .../fluid/contrib/slim/graph/graph_wrapper.py | 8 +- .../paddle/fluid/contrib/slim/nas/__init__.py | 29 ++ .../contrib/slim/nas/controller_server.py | 106 ++++++ .../contrib/slim/nas/light_nas_strategy.py | 178 +++++++++ python/paddle/fluid/contrib/slim/nas/lock.py | 36 ++ .../fluid/contrib/slim/nas/search_agent.py | 67 ++++ .../fluid/contrib/slim/nas/search_space.py | 43 +++ .../fluid/contrib/slim/prune/__init__.py | 3 + .../contrib/slim/prune/auto_prune_strategy.py | 249 +++++++++++++ .../contrib/slim/prune/prune_strategy.py | 129 ++++--- .../fluid/contrib/slim/searcher/__init__.py | 18 + .../fluid/contrib/slim/searcher/controller.py | 147 ++++++++ .../fluid/contrib/slim/tests/CMakeLists.txt | 4 + .../slim/tests/auto_pruning/compress.yaml | 30 ++ .../slim/tests/light_nas/compress.yaml | 22 ++ .../slim/tests/light_nas/light_nas_space.py | 277 ++++++++++++++ .../slim/tests/light_nas/light_nasnet.py | 339 ++++++++++++++++++ .../contrib/slim/tests/test_auto_pruning.py | 86 +++++ .../contrib/slim/tests/test_light_nas.py | 66 ++++ python/setup.py.in | 2 + 25 files changed, 1896 insertions(+), 99 deletions(-) create mode 100644 python/paddle/fluid/contrib/slim/nas/__init__.py create mode 100644 python/paddle/fluid/contrib/slim/nas/controller_server.py create mode 100644 python/paddle/fluid/contrib/slim/nas/light_nas_strategy.py create mode 100644 python/paddle/fluid/contrib/slim/nas/lock.py create mode 100644 python/paddle/fluid/contrib/slim/nas/search_agent.py create mode 100644 python/paddle/fluid/contrib/slim/nas/search_space.py create mode 100644 python/paddle/fluid/contrib/slim/prune/auto_prune_strategy.py create mode 100644 python/paddle/fluid/contrib/slim/searcher/__init__.py create mode 100644 python/paddle/fluid/contrib/slim/searcher/controller.py create mode 100644 python/paddle/fluid/contrib/slim/tests/auto_pruning/compress.yaml create mode 100644 python/paddle/fluid/contrib/slim/tests/light_nas/compress.yaml create mode 100644 python/paddle/fluid/contrib/slim/tests/light_nas/light_nas_space.py create mode 100644 python/paddle/fluid/contrib/slim/tests/light_nas/light_nasnet.py create mode 100644 python/paddle/fluid/contrib/slim/tests/test_auto_pruning.py create mode 100644 python/paddle/fluid/contrib/slim/tests/test_light_nas.py diff --git a/paddle/fluid/API.spec b/paddle/fluid/API.spec index 4ce2658601..67f654e72f 100644 --- a/paddle/fluid/API.spec +++ b/paddle/fluid/API.spec @@ -403,7 +403,7 @@ paddle.fluid.contrib.Calibrator.sample_data (ArgSpec(args=['self'], varargs=None paddle.fluid.contrib.Calibrator.save_int8_model (ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None), ('document', '6adf97f83acf6453d4a6a4b1070f3754')) paddle.fluid.contrib.distributed_sampler (ArgSpec(args=['reader', 'batch_size'], varargs=None, keywords=None, defaults=None), ('document', '9a271cd9700deb6d837ed724ba094315')) paddle.fluid.contrib.reader.ctr_reader.ctr_reader (ArgSpec(args=['feed_dict', 'file_type', 'file_format', 'dense_slot_index', 'sparse_slot_index', 'capacity', 'thread_num', 'batch_size', 'file_list', 'slots', 'name'], varargs=None, keywords=None, defaults=(None,)), ('document', 'b2ebf3de2a6ef1af2c3b88d2db7591ab')) -paddle.fluid.contrib.Compressor.__init__ (ArgSpec(args=['self', 'place', 'scope', 'train_program', 'train_reader', 'train_feed_list', 'train_fetch_list', 'eval_program', 'eval_reader', 'eval_feed_list', 'eval_fetch_list', 'teacher_programs', 'checkpoint_path', 'train_optimizer', 'distiller_optimizer'], varargs=None, keywords=None, defaults=(None, None, None, None, None, None, None, [], './checkpoints', None, None)), ('document', '31ae143830c9bf6b43547dd546c5ba80')) +paddle.fluid.contrib.Compressor.__init__ (ArgSpec(args=['self', 'place', 'scope', 'train_program', 'train_reader', 'train_feed_list', 'train_fetch_list', 'eval_program', 'eval_reader', 'eval_feed_list', 'eval_fetch_list', 'teacher_programs', 'checkpoint_path', 'train_optimizer', 'distiller_optimizer', 'search_space'], varargs=None, keywords=None, defaults=(None, None, None, None, None, None, None, [], None, None, None, None)), ('document', 'c195b3bba26169cff9439e8c467557c0')) paddle.fluid.contrib.Compressor.config (ArgSpec(args=['self', 'config_file'], varargs=None, keywords=None, defaults=None), ('document', '780d9c007276ccbb95b292400d7807b0')) paddle.fluid.contrib.Compressor.run (ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None), ('document', 'c6e43d6a078d307672283c1f36e04fe9')) paddle.fluid.contrib.load_persistables_for_increment (ArgSpec(args=['dirname', 'executor', 'program', 'lookup_table_var', 'lookup_table_var_path'], varargs=None, keywords=None, defaults=None), ('document', '2ab36d4f7a564f5f65e455807ad06c67')) diff --git a/python/paddle/fluid/compiler.py b/python/paddle/fluid/compiler.py index 09c3ecaf38..87a6ce0881 100644 --- a/python/paddle/fluid/compiler.py +++ b/python/paddle/fluid/compiler.py @@ -107,7 +107,6 @@ class CompiledProgram(object): raise ValueError("Wrong program_to_graph type: %s" % type(program_or_graph)) - self._program_desc = self._graph.origin_program_desc() self._scope = None self._place = None self._executor = None diff --git a/python/paddle/fluid/contrib/slim/core/compressor.py b/python/paddle/fluid/contrib/slim/core/compressor.py index 2344d95eae..f6c1552420 100644 --- a/python/paddle/fluid/contrib/slim/core/compressor.py +++ b/python/paddle/fluid/contrib/slim/core/compressor.py @@ -12,8 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ....core import CPUPlace +from ....core import CPUPlace, EOFException from .... import compiler +from ....framework import Variable from .... import io from .... import profiler from .... import scope_guard @@ -28,6 +29,7 @@ import logging import sys import pickle import functools +import traceback __all__ = ['Context', 'Compressor'] @@ -83,7 +85,8 @@ class Context(object): eval_reader=None, teacher_graphs=None, train_optimizer=None, - distiller_optimizer=None): + distiller_optimizer=None, + search_space=None): """ Args: place: The device place where the compression job running. @@ -119,6 +122,9 @@ class Context(object): self.cache_path = './eval_cache' self.eval_results = {} + self.skip_training = False + self.search_space = search_space + def to_file(self, file_name): """ Save the context into file. @@ -181,14 +187,30 @@ class Context(object): if sampled_rate: reader = cached_reader(reader, sampled_rate, self.cache_path, cached_id) - for data in reader(): - result = executor.run(eval_graph, self.scope, data=data) - result = [np.mean(r) for r in result] - results.append(result) - if batch_id % 20 == 0: - _logger.info("batch-{}; {}={}".format( - batch_id, eval_graph.out_nodes.keys(), result)) - batch_id += 1 + + if isinstance(reader, Variable): + reader.start() + try: + while True: + result = executor.run(eval_graph, self.scope) + result = [np.mean(r) for r in result] + results.append(result) + if batch_id % 20 == 0: + _logger.info("batch-{}; {}={}".format( + batch_id, eval_graph.out_nodes.keys(), result)) + batch_id += 1 + except EOFException: + reader.reset() + else: + for data in reader(): + result = executor.run(eval_graph, self.scope, data=data) + result = [np.mean(r) for r in result] + results.append(result) + if batch_id % 20 == 0: + _logger.info("batch-{}; {}={}".format( + batch_id, eval_graph.out_nodes.keys(), result)) + batch_id += 1 + result = np.mean(np.array(results), axis=0) _logger.info("Final eval result: {}={}".format( eval_graph.out_nodes.keys(), result)) @@ -221,9 +243,10 @@ class Compressor(object): eval_feed_list=None, eval_fetch_list=None, teacher_programs=[], - checkpoint_path='./checkpoints', + checkpoint_path=None, train_optimizer=None, - distiller_optimizer=None): + distiller_optimizer=None, + search_space=None): """ Args: place(fluid.Place): The device place where the compression job running. @@ -251,12 +274,14 @@ class Compressor(object): this optimizer is used to minimize the combined loss of student-net and teacher-net while train_optimizer is used to minimize loss of student-net in fine-tune stage. + search_space(slim.nas.SearchSpace): The instance that define the searching space. It must inherite + slim.nas.SearchSpace class and overwrite the abstract methods. """ - assert isinstance( + assert train_feed_list is None or isinstance( train_feed_list, list ), "train_feed_list should be a list of tuple, such as [('image', image.name), ('label', gt.name)]" - assert isinstance( + assert eval_feed_list is None or isinstance( eval_feed_list, list ), "eval_feed_list should be a list of tuple, such as [('image', image.name), ('label', gt.name)]" self.strategies = [] @@ -281,6 +306,8 @@ class Compressor(object): self.distiller_optimizer = distiller_optimizer self.init_model = None + self.search_space = search_space + def _add_strategy(self, strategy): """ Add a strategy to current compress pass. @@ -306,6 +333,9 @@ class Compressor(object): if 'init_model' in factory.compressor: self.init_model = factory.compressor['init_model'] + if 'eval_epoch' in factory.compressor: + self.eval_epoch = factory.compressor['eval_epoch'] + def _init_model(self, context): """ Load model that has been compressed. @@ -402,7 +432,8 @@ class Compressor(object): """ Train one epoch. """ - + if context.skip_training: + return executor = SlimGraphExecutor(self.place) if context.optimize_graph.compiled_graph is None: @@ -410,21 +441,44 @@ class Compressor(object): context.optimize_graph.program).with_data_parallel( loss_name=context.optimize_graph.out_nodes['loss']) - for data in context.train_reader(): - for strategy in self.strategies: - strategy.on_batch_begin(context) - results = executor.run(context.optimize_graph, - context.scope, - data=data) - results = [float(np.mean(result)) for result in results] - if context.batch_id % 20 == 0: - _logger.info("epoch:{}; batch_id:{}; {} = {}".format( - context.epoch_id, context.batch_id, - context.optimize_graph.out_nodes.keys( - ), [round(r, 3) for r in results])) - for strategy in self.strategies: - strategy.on_batch_end(context) - context.batch_id += 1 + if isinstance(context.train_reader, Variable): + context.train_reader.start() + try: + while True: + + for strategy in self.strategies: + strategy.on_batch_begin(context) + results = executor.run(context.optimize_graph, + context.scope) + results = [float(np.mean(result)) for result in results] + if context.batch_id % 20 == 0: + _logger.info("epoch:{}; batch_id:{}; {} = {}".format( + context.epoch_id, context.batch_id, + context.optimize_graph.out_nodes.keys( + ), [round(r, 3) for r in results])) + for strategy in self.strategies: + strategy.on_batch_end(context) + context.batch_id += 1 + + except EOFException: + context.train_reader.reset() + + else: + for data in context.train_reader(): + for strategy in self.strategies: + strategy.on_batch_begin(context) + results = executor.run(context.optimize_graph, + context.scope, + data=data) + results = [float(np.mean(result)) for result in results] + if context.batch_id % 20 == 0: + _logger.info("epoch:{}; batch_id:{}; {} = {}".format( + context.epoch_id, context.batch_id, + context.optimize_graph.out_nodes.keys( + ), [round(r, 3) for r in results])) + for strategy in self.strategies: + strategy.on_batch_end(context) + context.batch_id += 1 context.batch_id = 0 def _eval(self, context): @@ -450,7 +504,8 @@ class Compressor(object): eval_reader=self.eval_reader, teacher_graphs=self.teacher_graphs, train_optimizer=self.train_optimizer, - distiller_optimizer=self.distiller_optimizer) + distiller_optimizer=self.distiller_optimizer, + search_space=self.search_space) self.context = context if self.teacher_graphs: context.put('teachers', self.teacher_graphs) @@ -472,17 +527,20 @@ class Compressor(object): ]: return None start = context.epoch_id - self._eval(context) for epoch in range(start, self.epoch): context.epoch_id = epoch - for strategy in self.strategies: - strategy.on_epoch_begin(context) - self._train_one_epoch(context) - for strategy in self.strategies: - strategy.on_epoch_end(context) - if self.eval_epoch and epoch % self.eval_epoch == 0: - self._eval(context) - self._save_checkpoint(context) + try: + for strategy in self.strategies: + strategy.on_epoch_begin(context) + self._train_one_epoch(context) + if self.eval_epoch and epoch % self.eval_epoch == 0: + self._eval(context) + self._save_checkpoint(context) + for strategy in self.strategies: + strategy.on_epoch_end(context) + except Exception: + _logger.error(traceback.print_exc()) + continue for strategy in self.strategies: strategy.on_compression_end(context) return context.eval_graph diff --git a/python/paddle/fluid/contrib/slim/core/config.py b/python/paddle/fluid/contrib/slim/core/config.py index 9bb395aee9..9b08a0324a 100644 --- a/python/paddle/fluid/contrib/slim/core/config.py +++ b/python/paddle/fluid/contrib/slim/core/config.py @@ -20,11 +20,15 @@ from ..prune import * from ..quantization import * from .strategy import * from ..distillation import * +from ..searcher import * +from ..nas import * __all__ = ['ConfigFactory'] """This factory is used to create instances by loading and parsing configure file with yaml format. """ +PLUGINS = ['pruners', 'quantizers', 'quantizers', 'strategies', 'controllers'] + class ConfigFactory(object): def __init__(self, config): @@ -80,7 +84,7 @@ class ConfigFactory(object): assert self.version == int(key_values['version']) # parse pruners - if key == 'distillers' or key == 'pruners' or key == 'quantizers' or key == 'strategies': + if key in PLUGINS: instances = key_values[key] for name in instances: self._new_instance(name, instances[name]) @@ -91,8 +95,12 @@ class ConfigFactory(object): if 'init_model' in key_values[key]: self.compressor['init_model'] = key_values[key][ 'init_model'] - self.compressor['checkpoint_path'] = key_values[key][ - 'checkpoint_path'] + if 'checkpoint_path' in key_values[key]: + self.compressor['checkpoint_path'] = key_values[key][ + 'checkpoint_path'] + if 'eval_epoch' in key_values[key]: + self.compressor['eval_epoch'] = key_values[key][ + 'eval_epoch'] if 'strategies' in key_values[key]: for name in key_values[key]['strategies']: strategy = self.instance(name) diff --git a/python/paddle/fluid/contrib/slim/graph/executor.py b/python/paddle/fluid/contrib/slim/graph/executor.py index 70438a90eb..041ccbb3a3 100644 --- a/python/paddle/fluid/contrib/slim/graph/executor.py +++ b/python/paddle/fluid/contrib/slim/graph/executor.py @@ -41,6 +41,7 @@ class SlimGraphExecutor(object): results(list): A list of result with the same order indicated by graph.out_nodes. """ assert isinstance(graph, GraphWrapper) + feed = None if data is not None: feeder = DataFeeder( feed_list=graph.in_nodes.values(), diff --git a/python/paddle/fluid/contrib/slim/graph/graph_wrapper.py b/python/paddle/fluid/contrib/slim/graph/graph_wrapper.py index e7f5f0d6a2..689f644117 100644 --- a/python/paddle/fluid/contrib/slim/graph/graph_wrapper.py +++ b/python/paddle/fluid/contrib/slim/graph/graph_wrapper.py @@ -209,6 +209,7 @@ class GraphWrapper(object): if var.persistable: self.persistables[var.name] = var self.compiled_graph = None + in_nodes = [] if in_nodes is None else in_nodes self.in_nodes = OrderedDict(in_nodes) self.out_nodes = OrderedDict(out_nodes) self._attrs = OrderedDict() @@ -241,7 +242,7 @@ class GraphWrapper(object): """ return var._var.persistable - def compile(self, for_parallel=True, for_test=False): + def compile(self, for_parallel=True, for_test=False, mem_opt=False): """ Compile the program in this wrapper to framework.CompiledProgram for next running. This function must be called if the program is modified. @@ -257,8 +258,9 @@ class GraphWrapper(object): if for_parallel: # disable memory optimize for stable training build_strategy = compiler.BuildStrategy() - build_strategy.enable_inplace = False - build_strategy.memory_optimize = False + build_strategy.enable_inplace = mem_opt + build_strategy.memory_optimize = mem_opt + # build_strategy.async_mode = False self.compiled_graph = compiler.CompiledProgram( target).with_data_parallel( loss_name=loss, build_strategy=build_strategy) diff --git a/python/paddle/fluid/contrib/slim/nas/__init__.py b/python/paddle/fluid/contrib/slim/nas/__init__.py new file mode 100644 index 0000000000..7330a20751 --- /dev/null +++ b/python/paddle/fluid/contrib/slim/nas/__init__.py @@ -0,0 +1,29 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import light_nas_strategy +from .light_nas_strategy import * +from . import controller_server +from .controller_server import * +from . import search_agent +from .search_agent import * +from . import search_space +from .search_space import * +from . import lock +from .lock import * + +__all__ = light_nas_strategy.__all__ +__all__ += controller_server.__all__ +__all__ += search_agent.__all__ +__all__ += search_space.__all__ diff --git a/python/paddle/fluid/contrib/slim/nas/controller_server.py b/python/paddle/fluid/contrib/slim/nas/controller_server.py new file mode 100644 index 0000000000..d6091007c1 --- /dev/null +++ b/python/paddle/fluid/contrib/slim/nas/controller_server.py @@ -0,0 +1,106 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import socket +from threading import Thread + +__all__ = ['ControllerServer'] + +logging.basicConfig( + format='ControllerServer-%(asctime)s-%(levelname)s: %(message)s') +_logger = logging.getLogger(__name__) +_logger.setLevel(logging.INFO) + + +class ControllerServer(object): + """ + The controller wrapper with a socket server to handle the request of search agentt. + """ + + def __init__(self, + controller=None, + address=('', 0), + max_client_num=100, + search_steps=None, + key=None): + """ + Args: + controller(slim.searcher.Controller): The controller used to generate tokens. + address(tuple): The address of current server binding with format (ip, port). Default: ('', 0). + which means setting ip automatically + max_client_num(int): The maximum number of clients connecting to current server simultaneously. Default: 100. + search_steps(int): The total steps of searching. None means never stopping. Default: None + """ + self._controller = controller + self._address = address + self._max_client_num = max_client_num + self._search_steps = search_steps + self._closed = False + self._port = address[1] + self._ip = address[0] + self._key = key + + def start(self): + self._socket_server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._socket_server.bind(self._address) + self._socket_server.listen(self._max_client_num) + self._port = self._socket_server.getsockname()[1] + self._ip = self._socket_server.getsockname()[0] + _logger.info("listen on: [{}:{}]".format(self._ip, self._port)) + thread = Thread(target=self.run) + thread.start() + return str(thread) + + def close(self): + """Close the server.""" + self._closed = True + + def port(self): + """Get the port.""" + return self._port + + def ip(self): + """Get the ip.""" + return self._ip + + def run(self): + _logger.info("Controller Server run...") + while ((self._search_steps is None) or + (self._controller._iter < + (self._search_steps))) and not self._closed: + conn, addr = self._socket_server.accept() + message = conn.recv(1024).decode() + if message.strip("\n") == "next_tokens": + tokens = self._controller.next_tokens() + tokens = ",".join([str(token) for token in tokens]) + conn.send(tokens.encode()) + else: + _logger.info("recv message from {}: [{}]".format(addr, message)) + messages = message.strip('\n').split("\t") + if (len(messages) < 3) or (messages[0] != self._key): + _logger.info("recv noise from {}: [{}]".format(addr, + message)) + continue + tokens = messages[1] + reward = messages[2] + tokens = [int(token) for token in tokens.split(",")] + self._controller.update(tokens, float(reward)) + tokens = self._controller.next_tokens() + tokens = ",".join([str(token) for token in tokens]) + conn.send(tokens.encode()) + _logger.info("send message to {}: [{}]".format(addr, tokens)) + conn.close() + self._socket_server.close() + _logger.info("server closed!") diff --git a/python/paddle/fluid/contrib/slim/nas/light_nas_strategy.py b/python/paddle/fluid/contrib/slim/nas/light_nas_strategy.py new file mode 100644 index 0000000000..f5290a49b1 --- /dev/null +++ b/python/paddle/fluid/contrib/slim/nas/light_nas_strategy.py @@ -0,0 +1,178 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from ..core.strategy import Strategy +from ..graph import GraphWrapper +from .controller_server import ControllerServer +from .search_agent import SearchAgent +from ....executor import Executor +import re +import logging +import functools +import socket +from .lock import lock, unlock + +__all__ = ['LightNASStrategy'] + +logging.basicConfig( + format='LightNASStrategy-%(asctime)s-%(levelname)s: %(message)s') +_logger = logging.getLogger(__name__) +_logger.setLevel(logging.INFO) + + +class LightNASStrategy(Strategy): + """ + Light-NAS search strategy. + """ + + def __init__(self, + controller=None, + end_epoch=1000, + target_flops=629145600, + retrain_epoch=1, + metric_name='top1_acc', + server_ip=None, + server_port=0, + is_server=False, + max_client_num=100, + search_steps=None, + key="light-nas"): + """ + Args: + controller(searcher.Controller): The searching controller. Default: None. + end_epoch(int): The 'on_epoch_end' function will be called in end_epoch. Default: 0 + target_flops(int): The constraint of FLOPS. + retrain_epoch(int): The number of training epochs before evaluating structure generated by controller. Default: 1. + metric_name(str): The metric used to evaluate the model. + It should be one of keys in out_nodes of graph wrapper. Default: 'top1_acc' + server_ip(str): The ip that controller server listens on. None means getting the ip automatically. Default: None. + server_port(int): The port that controller server listens on. 0 means getting usable port automatically. Default: 0. + is_server(bool): Whether current host is controller server. Default: False. + max_client_num(int): The maximum number of clients that connect to controller server concurrently. Default: 100. + search_steps(int): The total steps of searching. Default: None. + key(str): The key used to identify legal agent for controller server. Default: "light-nas" + """ + self.start_epoch = 0 + self.end_epoch = end_epoch + self._max_flops = target_flops + self._metric_name = metric_name + self._controller = controller + self._retrain_epoch = 0 + self._server_ip = server_ip + self._server_port = server_port + self._is_server = is_server + self._retrain_epoch = retrain_epoch + self._search_steps = search_steps + self._max_client_num = max_client_num + self._max_try_times = 100 + self._key = key + + if self._server_ip is None: + self._server_ip = self._get_host_ip() + + def _get_host_ip(self): + + try: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect(('8.8.8.8', 80)) + ip = s.getsockname()[0] + finally: + s.close() + + return ip + + def on_compression_begin(self, context): + self._current_tokens = context.search_space.init_tokens() + constrain_func = functools.partial( + self._constrain_func, context=context) + self._controller.reset(context.search_space.range_table(), + self._current_tokens, None) + + # create controller server + if self._is_server: + open("./slim_LightNASStrategy_controller_server.socket", + 'a').close() + socket_file = open( + "./slim_LightNASStrategy_controller_server.socket", 'r+') + lock(socket_file) + tid = socket_file.readline() + if tid == '': + _logger.info("start controller server...") + self._server = ControllerServer( + controller=self._controller, + address=(self._server_ip, self._server_port), + max_client_num=self._max_client_num, + search_steps=self._search_steps, + key=self._key) + tid = self._server.start() + self._server_port = self._server.port() + socket_file.write(tid) + _logger.info("started controller server...") + unlock(socket_file) + socket_file.close() + _logger.info("self._server_ip: {}; self._server_port: {}".format( + self._server_ip, self._server_port)) + # create client + self._search_agent = SearchAgent( + self._server_ip, self._server_port, key=self._key) + + def _constrain_func(self, tokens, context=None): + """Check whether the tokens meet constraint.""" + _, _, test_prog, _, _, _, _ = context.search_space.create_net(tokens) + flops = GraphWrapper(test_prog).flops() + if flops <= self._max_flops: + return True + else: + return False + + def on_epoch_begin(self, context): + if context.epoch_id >= self.start_epoch and context.epoch_id <= self.end_epoch and ( + self._retrain_epoch == 0 or + (context.epoch_id - self.start_epoch) % self._retrain_epoch == 0): + _logger.info("light nas strategy on_epoch_begin") + for _ in range(self._max_try_times): + startup_p, train_p, test_p, _, _, train_reader, test_reader = context.search_space.create_net( + self._current_tokens) + _logger.info("try [{}]".format(self._current_tokens)) + context.eval_graph.program = test_p + flops = context.eval_graph.flops() + if flops <= self._max_flops: + break + else: + self._current_tokens = self._search_agent.next_tokens() + + context.train_reader = train_reader + context.eval_reader = test_reader + + exe = Executor(context.place) + exe.run(startup_p) + + context.optimize_graph.program = train_p + context.optimize_graph.compile() + + context.skip_training = (self._retrain_epoch == 0) + + def on_epoch_end(self, context): + if context.epoch_id >= self.start_epoch and context.epoch_id < self.end_epoch and ( + self._retrain_epoch == 0 or + (context.epoch_id - self.start_epoch + 1 + ) % self._retrain_epoch == 0): + + self._current_reward = context.eval_results[self._metric_name][-1] + flops = context.eval_graph.flops() + if flops > self._max_flops: + self._current_reward = 0.0 + _logger.info("reward: {}; flops: {}; tokens: {}".format( + self._current_reward, flops, self._current_tokens)) + self._current_tokens = self._search_agent.update( + self._current_tokens, self._current_reward) diff --git a/python/paddle/fluid/contrib/slim/nas/lock.py b/python/paddle/fluid/contrib/slim/nas/lock.py new file mode 100644 index 0000000000..5edcd31730 --- /dev/null +++ b/python/paddle/fluid/contrib/slim/nas/lock.py @@ -0,0 +1,36 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +__All__ = ['lock', 'unlock'] +if os.name == 'nt': + + def lock(file): + raise NotImplementedError('Windows is not supported.') + + def unlock(file): + raise NotImplementedError('Windows is not supported.') + +elif os.name == 'posix': + from fcntl import flock, LOCK_EX, LOCK_UN + + def lock(file): + """Lock the file in local file system.""" + flock(file.fileno(), LOCK_EX) + + def unlock(file): + """Unlock the file in local file system.""" + flock(file.fileno(), LOCK_UN) +else: + raise RuntimeError("File Locker only support NT and Posix platforms!") diff --git a/python/paddle/fluid/contrib/slim/nas/search_agent.py b/python/paddle/fluid/contrib/slim/nas/search_agent.py new file mode 100644 index 0000000000..3d756c95bf --- /dev/null +++ b/python/paddle/fluid/contrib/slim/nas/search_agent.py @@ -0,0 +1,67 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import socket + +__all__ = ['SearchAgent'] + +logging.basicConfig(format='%(asctime)s-%(levelname)s: %(message)s') +_logger = logging.getLogger(__name__) +_logger.setLevel(logging.INFO) + + +class SearchAgent(object): + """ + Search agent. + """ + + def __init__(self, server_ip=None, server_port=None, key=None): + """ + Args: + server_ip(str): The ip that controller server listens on. None means getting the ip automatically. Default: None. + server_port(int): The port that controller server listens on. 0 means getting usable port automatically. Default: 0. + key(str): The key used to identify legal agent for controller server. Default: "light-nas" + """ + self.server_ip = server_ip + self.server_port = server_port + self.socket_client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._key = key + + def update(self, tokens, reward): + """ + Update the controller according to latest tokens and reward. + Args: + tokens(list): The tokens generated in last step. + reward(float): The reward of tokens. + """ + socket_client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + socket_client.connect((self.server_ip, self.server_port)) + tokens = ",".join([str(token) for token in tokens]) + socket_client.send("{}\t{}\t{}".format(self._key, tokens, reward) + .encode()) + tokens = socket_client.recv(1024).decode() + tokens = [int(token) for token in tokens.strip("\n").split(",")] + return tokens + + def next_tokens(self): + """ + Get next tokens. + """ + socket_client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + socket_client.connect((self.server_ip, self.server_port)) + socket_client.send("next_tokens".encode()) + tokens = socket_client.recv(1024).decode() + tokens = [int(token) for token in tokens.strip("\n").split(",")] + return tokens diff --git a/python/paddle/fluid/contrib/slim/nas/search_space.py b/python/paddle/fluid/contrib/slim/nas/search_space.py new file mode 100644 index 0000000000..af23d0a838 --- /dev/null +++ b/python/paddle/fluid/contrib/slim/nas/search_space.py @@ -0,0 +1,43 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The search space used to search neural architecture""" + +__all__ = ['SearchSpace'] + + +class SearchSpace(object): + """Controller for Neural Architecture Search. + """ + + def __init__(self, *args, **kwargs): + pass + + def init_tokens(self): + """Get init tokens in search space. + """ + raise NotImplementedError('Abstract method.') + + def range_table(self): + """Get range table of current search space. + """ + raise NotImplementedError('Abstract method.') + + def create_net(self, tokens): + """Create networks for training and evaluation according to tokens. + Args: + tokens(list): The tokens which represent a network. + Return: + (tuple): startup_program, train_program, evaluation_program, train_metrics, test_metrics + """ + raise NotImplementedError('Abstract method.') diff --git a/python/paddle/fluid/contrib/slim/prune/__init__.py b/python/paddle/fluid/contrib/slim/prune/__init__.py index 764a45bb13..ae487a21e3 100644 --- a/python/paddle/fluid/contrib/slim/prune/__init__.py +++ b/python/paddle/fluid/contrib/slim/prune/__init__.py @@ -16,6 +16,9 @@ from . import pruner from .pruner import * from . import prune_strategy from .prune_strategy import * +from . import auto_prune_strategy +from .auto_prune_strategy import * __all__ = pruner.__all__ __all__ += prune_strategy.__all__ +__all__ += auto_prune_strategy.__all__ diff --git a/python/paddle/fluid/contrib/slim/prune/auto_prune_strategy.py b/python/paddle/fluid/contrib/slim/prune/auto_prune_strategy.py new file mode 100644 index 0000000000..680b644fdd --- /dev/null +++ b/python/paddle/fluid/contrib/slim/prune/auto_prune_strategy.py @@ -0,0 +1,249 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .prune_strategy import PruneStrategy +import re +import logging +import functools +import copy + +__all__ = ['AutoPruneStrategy'] + +logging.basicConfig(format='%(asctime)s-%(levelname)s: %(message)s') +_logger = logging.getLogger(__name__) +_logger.setLevel(logging.INFO) + + +class AutoPruneStrategy(PruneStrategy): + """ + Automatic pruning strategy. + """ + + def __init__(self, + pruner=None, + controller=None, + start_epoch=0, + end_epoch=10, + min_ratio=0.5, + max_ratio=0.7, + metric_name='top1_acc', + pruned_params='conv.*_weights', + retrain_epoch=0): + """ + Args: + pruner(slim.Pruner): The pruner used to prune the parameters. Default: None. + controller(searcher.Controller): The searching controller. Default: None. + start_epoch(int): The 'on_epoch_begin' function will be called in start_epoch. Default: 0 + end_epoch(int): The 'on_epoch_end' function will be called in end_epoch. Default: 0 + min_ratio(float): The maximum pruned ratio. Default: 0.7 + max_ratio(float): The minimum pruned ratio. Default: 0.5 + metric_name(str): The metric used to evaluate the model. + It should be one of keys in out_nodes of graph wrapper. Default: 'top1_acc' + pruned_params(str): The pattern str to match the parameter names to be pruned. Default: 'conv.*_weights' + retrain_epoch(int): The training epochs in each seaching step. Default: 0 + """ + super(AutoPruneStrategy, self).__init__(pruner, start_epoch, end_epoch, + 0.0, metric_name, pruned_params) + self._max_ratio = max_ratio + self._min_ratio = min_ratio + self._controller = controller + self._metric_name = metric_name + self._pruned_param_names = [] + self._retrain_epoch = 0 + + self._current_tokens = None + + def on_compression_begin(self, context): + """ + Prepare some information for searching strategy. + step 1: Find all the parameters to be pruned. + step 2: Get initial tokens and setup controller. + """ + pruned_params = [] + for param in context.eval_graph.all_parameters(): + if re.match(self.pruned_params, param.name()): + self._pruned_param_names.append(param.name()) + + self._current_tokens = self._get_init_tokens(context) + self._range_table = copy.deepcopy(self._current_tokens) + + constrain_func = functools.partial( + self._constrain_func, context=context) + + self._controller.reset(self._range_table, self._current_tokens, + constrain_func) + + def _constrain_func(self, tokens, context=None): + """Check whether the tokens meet constraint.""" + ori_flops = context.eval_graph.flops() + ratios = self._tokens_to_ratios(tokens) + params = self._pruned_param_names + param_shape_backup = {} + self._prune_parameters( + context.eval_graph, + context.scope, + params, + ratios, + context.place, + only_graph=True, + param_shape_backup=param_shape_backup) + context.eval_graph.update_groups_of_conv() + flops = context.eval_graph.flops() + for param in param_shape_backup.keys(): + context.eval_graph.var(param).set_shape(param_shape_backup[param]) + flops_ratio = (1 - float(flops) / ori_flops) + if flops_ratio >= self._min_ratio and flops_ratio <= self._max_ratio: + return True + else: + return False + + def _get_init_tokens(self, context): + """Get initial tokens. + """ + ratios = self._get_uniform_ratios(context) + return self._ratios_to_tokens(ratios) + + def _ratios_to_tokens(self, ratios): + """Convert pruned ratios to tokens. + """ + return [int(ratio / 0.01) for ratio in ratios] + + def _tokens_to_ratios(self, tokens): + """Convert tokens to pruned ratios. + """ + return [token * 0.01 for token in tokens] + + def _get_uniform_ratios(self, context): + """ + Search a group of uniform ratios. + """ + min_ratio = 0. + max_ratio = 1. + target = (self._min_ratio + self._max_ratio) / 2 + flops = context.eval_graph.flops() + model_size = context.eval_graph.numel_params() + ratios = None + while min_ratio < max_ratio: + ratio = (max_ratio + min_ratio) / 2 + ratios = [ratio] * len(self._pruned_param_names) + param_shape_backup = {} + self._prune_parameters( + context.eval_graph, + context.scope, + self._pruned_param_names, + ratios, + context.place, + only_graph=True, + param_shape_backup=param_shape_backup) + + pruned_flops = 1 - (float(context.eval_graph.flops()) / flops) + pruned_size = 1 - (float(context.eval_graph.numel_params()) / + model_size) + for param in param_shape_backup.keys(): + context.eval_graph.var(param).set_shape(param_shape_backup[ + param]) + + if abs(pruned_flops - target) < 1e-2: + break + if pruned_flops > target: + max_ratio = ratio + else: + min_ratio = ratio + _logger.info('Get ratios: {}'.format([round(r, 2) for r in ratios])) + return ratios + + def on_epoch_begin(self, context): + """ + step 1: Get a new tokens from controller. + step 2: Pruning eval_graph and optimize_program by tokens + """ + if context.epoch_id >= self.start_epoch and context.epoch_id <= self.end_epoch and ( + self._retrain_epoch == 0 or + (context.epoch_id - self.start_epoch) % self._retrain_epoch == 0): + self._current_tokens = self._controller.next_tokens() + params = self._pruned_param_names + ratios = self._tokens_to_ratios(self._current_tokens) + + self._param_shape_backup = {} + self._param_backup = {} + self._prune_parameters( + context.optimize_graph, + context.scope, + params, + ratios, + context.place, + param_backup=self._param_backup, + param_shape_backup=self._param_shape_backup) + self._prune_graph(context.eval_graph, context.optimize_graph) + context.optimize_graph.update_groups_of_conv() + context.eval_graph.update_groups_of_conv() + context.optimize_graph.compile( + mem_opt=True) # to update the compiled program + context.skip_training = (self._retrain_epoch == 0) + + def on_epoch_end(self, context): + """ + step 1: Get reward of current tokens and update controller. + step 2: Restore eval_graph and optimize_graph + """ + if context.epoch_id >= self.start_epoch and context.epoch_id < self.end_epoch and ( + self._retrain_epoch == 0 or + (context.epoch_id - self.start_epoch) % self._retrain_epoch == 0): + reward = context.eval_results[self._metric_name][-1] + self._controller.update(self._current_tokens, reward) + + # restore pruned parameters + for param_name in self._param_backup.keys(): + param_t = context.scope.find_var(param_name).get_tensor() + param_t.set(self._param_backup[param_name], context.place) + self._param_backup = {} + # restore shape of parameters + for param in self._param_shape_backup.keys(): + context.optimize_graph.var(param).set_shape( + self._param_shape_backup[param]) + self._param_shape_backup = {} + self._prune_graph(context.eval_graph, context.optimize_graph) + + context.optimize_graph.update_groups_of_conv() + context.eval_graph.update_groups_of_conv() + context.optimize_graph.compile( + mem_opt=True) # to update the compiled program + + elif context.epoch_id == self.end_epoch: # restore graph for final training + # restore pruned parameters + for param_name in self._param_backup.keys(): + param_t = context.scope.find_var(param_name).get_tensor() + param_t.set(self.param_backup[param_name], context.place) + # restore shape of parameters + for param in self._param_shape_backup.keys(): + context.eval_graph.var(param).set_shape( + self._param_shape_backup[param]) + context.optimize_graph.var(param).set_shape( + self._param_shape_backup[param]) + + context.optimize_graph.update_groups_of_conv() + context.eval_graph.update_groups_of_conv() + + params, ratios = self._get_prune_ratios( + self._controller._best_tokens) + self._prune_parameters(context.optimize_graph, context.scope, + params, ratios, context.place) + + self._prune_graph(context.eval_graph, context.optimize_graph) + context.optimize_graph.update_groups_of_conv() + context.eval_graph.update_groups_of_conv() + context.optimize_graph.compile( + mem_opt=True) # to update the compiled program + + context.skip_training = False diff --git a/python/paddle/fluid/contrib/slim/prune/prune_strategy.py b/python/paddle/fluid/contrib/slim/prune/prune_strategy.py index 7a25c3a61e..8bdcd8a90f 100644 --- a/python/paddle/fluid/contrib/slim/prune/prune_strategy.py +++ b/python/paddle/fluid/contrib/slim/prune/prune_strategy.py @@ -26,7 +26,7 @@ import pickle import logging import sys -__all__ = ['SensitivePruneStrategy', 'UniformPruneStrategy'] +__all__ = ['SensitivePruneStrategy', 'UniformPruneStrategy', 'PruneStrategy'] logging.basicConfig(format='%(asctime)s-%(levelname)s: %(message)s') _logger = logging.getLogger(__name__) @@ -61,8 +61,6 @@ class PruneStrategy(Strategy): self.metric_name = metric_name self.pruned_params = pruned_params self.pruned_list = [] - self.backup = {} - self.param_shape_backup = {} def _eval_graph(self, context, sampled_rate=None, cached_id=0): """ @@ -82,7 +80,9 @@ class PruneStrategy(Strategy): ratio, place, lazy=False, - only_graph=False): + only_graph=False, + param_shape_backup=None, + param_backup=None): """ Pruning filters by given ratio. Args: @@ -103,16 +103,16 @@ class PruneStrategy(Strategy): for param in params: assert isinstance(param, VarWrapper) param_t = scope.find_var(param.name()).get_tensor() - if lazy: - self.backup[param.name()] = copy.deepcopy(np.array(param_t)) + if param_backup is not None and (param.name() not in param_backup): + param_backup[param.name()] = copy.deepcopy(np.array(param_t)) pruned_param = self.pruner.prune_tensor( np.array(param_t), pruned_idx, pruned_axis=0, lazy=lazy) if not only_graph: param_t.set(pruned_param, place) ori_shape = param.shape() - if param.name() not in self.param_shape_backup: - self.param_shape_backup[param.name()] = copy.deepcopy( - param.shape()) + if param_shape_backup is not None and ( + param.name() not in param_shape_backup): + param_shape_backup[param.name()] = copy.deepcopy(param.shape()) new_shape = list(param.shape()) new_shape[0] = pruned_param.shape[0] param.set_shape(new_shape) @@ -120,7 +120,8 @@ class PruneStrategy(Strategy): '|----------------------------------------+----+------------------------------+------------------------------|' ) _logger.debug('|{:^40}|{:^4}|{:^30}|{:^30}|'.format( - str(param.name()), str(0), str(ori_shape), str(param.shape()))) + str(param.name()), + str(ratio), str(ori_shape), str(param.shape()))) self.pruned_list[0].append(param.name()) return pruned_idx @@ -131,7 +132,9 @@ class PruneStrategy(Strategy): pruned_axis, place, lazy=False, - only_graph=False): + only_graph=False, + param_shape_backup=None, + param_backup=None): """ Pruning parameters in given axis. Args: @@ -150,16 +153,17 @@ class PruneStrategy(Strategy): for param in params: assert isinstance(param, VarWrapper) param_t = scope.find_var(param.name()).get_tensor() - if lazy: - self.backup[param.name()] = copy.deepcopy(np.array(param_t)) + if param_backup is not None and (param.name() not in param_backup): + param_backup[param.name()] = copy.deepcopy(np.array(param_t)) pruned_param = self.pruner.prune_tensor( np.array(param_t), pruned_idx, pruned_axis, lazy=lazy) if not only_graph: param_t.set(pruned_param, place) ori_shape = param.shape() - if param.name() not in self.param_shape_backup: - self.param_shape_backup[param.name()] = copy.deepcopy( - param.shape()) + + if param_shape_backup is not None and ( + param.name() not in param_shape_backup): + param_shape_backup[param.name()] = copy.deepcopy(param.shape()) new_shape = list(param.shape()) new_shape[pruned_axis] = pruned_param.shape[pruned_axis] param.set_shape(new_shape) @@ -251,7 +255,9 @@ class PruneStrategy(Strategy): ratio=None, pruned_idxs=None, lazy=False, - only_graph=False): + only_graph=False, + param_backup=None, + param_shape_backup=None): """ Pruning all the parameters affected by the pruning of given parameter. Args: @@ -284,7 +290,9 @@ class PruneStrategy(Strategy): pruned_axis=0, place=place, lazy=lazy, - only_graph=only_graph) + only_graph=only_graph, + param_backup=param_backup, + param_shape_backup=param_shape_backup) else: pruned_idxs = self._prune_filters_by_ratio( @@ -292,7 +300,9 @@ class PruneStrategy(Strategy): ratio, place, lazy=lazy, - only_graph=only_graph) + only_graph=only_graph, + param_backup=param_backup, + param_shape_backup=param_shape_backup) corrected_idxs = pruned_idxs[:] for idx, op in enumerate(related_ops): @@ -307,7 +317,9 @@ class PruneStrategy(Strategy): pruned_axis=1, place=place, lazy=lazy, - only_graph=only_graph) + only_graph=only_graph, + param_backup=param_backup, + param_shape_backup=param_shape_backup) if op.type() == "depthwise_conv2d": for in_var in op.all_inputs(): if graph.is_parameter(in_var): @@ -319,7 +331,9 @@ class PruneStrategy(Strategy): pruned_axis=0, place=place, lazy=lazy, - only_graph=only_graph) + only_graph=only_graph, + param_backup=param_backup, + param_shape_backup=param_shape_backup) elif op.type() == "elementwise_add": # pruning bias for in_var in op.all_inputs(): @@ -332,7 +346,9 @@ class PruneStrategy(Strategy): pruned_axis=0, place=place, lazy=lazy, - only_graph=only_graph) + only_graph=only_graph, + param_backup=param_backup, + param_shape_backup=param_shape_backup) elif op.type() == "mul": # pruning fc layer fc_input = None fc_param = None @@ -354,7 +370,9 @@ class PruneStrategy(Strategy): pruned_axis=0, place=place, lazy=lazy, - only_graph=only_graph) + only_graph=only_graph, + param_backup=param_backup, + param_shape_backup=param_shape_backup) elif op.type() == "concat": concat_inputs = op.all_inputs() @@ -378,28 +396,36 @@ class PruneStrategy(Strategy): pruned_axis=0, place=place, lazy=lazy, - only_graph=only_graph) + only_graph=only_graph, + param_backup=param_backup, + param_shape_backup=param_shape_backup) self._prune_parameter_by_idx( scope, [variance] + self._get_accumulator(graph, variance), corrected_idxs, pruned_axis=0, place=place, lazy=lazy, - only_graph=only_graph) + only_graph=only_graph, + param_backup=param_backup, + param_shape_backup=param_shape_backup) self._prune_parameter_by_idx( scope, [alpha] + self._get_accumulator(graph, alpha), corrected_idxs, pruned_axis=0, place=place, lazy=lazy, - only_graph=only_graph) + only_graph=only_graph, + param_backup=param_backup, + param_shape_backup=param_shape_backup) self._prune_parameter_by_idx( scope, [beta] + self._get_accumulator(graph, beta), corrected_idxs, pruned_axis=0, place=place, lazy=lazy, - only_graph=only_graph) + only_graph=only_graph, + param_backup=param_backup, + param_shape_backup=param_shape_backup) def _prune_parameters(self, graph, @@ -408,7 +434,9 @@ class PruneStrategy(Strategy): ratios, place, lazy=False, - only_graph=False): + only_graph=False, + param_backup=None, + param_shape_backup=None): """ Pruning the given parameters. Args: @@ -444,7 +472,9 @@ class PruneStrategy(Strategy): place, ratio=ratio, lazy=lazy, - only_graph=only_graph) + only_graph=only_graph, + param_backup=param_backup, + param_shape_backup=param_shape_backup) ops = param.outputs() for op in ops: if op.type() == 'conv2d': @@ -458,7 +488,9 @@ class PruneStrategy(Strategy): place, ratio=ratio, lazy=lazy, - only_graph=only_graph) + only_graph=only_graph, + param_backup=param_backup, + param_shape_backup=param_shape_backup) _logger.debug( '|----------------------------------------+----+------------------------------+------------------------------|' ) @@ -575,23 +607,24 @@ class UniformPruneStrategy(PruneStrategy): _logger.debug( '-----------Try pruning ratio: {:.2f}-----------'.format(ratio)) ratios = [ratio] * len(pruned_params) + param_shape_backup = {} self._prune_parameters( context.eval_graph, context.scope, pruned_params, ratios, context.place, - only_graph=True) + only_graph=True, + param_shape_backup=param_shape_backup) pruned_flops = 1 - (float(context.eval_graph.flops()) / flops) pruned_size = 1 - (float(context.eval_graph.numel_params()) / model_size) _logger.debug('Pruned flops: {:.2f}'.format(pruned_flops)) _logger.debug('Pruned model size: {:.2f}'.format(pruned_size)) - for param in self.param_shape_backup.keys(): - context.eval_graph.var(param).set_shape(self.param_shape_backup[ + for param in param_shape_backup.keys(): + context.eval_graph.var(param).set_shape(param_shape_backup[ param]) - self.param_shape_backup = {} if abs(pruned_flops - self.target_ratio) < 1e-2: break @@ -672,8 +705,6 @@ class SensitivePruneStrategy(PruneStrategy): self.pruned_list = [] self.sensitivities = sensitivities self.sensitivities_file = sensitivities_file - self.backup = {} - self.param_shape_backup = {} self.num_steps = num_steps self.eval_rate = eval_rate self.pruning_step = 1 - pow((1 - target_ratio), 1.0 / self.num_steps) @@ -728,8 +759,6 @@ class SensitivePruneStrategy(PruneStrategy): Computing the sensitivities of all parameters. """ _logger.info("calling _compute_sensitivities.") - self.param_shape_backup = {} - self.backup = {} cached_id = np.random.randint(1000) if self.start_epoch == context.epoch_id: sensitivities_file = self.sensitivities_file @@ -761,12 +790,15 @@ class SensitivePruneStrategy(PruneStrategy): if metric is None: metric = self._eval_graph(context, self.eval_rate, cached_id) + + param_backup = {} # prune parameter by ratio self._prune_parameters( context.eval_graph, context.scope, [param], [ratio], context.place, - lazy=True) + lazy=True, + param_backup=param_backup) self.pruned_list[0] # get accuracy after pruning and update self.sensitivities pruned_metric = self._eval_graph(context, self.eval_rate, @@ -787,12 +819,11 @@ class SensitivePruneStrategy(PruneStrategy): self._save_sensitivities(sensitivities, sensitivities_file) # restore pruned parameters - for param_name in self.backup.keys(): + for param_name in param_backup.keys(): param_t = context.scope.find_var(param_name).get_tensor() - param_t.set(self.backup[param_name], context.place) + param_t.set(self.param_backup[param_name], context.place) # pruned_metric = self._eval_graph(context) - self.backup = {} ratio += self.delta_rate return sensitivities @@ -803,8 +834,6 @@ class SensitivePruneStrategy(PruneStrategy): """ _logger.info('_get_best_ratios for pruning ratie: {}'.format( target_ratio)) - self.param_shape_backup = {} - self.backup = {} def func(params, x): a, b, c, d = params @@ -854,23 +883,24 @@ class SensitivePruneStrategy(PruneStrategy): _logger.info('Pruned ratios={}'.format( [round(ratio, 3) for ratio in ratios])) # step 2.2: Pruning by current ratios + param_shape_backup = {} self._prune_parameters( context.eval_graph, context.scope, sensitivities.keys(), ratios, context.place, - only_graph=True) + only_graph=True, + param_shape_backup=param_shape_backup) pruned_flops = 1 - (float(context.eval_graph.flops()) / flops) pruned_size = 1 - (float(context.eval_graph.numel_params()) / model_size) _logger.info('Pruned flops: {:.4f}'.format(pruned_flops)) _logger.info('Pruned model size: {:.4f}'.format(pruned_size)) - for param in self.param_shape_backup.keys(): - context.eval_graph.var(param).set_shape(self.param_shape_backup[ + for param in param_shape_backup.keys(): + context.eval_graph.var(param).set_shape(param_shape_backup[ param]) - self.param_shape_backup = {} # step 2.3: Check whether current ratios is enough if abs(pruned_flops - target_ratio) < 0.015: @@ -902,9 +932,6 @@ class SensitivePruneStrategy(PruneStrategy): self._prune_parameters(context.optimize_graph, context.scope, params, ratios, context.place) - self.param_shape_backup = {} - self.backup = {} - model_size = context.eval_graph.numel_params() flops = context.eval_graph.flops() _logger.debug('################################') diff --git a/python/paddle/fluid/contrib/slim/searcher/__init__.py b/python/paddle/fluid/contrib/slim/searcher/__init__.py new file mode 100644 index 0000000000..734811e318 --- /dev/null +++ b/python/paddle/fluid/contrib/slim/searcher/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import controller +from .controller import * + +__all__ = controller.__all__ diff --git a/python/paddle/fluid/contrib/slim/searcher/controller.py b/python/paddle/fluid/contrib/slim/searcher/controller.py new file mode 100644 index 0000000000..67830d34e1 --- /dev/null +++ b/python/paddle/fluid/contrib/slim/searcher/controller.py @@ -0,0 +1,147 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The controller used to search hyperparameters or neural architecture""" + +import numpy as np +import copy +import math +import logging + +__all__ = ['EvolutionaryController', 'SAController'] + +logging.basicConfig(format='%(asctime)s-%(levelname)s: %(message)s') +_logger = logging.getLogger(__name__) +_logger.setLevel(logging.INFO) + + +class EvolutionaryController(object): + """Abstract controller for all evolutionary searching method. + """ + + def __init__(self, *args, **kwargs): + pass + + def update(self, tokens, reward): + """Update the status of controller according current tokens and reward. + Args: + tokens(list): A solution of searching task. + reward(list): The reward of tokens. + """ + raise NotImplementedError('Abstract method.') + + def reset(self, range_table, constrain_func=None): + """Reset the controller. + Args: + range_table(list): It is used to define the searching space of controller. + The tokens[i] generated by controller should be in [0, range_table[i]). + constrain_func(function): It is used to check whether tokens meet the constraint. + None means there is no constraint. Default: None. + """ + raise NotImplementedError('Abstract method.') + + def next_tokens(self): + """Generate new tokens. + """ + raise NotImplementedError('Abstract method.') + + +class SAController(EvolutionaryController): + """Simulated annealing controller.""" + + def __init__(self, + range_table=None, + reduce_rate=0.85, + init_temperature=1024, + max_iter_number=300): + """Initialize. + Args: + range_table(list): Range table. + reduce_rate(float): The decay rate of temperature. + init_temperature(float): Init temperature. + max_iter_number(int): max iteration number. + """ + super(SAController, self).__init__() + self._range_table = range_table + self._reduce_rate = reduce_rate + self._init_temperature = init_temperature + self._max_iter_number = max_iter_number + self._reward = -1 + self._tokens = None + self._max_reward = -1 + self._best_tokens = None + self._iter = 0 + + def __getstate__(self): + d = {} + for key in self.__dict__: + if key != "_constrain_func": + d[key] = self.__dict__[key] + return d + + def reset(self, range_table, init_tokens, constrain_func=None): + """ + Reset the status of current controller. + Args: + range_table(list): The range of value in each position of tokens generated by current controller. The range of tokens[i] is [0, range_table[i]). + init_tokens(list): The initial tokens. + constrain_func(function): The callback function used to check whether the tokens meet constraint. None means there is no constraint. Default: None. + """ + self._range_table = range_table + self._constrain_func = constrain_func + self._tokens = init_tokens + self._iter = 0 + + def update(self, tokens, reward): + """ + Update the controller according to latest tokens and reward. + Args: + tokens(list): The tokens generated in last step. + reward(float): The reward of tokens. + """ + self._iter += 1 + temperature = self._init_temperature * self._reduce_rate**self._iter + if (reward > self._reward) or (np.random.random() <= math.exp( + (reward - self._reward) / temperature)): + self._reward = reward + self._tokens = tokens + if reward > self._max_reward: + self._max_reward = reward + self._best_tokens = tokens + _logger.info("iter: {}; max_reward: {}; best_tokens: {}".format( + self._iter, self._max_reward, self._best_tokens)) + _logger.info("current_reward: {}; current tokens: {}".format( + self._reward, self._tokens)) + + def next_tokens(self): + """ + Get next tokens. + """ + tokens = self._tokens + new_tokens = tokens[:] + index = int(len(self._range_table) * np.random.random()) + new_tokens[index] = ( + new_tokens[index] + np.random.randint(self._range_table[index] - 1) + + 1) % self._range_table[index] + _logger.info("change index[{}] from {} to {}".format(index, tokens[ + index], new_tokens[index])) + if self._constrain_func is None: + return new_tokens + for _ in range(self._max_iter_number): + if not self._constrain_func(new_tokens): + index = int(len(self._range_table) * np.random.random()) + new_tokens = tokens[:] + new_tokens[index] = np.random.randint(self._range_table[index]) + else: + break + return new_tokens diff --git a/python/paddle/fluid/contrib/slim/tests/CMakeLists.txt b/python/paddle/fluid/contrib/slim/tests/CMakeLists.txt index 53a89a3fac..6c8a487dbc 100644 --- a/python/paddle/fluid/contrib/slim/tests/CMakeLists.txt +++ b/python/paddle/fluid/contrib/slim/tests/CMakeLists.txt @@ -16,6 +16,10 @@ endfunction() # Need to figure out the root cause and then add it back list(REMOVE_ITEM TEST_OPS test_distillation_strategy) +if(WIN32) + list(REMOVE_ITEM TEST_OPS test_light_nas) +endif() + # int8 image classification python api test if(LINUX AND WITH_MKLDNN) set(INT8_DATA_DIR "${INFERENCE_DEMO_INSTALL_DIR}/int8v2") diff --git a/python/paddle/fluid/contrib/slim/tests/auto_pruning/compress.yaml b/python/paddle/fluid/contrib/slim/tests/auto_pruning/compress.yaml new file mode 100644 index 0000000000..8f0ab5fbdd --- /dev/null +++ b/python/paddle/fluid/contrib/slim/tests/auto_pruning/compress.yaml @@ -0,0 +1,30 @@ +version: 1.0 +pruners: + pruner_1: + class: 'StructurePruner' + pruning_axis: + '*': 0 + criterions: + '*': 'l1_norm' +controllers: + sa_controller: + class: 'SAController' + reduce_rate: 0.9 + init_temperature: 1024 + max_iter_number: 300 +strategies: + auto_pruning_strategy: + class: 'AutoPruneStrategy' + pruner: 'pruner_1' + controller: 'sa_controller' + start_epoch: 0 + end_epoch: 2 + max_ratio: 0.7 + min_ratio: 0.5 + pruned_params: '.*_sep_weights' + metric_name: 'acc_top5' +compressor: + epoch: 2 + checkpoint_path: './checkpoints_auto_pruning/' + strategies: + - auto_pruning_strategy diff --git a/python/paddle/fluid/contrib/slim/tests/light_nas/compress.yaml b/python/paddle/fluid/contrib/slim/tests/light_nas/compress.yaml new file mode 100644 index 0000000000..9a37ea987d --- /dev/null +++ b/python/paddle/fluid/contrib/slim/tests/light_nas/compress.yaml @@ -0,0 +1,22 @@ +version: 1.0 +controllers: + sa_controller: + class: 'SAController' + reduce_rate: 0.9 + init_temperature: 1024 + max_iter_number: 300 +strategies: + light_nas_strategy: + class: 'LightNASStrategy' + controller: 'sa_controller' + target_flops: 629145600 + end_epoch: 2 + retrain_epoch: 1 + metric_name: 'acc_top1' + is_server: 1 + max_client_num: 100 + search_steps: 2 +compressor: + epoch: 2 + strategies: + - light_nas_strategy diff --git a/python/paddle/fluid/contrib/slim/tests/light_nas/light_nas_space.py b/python/paddle/fluid/contrib/slim/tests/light_nas/light_nas_space.py new file mode 100644 index 0000000000..e6ad173c8f --- /dev/null +++ b/python/paddle/fluid/contrib/slim/tests/light_nas/light_nas_space.py @@ -0,0 +1,277 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.fluid.contrib.slim.nas import SearchSpace +from light_nasnet import LightNASNet +import paddle.fluid as fluid +import paddle +import json + +total_images = 1281167 +lr = 0.1 +num_epochs = 1 +batch_size = 256 +lr_strategy = "cosine_decay" +l2_decay = 4e-5 +momentum_rate = 0.9 +image_shape = [1, 28, 28] + +__all__ = ['LightNASSpace'] + +NAS_FILTER_SIZE = [[18, 24, 30], [24, 32, 40], [48, 64, 80], [72, 96, 120], + [120, 160, 192]] +NAS_LAYERS_NUMBER = [[1, 2, 3], [2, 3, 4], [3, 4, 5], [2, 3, 4], [2, 3, 4]] +NAS_KERNEL_SIZE = [3, 5] +NAS_FILTERS_MULTIPLIER = [3, 4, 5, 6] +NAS_SHORTCUT = [0, 1] +NAS_SE = [0, 1] + + +def get_bottleneck_params_list(var): + """Get bottleneck_params_list from var. + Args: + var: list, variable list. + Returns: + list, bottleneck_params_list. + """ + params_list = [ + 1, 16, 1, 1, 3, 1, 0, \ + 6, 24, 2, 2, 3, 1, 0, \ + 6, 32, 3, 2, 3, 1, 0, \ + 6, 64, 4, 2, 3, 1, 0, \ + 6, 96, 3, 1, 3, 1, 0, \ + 6, 160, 3, 2, 3, 1, 0, \ + 6, 320, 1, 1, 3, 1, 0, \ + ] + for i in range(5): + params_list[i * 7 + 7] = NAS_FILTERS_MULTIPLIER[var[i * 6]] + params_list[i * 7 + 8] = NAS_FILTER_SIZE[i][var[i * 6 + 1]] + params_list[i * 7 + 9] = NAS_LAYERS_NUMBER[i][var[i * 6 + 2]] + params_list[i * 7 + 11] = NAS_KERNEL_SIZE[var[i * 6 + 3]] + params_list[i * 7 + 12] = NAS_SHORTCUT[var[i * 6 + 4]] + params_list[i * 7 + 13] = NAS_SE[var[i * 6 + 5]] + return params_list + + +class LightNASSpace(SearchSpace): + def __init__(self): + super(LightNASSpace, self).__init__() + + def init_tokens(self): + """Get init tokens in search space. + """ + return [ + 0, 1, 2, 0, 1, 0, 0, 2, 1, 1, 1, 0, 3, 2, 0, 1, 1, 0, 3, 1, 0, 0, 1, + 0, 3, 2, 2, 1, 1, 0 + ] + + def range_table(self): + """Get range table of current search space. + """ + # [NAS_FILTER_SIZE, NAS_LAYERS_NUMBER, NAS_KERNEL_SIZE, NAS_FILTERS_MULTIPLIER, NAS_SHORTCUT, NAS_SE] + return [ + 4, 3, 3, 2, 2, 2, 4, 3, 3, 2, 2, 2, 4, 3, 3, 2, 2, 2, 4, 3, 3, 2, 2, + 2, 4, 3, 3, 2, 2, 2 + ] + + def create_net(self, tokens=None): + """Create a network for training by tokens. + """ + if tokens is None: + tokens = self.init_tokens() + + bottleneck_params_list = get_bottleneck_params_list(tokens) + + startup_prog = fluid.Program() + train_prog = fluid.Program() + test_prog = fluid.Program() + train_py_reader, train_cost, train_acc1, train_acc5, global_lr = build_program( + is_train=True, + main_prog=train_prog, + startup_prog=startup_prog, + bottleneck_params_list=bottleneck_params_list) + test_py_reader, test_cost, test_acc1, test_acc5 = build_program( + is_train=False, + main_prog=test_prog, + startup_prog=startup_prog, + bottleneck_params_list=bottleneck_params_list) + test_prog = test_prog.clone(for_test=True) + train_batch_size = batch_size / 1 + test_batch_size = batch_size + train_reader = paddle.batch( + paddle.dataset.mnist.train(), + batch_size=train_batch_size, + drop_last=True) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=test_batch_size) + + with fluid.program_guard(train_prog, startup_prog): + train_py_reader.decorate_paddle_reader(train_reader) + + with fluid.program_guard(test_prog, startup_prog): + test_py_reader.decorate_paddle_reader(test_reader) + return startup_prog, train_prog, test_prog, ( + train_cost, train_acc1, train_acc5, + global_lr), (test_cost, test_acc1, + test_acc5), train_py_reader, test_py_reader + + +def build_program(is_train, + main_prog, + startup_prog, + bottleneck_params_list=None): + with fluid.program_guard(main_prog, startup_prog): + py_reader = fluid.layers.py_reader( + capacity=16, + shapes=[[-1] + image_shape, [-1, 1]], + lod_levels=[0, 0], + dtypes=["float32", "int64"], + use_double_buffer=False) + with fluid.unique_name.guard(): + image, label = fluid.layers.read_file(py_reader) + model = LightNASNet() + avg_cost, acc_top1, acc_top5 = net_config( + image, + label, + model, + class_dim=10, + bottleneck_params_list=bottleneck_params_list, + scale_loss=1.0) + + avg_cost.persistable = True + acc_top1.persistable = True + acc_top5.persistable = True + if is_train: + params = model.params + params["total_images"] = total_images + params["lr"] = lr + params["num_epochs"] = num_epochs + params["learning_strategy"]["batch_size"] = batch_size + params["learning_strategy"]["name"] = lr_strategy + params["l2_decay"] = l2_decay + params["momentum_rate"] = momentum_rate + optimizer = optimizer_setting(params) + optimizer.minimize(avg_cost) + global_lr = optimizer._global_learning_rate() + + if is_train: + return py_reader, avg_cost, acc_top1, acc_top5, global_lr + else: + return py_reader, avg_cost, acc_top1, acc_top5 + + +def net_config(image, + label, + model, + class_dim=1000, + bottleneck_params_list=None, + scale_loss=1.0): + bottleneck_params_list = [ + bottleneck_params_list[i:i + 7] + for i in range(0, len(bottleneck_params_list), 7) + ] + out = model.net(input=image, + bottleneck_params_list=bottleneck_params_list, + class_dim=class_dim) + cost, pred = fluid.layers.softmax_with_cross_entropy( + out, label, return_softmax=True) + if scale_loss > 1: + avg_cost = fluid.layers.mean(x=cost) * float(scale_loss) + else: + avg_cost = fluid.layers.mean(x=cost) + acc_top1 = fluid.layers.accuracy(input=pred, label=label, k=1) + acc_top5 = fluid.layers.accuracy(input=pred, label=label, k=5) + return avg_cost, acc_top1, acc_top5 + + +def optimizer_setting(params): + """optimizer setting. + Args: + params: dict, params. + """ + ls = params["learning_strategy"] + l2_decay = params["l2_decay"] + momentum_rate = params["momentum_rate"] + if ls["name"] == "piecewise_decay": + if "total_images" not in params: + total_images = IMAGENET1000 + else: + total_images = params["total_images"] + batch_size = ls["batch_size"] + step = int(total_images / batch_size + 1) + bd = [step * e for e in ls["epochs"]] + base_lr = params["lr"] + lr = [] + lr = [base_lr * (0.1**i) for i in range(len(bd) + 1)] + optimizer = fluid.optimizer.Momentum( + learning_rate=fluid.layers.piecewise_decay( + boundaries=bd, values=lr), + momentum=momentum_rate, + regularization=fluid.regularizer.L2Decay(l2_decay)) + elif ls["name"] == "cosine_decay": + if "total_images" not in params: + total_images = IMAGENET1000 + else: + total_images = params["total_images"] + batch_size = ls["batch_size"] + step = int(total_images / batch_size + 1) + lr = params["lr"] + num_epochs = params["num_epochs"] + optimizer = fluid.optimizer.Momentum( + learning_rate=fluid.layers.cosine_decay( + learning_rate=lr, step_each_epoch=step, epochs=num_epochs), + momentum=momentum_rate, + regularization=fluid.regularizer.L2Decay(l2_decay)) + elif ls["name"] == "cosine_warmup_decay": + if "total_images" not in params: + total_images = IMAGENET1000 + else: + total_images = params["total_images"] + batch_size = ls["batch_size"] + l2_decay = params["l2_decay"] + momentum_rate = params["momentum_rate"] + step = int(math.ceil(float(total_images) / batch_size)) + lr = params["lr"] + num_epochs = params["num_epochs"] + optimizer = fluid.optimizer.Momentum( + learning_rate=cosine_decay_with_warmup( + learning_rate=lr, step_each_epoch=step, epochs=num_epochs), + momentum=momentum_rate, + regularization=fluid.regularizer.L2Decay(l2_decay)) + elif ls["name"] == "linear_decay": + if "total_images" not in params: + total_images = IMAGENET1000 + else: + total_images = params["total_images"] + batch_size = ls["batch_size"] + num_epochs = params["num_epochs"] + start_lr = params["lr"] + end_lr = 0 + total_step = int((total_images / batch_size) * num_epochs) + lr = fluid.layers.polynomial_decay( + start_lr, total_step, end_lr, power=1) + optimizer = fluid.optimizer.Momentum( + learning_rate=lr, + momentum=momentum_rate, + regularization=fluid.regularizer.L2Decay(l2_decay)) + elif ls["name"] == "adam": + lr = params["lr"] + optimizer = fluid.optimizer.Adam(learning_rate=lr) + else: + lr = params["lr"] + optimizer = fluid.optimizer.Momentum( + learning_rate=lr, + momentum=momentum_rate, + regularization=fluid.regularizer.L2Decay(l2_decay)) + return optimizer diff --git a/python/paddle/fluid/contrib/slim/tests/light_nas/light_nasnet.py b/python/paddle/fluid/contrib/slim/tests/light_nas/light_nasnet.py new file mode 100644 index 0000000000..0ac3ac55b5 --- /dev/null +++ b/python/paddle/fluid/contrib/slim/tests/light_nas/light_nasnet.py @@ -0,0 +1,339 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""LightNASNet.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import math +import paddle.fluid as fluid +from paddle.fluid.param_attr import ParamAttr + +__all__ = ['LightNASNet'] + +train_parameters = { + "input_size": [3, 224, 224], + "input_mean": [0.485, 0.456, 0.406], + "input_std": [0.229, 0.224, 0.225], + "learning_strategy": { + "name": "piecewise_decay", + "batch_size": 256, + "epochs": [30, 60, 90], + "steps": [0.1, 0.01, 0.001, 0.0001] + } +} + + +class LightNASNet(object): + """LightNASNet.""" + + def __init__(self): + self.params = train_parameters + + def net(self, input, bottleneck_params_list=None, class_dim=1000, + scale=1.0): + """Build network. + Args: + input: Variable, input. + class_dim: int, class dim. + scale: float, scale. + Returns: + Variable, network output. + """ + if bottleneck_params_list is None: + # MobileNetV2 + # bottleneck_params_list = [ + # (1, 16, 1, 1, 3, 1, 0), + # (6, 24, 2, 2, 3, 1, 0), + # (6, 32, 3, 2, 3, 1, 0), + # (6, 64, 4, 2, 3, 1, 0), + # (6, 96, 3, 1, 3, 1, 0), + # (6, 160, 3, 2, 3, 1, 0), + # (6, 320, 1, 1, 3, 1, 0), + # ] + bottleneck_params_list = [ + (1, 16, 1, 1, 3, 1, 0), + (3, 24, 3, 2, 3, 1, 0), + (3, 40, 3, 2, 5, 1, 0), + (6, 80, 3, 2, 5, 1, 0), + (6, 96, 2, 1, 3, 1, 0), + (6, 192, 4, 2, 5, 1, 0), + (6, 320, 1, 1, 3, 1, 0), + ] + + #conv1 + input = self.conv_bn_layer( + input, + num_filters=int(32 * scale), + filter_size=3, + stride=2, + padding=1, + if_act=True, + name='conv1_1') + + # bottleneck sequences + i = 1 + in_c = int(32 * scale) + for layer_setting in bottleneck_params_list: + t, c, n, s, k, ifshortcut, ifse = layer_setting + i += 1 + input = self.invresi_blocks( + input=input, + in_channel=in_c, + expansion=t, + out_channel=int(c * scale), + num_layers=n, + stride=s, + filter_size=k, + shortcut=ifshortcut, + squeeze=ifse, + name='conv' + str(i)) + in_c = int(c * scale) + #last_conv + input = self.conv_bn_layer( + input=input, + num_filters=int(1280 * scale) if scale > 1.0 else 1280, + filter_size=1, + stride=1, + padding=0, + if_act=True, + name='conv9') + + input = fluid.layers.pool2d( + input=input, + pool_size=7, + pool_stride=1, + pool_type='avg', + global_pooling=True) + + output = fluid.layers.fc(input=input, + size=class_dim, + param_attr=ParamAttr(name='fc10_weights'), + bias_attr=ParamAttr(name='fc10_offset')) + return output + + def conv_bn_layer(self, + input, + filter_size, + num_filters, + stride, + padding, + num_groups=1, + if_act=True, + name=None, + use_cudnn=True): + """Build convolution and batch normalization layers. + Args: + input: Variable, input. + filter_size: int, filter size. + num_filters: int, number of filters. + stride: int, stride. + padding: int, padding. + num_groups: int, number of groups. + if_act: bool, whether using activation. + name: str, name. + use_cudnn: bool, whether use cudnn. + Returns: + Variable, layers output. + """ + conv = fluid.layers.conv2d( + input=input, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=padding, + groups=num_groups, + act=None, + use_cudnn=use_cudnn, + param_attr=ParamAttr(name=name + '_weights'), + bias_attr=False) + bn_name = name + '_bn' + bn = fluid.layers.batch_norm( + input=conv, + param_attr=ParamAttr(name=bn_name + "_scale"), + bias_attr=ParamAttr(name=bn_name + "_offset"), + moving_mean_name=bn_name + '_mean', + moving_variance_name=bn_name + '_variance') + if if_act: + return fluid.layers.relu6(bn) + else: + return bn + + def shortcut(self, input, data_residual): + """Build shortcut layer. + Args: + input: Variable, input. + data_residual: Variable, residual layer. + Returns: + Variable, layer output. + """ + return fluid.layers.elementwise_add(input, data_residual) + + def squeeze_excitation(self, + input, + num_channels, + reduction_ratio, + name=None): + """Build squeeze excitation layers. + Args: + input: Variable, input. + num_channels: int, number of channels. + reduction_ratio: float, reduction ratio. + name: str, name. + Returns: + Variable, layers output. + """ + pool = fluid.layers.pool2d( + input=input, pool_size=0, pool_type='avg', global_pooling=True) + stdv = 1.0 / math.sqrt(pool.shape[1] * 1.0) + squeeze = fluid.layers.fc( + input=pool, + size=num_channels // reduction_ratio, + act='relu', + param_attr=fluid.param_attr.ParamAttr( + initializer=fluid.initializer.Uniform(-stdv, stdv), + name=name + '_sqz_weights'), + bias_attr=ParamAttr(name=name + '_sqz_offset')) + stdv = 1.0 / math.sqrt(squeeze.shape[1] * 1.0) + excitation = fluid.layers.fc( + input=squeeze, + size=num_channels, + act='sigmoid', + param_attr=fluid.param_attr.ParamAttr( + initializer=fluid.initializer.Uniform(-stdv, stdv), + name=name + '_exc_weights'), + bias_attr=ParamAttr(name=name + '_exc_offset')) + scale = fluid.layers.elementwise_mul(x=input, y=excitation, axis=0) + return scale + + def inverted_residual_unit(self, + input, + num_in_filter, + num_filters, + ifshortcut, + ifse, + stride, + filter_size, + expansion_factor, + reduction_ratio=4, + name=None): + """Build inverted residual unit. + Args: + input(Variable): Theinput. + num_in_filter(int): The number of input filters. + num_filters(int): The number of filters. + ifshortcut(bool): Whether to use shortcut. + stride(int): The stride. + filter_size(int): The filter size. + padding(int): The padding. + expansion_factor(float): Expansion factor. + name(str): The name. + Returns: + Variable, layers output. + """ + num_expfilter = int(round(num_in_filter * expansion_factor)) + channel_expand = self.conv_bn_layer( + input=input, + num_filters=num_expfilter, + filter_size=1, + stride=1, + padding=0, + num_groups=1, + if_act=True, + name=name + '_expand') + + bottleneck_conv = self.conv_bn_layer( + input=channel_expand, + num_filters=num_expfilter, + filter_size=filter_size, + stride=stride, + padding=int((filter_size - 1) / 2), + num_groups=num_expfilter, + if_act=True, + name=name + '_dwise', + use_cudnn=False) + + linear_out = self.conv_bn_layer( + input=bottleneck_conv, + num_filters=num_filters, + filter_size=1, + stride=1, + padding=0, + num_groups=1, + if_act=False, + name=name + '_linear') + out = linear_out + if ifshortcut: + out = self.shortcut(input=input, data_residual=out) + if ifse: + scale = self.squeeze_excitation( + input=linear_out, + num_channels=num_filters, + reduction_ratio=reduction_ratio, + name=name + '_fc') + out = fluid.layers.elementwise_add(x=out, y=scale, act='relu') + return out + + def invresi_blocks(self, + input, + in_channel, + expansion, + out_channel, + num_layers, + stride, + filter_size, + shortcut, + squeeze, + name=None): + """Build inverted residual blocks. + Args: + input(Variable): The input feture map. + in_channel(int): The number of input channel. + expansion(float): Expansion factor. + out_channel(int): The number of output channel. + num_layers(int): The number of layers. + stride(int): The stride. + filter_size(int): The size of filter. + shortcut(bool): Whether to add shortcut layers. + squeeze(bool): Whether to add squeeze excitation layers. + name(str): The name. + Returns: + Variable, layers output. + """ + first_block = self.inverted_residual_unit( + input=input, + num_in_filter=in_channel, + num_filters=out_channel, + ifshortcut=False, + ifse=squeeze, + stride=stride, + filter_size=filter_size, + expansion_factor=expansion, + name=name + '_1') + + last_residual_block = first_block + last_c = out_channel + + for i in range(1, num_layers): + last_residual_block = self.inverted_residual_unit( + input=last_residual_block, + num_in_filter=last_c, + num_filters=out_channel, + ifshortcut=shortcut, + ifse=squeeze, + stride=1, + filter_size=filter_size, + expansion_factor=expansion, + name=name + '_' + str(i + 1)) + return last_residual_block diff --git a/python/paddle/fluid/contrib/slim/tests/test_auto_pruning.py b/python/paddle/fluid/contrib/slim/tests/test_auto_pruning.py new file mode 100644 index 0000000000..006e5adb25 --- /dev/null +++ b/python/paddle/fluid/contrib/slim/tests/test_auto_pruning.py @@ -0,0 +1,86 @@ +# copyright (c) 2019 paddlepaddle authors. all rights reserved. +# +# licensed under the apache license, version 2.0 (the "license"); +# you may not use this file except in compliance with the license. +# you may obtain a copy of the license at +# +# http://www.apache.org/licenses/license-2.0 +# +# unless required by applicable law or agreed to in writing, software +# distributed under the license is distributed on an "as is" basis, +# without warranties or conditions of any kind, either express or implied. +# see the license for the specific language governing permissions and +# limitations under the license. + +import paddle +import unittest +import paddle.fluid as fluid +from mobilenet import MobileNet +from paddle.fluid.contrib.slim.core import Compressor +from paddle.fluid.contrib.slim.graph import GraphWrapper + + +class TestFilterPruning(unittest.TestCase): + def test_compression(self): + """ + Model: mobilenet_v1 + data: mnist + step1: Training one epoch + step2: pruning flops + step3: fine-tune one epoch + step4: check top1_acc. + """ + if not fluid.core.is_compiled_with_cuda(): + return + class_dim = 10 + image_shape = [1, 28, 28] + image = fluid.layers.data( + name='image', shape=image_shape, dtype='float32') + image.stop_gradient = False + label = fluid.layers.data(name='label', shape=[1], dtype='int64') + out = MobileNet("auto_pruning").net(input=image, class_dim=class_dim) + acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1) + acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5) + val_program = fluid.default_main_program().clone(for_test=False) + + cost = fluid.layers.cross_entropy(input=out, label=label) + avg_cost = fluid.layers.mean(x=cost) + + optimizer = fluid.optimizer.Momentum( + momentum=0.9, + learning_rate=0.01, + regularization=fluid.regularizer.L2Decay(4e-5)) + + place = fluid.CUDAPlace(0) + exe = fluid.Executor(place) + exe.run(fluid.default_startup_program()) + + val_reader = paddle.batch(paddle.dataset.mnist.test(), batch_size=128) + + val_feed_list = [('img', image.name), ('label', label.name)] + val_fetch_list = [('acc_top1', acc_top1.name), ('acc_top5', + acc_top5.name)] + + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=128) + train_feed_list = [('img', image.name), ('label', label.name)] + train_fetch_list = [('loss', avg_cost.name)] + + com_pass = Compressor( + place, + fluid.global_scope(), + fluid.default_main_program(), + train_reader=train_reader, + train_feed_list=train_feed_list, + train_fetch_list=train_fetch_list, + eval_program=val_program, + eval_reader=val_reader, + eval_feed_list=val_feed_list, + eval_fetch_list=val_fetch_list, + train_optimizer=optimizer) + com_pass.config('./auto_pruning/compress.yaml') + eval_graph = com_pass.run() + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/fluid/contrib/slim/tests/test_light_nas.py b/python/paddle/fluid/contrib/slim/tests/test_light_nas.py new file mode 100644 index 0000000000..e3f8d9976a --- /dev/null +++ b/python/paddle/fluid/contrib/slim/tests/test_light_nas.py @@ -0,0 +1,66 @@ +# copyright (c) 2019 paddlepaddle authors. all rights reserved. +# +# licensed under the apache license, version 2.0 (the "license"); +# you may not use this file except in compliance with the license. +# you may obtain a copy of the license at +# +# http://www.apache.org/licenses/license-2.0 +# +# unless required by applicable law or agreed to in writing, software +# distributed under the license is distributed on an "as is" basis, +# without warranties or conditions of any kind, either express or implied. +# see the license for the specific language governing permissions and +# limitations under the license. + +import paddle +import unittest +import paddle.fluid as fluid +from mobilenet import MobileNet +from paddle.fluid.contrib.slim.core import Compressor +from paddle.fluid.contrib.slim.graph import GraphWrapper +import sys +sys.path.append("./light_nas") +from light_nas_space import LightNASSpace + + +class TestLightNAS(unittest.TestCase): + def test_compression(self): + if not fluid.core.is_compiled_with_cuda(): + return + class_dim = 10 + image_shape = [1, 28, 28] + + space = LightNASSpace() + + startup_prog, train_prog, test_prog, train_metrics, test_metrics, train_reader, test_reader = space.create_net( + ) + train_cost, train_acc1, train_acc5, global_lr = train_metrics + test_cost, test_acc1, test_acc5 = test_metrics + + place = fluid.CUDAPlace(0) + exe = fluid.Executor(place) + exe.run(startup_prog) + + val_fetch_list = [('acc_top1', test_acc1.name), ('acc_top5', + test_acc5.name)] + train_fetch_list = [('loss', train_cost.name)] + + com_pass = Compressor( + place, + fluid.global_scope(), + train_prog, + train_reader=train_reader, + train_feed_list=None, + train_fetch_list=train_fetch_list, + eval_program=test_prog, + eval_reader=test_reader, + eval_feed_list=None, + eval_fetch_list=val_fetch_list, + train_optimizer=None, + search_space=space) + com_pass.config('./light_nas/compress.yaml') + eval_graph = com_pass.run() + + +if __name__ == '__main__': + unittest.main() diff --git a/python/setup.py.in b/python/setup.py.in index 764b3ea6fe..17071c1f41 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -118,6 +118,8 @@ packages=['paddle', 'paddle.fluid.contrib.slim.prune', 'paddle.fluid.contrib.slim.quantization', 'paddle.fluid.contrib.slim.distillation', + 'paddle.fluid.contrib.slim.nas', + 'paddle.fluid.contrib.slim.searcher', 'paddle.fluid.contrib.utils', 'paddle.fluid.contrib.extend_optimizer', 'paddle.fluid.contrib.mixed_precision', -- GitLab