From eb48cb63e02b992693c5a75be1b8432c0deed10e Mon Sep 17 00:00:00 2001 From: Bai Yifan Date: Tue, 21 Jan 2020 10:20:09 +0800 Subject: [PATCH] fix distillation api examples (#50) --- docs/docs/api/single_distiller_api.md | 16 +- paddleslim/dist/__init__.py | 2 + paddleslim/dist/mp_distiller.py | 223 -------------------------- 3 files changed, 10 insertions(+), 231 deletions(-) delete mode 100755 paddleslim/dist/mp_distiller.py diff --git a/docs/docs/api/single_distiller_api.md b/docs/docs/api/single_distiller_api.md index e6d382d6..ed8e6115 100644 --- a/docs/docs/api/single_distiller_api.md +++ b/docs/docs/api/single_distiller_api.md @@ -76,10 +76,10 @@ with fluid.program_guard(teacher_program): data_name_map = {'y':'x'} USE_GPU = False place = fluid.CUDAPlace(0) if USE_GPU else fluid.CPUPlace() -merge(teacher_program, student_program, data_name_map, place) +dist.merge(teacher_program, student_program, data_name_map, place) with fluid.program_guard(student_program): distillation_loss = dist.fsp_loss('teacher_t1.tmp_1', 'teacher_t2.tmp_1', - 's1.tmp_1', 's2.tmp_1', main_program) + 's1.tmp_1', 's2.tmp_1', student_program) ``` @@ -116,10 +116,10 @@ with fluid.program_guard(teacher_program): data_name_map = {'y':'x'} USE_GPU = False place = fluid.CUDAPlace(0) if USE_GPU else fluid.CPUPlace() -merge(teacher_program, student_program, data_name_map, place) +dist.merge(teacher_program, student_program, data_name_map, place) with fluid.program_guard(student_program): distillation_loss = dist.l2_loss('teacher_t2.tmp_1', 's2.tmp_1', - main_program) + student_program) ``` @@ -158,10 +158,10 @@ with fluid.program_guard(teacher_program): data_name_map = {'y':'x'} USE_GPU = False place = fluid.CUDAPlace(0) if USE_GPU else fluid.CPUPlace() -merge(teacher_program, student_program, data_name_map, place) +dist.merge(teacher_program, student_program, data_name_map, place) with fluid.program_guard(student_program): distillation_loss = dist.soft_label_loss('teacher_t2.tmp_1', - 's2.tmp_1', main_program, 1., 1.) + 's2.tmp_1', student_program, 1., 1.) ``` @@ -198,14 +198,14 @@ with fluid.program_guard(teacher_program): data_name_map = {'y':'x'} USE_GPU = False place = fluid.CUDAPlace(0) if USE_GPU else fluid.CPUPlace() -merge(teacher_program, student_program, data_name_map, place) +dist.merge(teacher_program, student_program, data_name_map, place) def adaptation_loss(t_var, s_var): teacher_channel = t_var.shape[1] s_hint = fluid.layers.conv2d(s_var, teacher_channel, 1) hint_loss = fluid.layers.reduce_mean(fluid.layers.square(s_hint - t_var)) return hint_loss with fluid.program_guard(student_program): - distillation_loss = dist.loss(main_program, adaptation_loss, + distillation_loss = dist.loss(adaptation_loss, student_program, t_var='teacher_t2.tmp_1', s_var='s2.tmp_1') ``` diff --git a/paddleslim/dist/__init__.py b/paddleslim/dist/__init__.py index 9d053150..bdc208fa 100644 --- a/paddleslim/dist/__init__.py +++ b/paddleslim/dist/__init__.py @@ -11,3 +11,5 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from .single_distiller import merge, fsp_loss, l2_loss, soft_label_loss, loss diff --git a/paddleslim/dist/mp_distiller.py b/paddleslim/dist/mp_distiller.py deleted file mode 100755 index ff15f5f1..00000000 --- a/paddleslim/dist/mp_distiller.py +++ /dev/null @@ -1,223 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License" -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import logging -import numpy as np -from six.moves.queue import Queue - -import paddle.fluid as fluid -from paddle.fluid.framework import Variable -from paddle.fluid.reader import DataLoaderBase -from paddle.fluid.core import EOFException -from paddle.fluid.incubate.fleet.utils.hdfs import HDFSClient - -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - -__all__ = ['Knowledge'] - - -class Knowledge(object): - """ - The knowledge class describes how to extract and store the dark knowledge - of the teacher model, and how the student model learns these dark knowledge. - """ - - def __init__(self, - path, - items, - reduce_strategy={'type': 'sum', - 'key': 'image'}): - """Init a knowledge instance. - Args: - path(list, str, optional): Specifies the storage path of the knowledge, - supports AFS/HDFS, local file system, and memory. - items(list): Save the tensor of the specified name - reduce_strategy(dict, optional): The policy for performing the reduce - operation. If it is set to None, - the reduce operation is not performed. - reduce_strategy.type(str): Type of reduce operation. - reduce_strategy.key(str): The key of the reduce operation. - It is an element in the item. - """ - assert (isinstance(path, list) or isinstance(path, str) or - (path is None)), "path type should be list or str or None" - assert (isinstance(items, list)), "items should be a list" - assert (isinstance(reduce_strategy, - dict)), "reduce_strategy should be a dict" - self.path = path - if isinstance(self.path, list): - self.write_type = 'HDFS/AFS' - assert ( - len(self.path) == 4 and isinstance(self.path[0], str) and - isinstance(self.path[1], str) and - isinstance(self.path[2], str) and isinstance(self.path[3], str) - ), "path should contains four str, ['local hadoop home', 'fs.default.name', 'hadoop.job.ugi', 'FS path']" - - hadoop_home = self.path[0] - configs = { - "fs.default.name": self.path[1], - "hadoop.job.ugi": self.path[2] - } - self.client = HDFSClient(hadoop_home, configs) - assert ( - self.client.is_exist(self.path[3]) == True - ), "Plese make sure your hadoop confiuration is correct and FS path exists" - - self.hdfs_local_path = "./teacher_knowledge" - if not os.path.exists(self.hdfs_local_path): - os.mkdir(self.hdfs_local_path) - elif isinstance(self.path, str): - self.write_type = "LocalFS" - if not os.path.exists(path): - raise ValueError("The local path [%s] does not exist." % - (path)) - else: - self.write_type = "MEM" - self.knowledge_queue = Queue(64) - - self.items = items - self.reduce_strategy = reduce_strategy - - def _write(self, data): - if self.write_type == 'HDFS/AFS': - file_name = 'knowledge_' + str(self.file_cnt) - file_path = os.path.join(self.hdfs_local_path, file_name) - file_path += ".npy" - np.save(file_path, data) - self.file_cnt += 1 - self.client.upload(self.path[3], file_path) - logger.info('{}.npy pushed to HDFS/AFS: {}'.format(file_name, - self.path[3])) - - elif self.write_type == 'LocalFS': - file_name = 'knowledge_' + str(self.file_cnt) - file_path = os.path.join(self.path, file_name) - np.save(file_path, data) - logger.info('{}.npy saved'.format(file_name)) - self.file_cnt += 1 - - else: - self.knowledge_queue.put(data) - logger.info('{} pushed to Queue'.format(file_name)) - - def run(self, teacher_program, exe, place, scope, reader, inputs, outputs, - call_back): - """Start teacher model to do information. - Args: - teacher_program(Program): teacher program. - scope(Scope): The scope used to execute the teacher, - which contains the initialized variables. - reader(reader): The data reader used by the teacher. - inputs(list): The name of variables to feed the teacher program. - outputs(list): Need to write to the variable instance's names of - the Knowledge instance, which needs to correspond - to the Knowledge's items. - call_back(func, optional): The callback function that handles the - outputs of the teacher, which is none by default, - that is, the output of the teacher is concat directly. - Return: - (bool): Whether the teacher task was successfully registered and started - """ - assert (isinstance( - teacher_program, - fluid.Program)), "teacher_program should be a fluid.Program" - assert (isinstance(inputs, list)), "inputs should be a list" - assert (isinstance(outputs, list)), "outputs should be a list" - assert (len(self.items) == len(outputs) - ), "the length of outputs list should be equal with items list" - assert (callable(call_back) or (call_back is None) - ), "call_back should be a callable function or NoneType." - - for var in teacher_program.list_vars(): - var.stop_gradient = True - - compiled_teacher_program = fluid.compiler.CompiledProgram( - teacher_program) - self.file_cnt = 0 - if isinstance(reader, Variable) or ( - isinstance(reader, DataLoaderBase) and (not reader.iterable)): - reader.start() - try: - while True: - logits = exe.run(compiled_teacher_program, - scope=scope, - fetch_list=outputs, - feed=None) - knowledge = dict() - for index, array in enumerate(logits): - knowledge[self.items[index]] = array - self._write(knowledge) - except EOFException: - reader.reset() - - else: - if not isinstance(reader, DataLoaderBase): - feeder = fluid.DataFeeder( - feed_list=inputs, place=place, program=teacher_program) - for batch_id, data in enumerate(reader()): - if not isinstance(reader, DataLoaderBase): - data = feeder.feed(data) - logits = exe.run(compiled_teacher_program, - scope=scope, - fetch_list=outputs, - feed=data) - knowledge = dict() - for index, array in enumerate(logits): - knowledge[self.items[index]] = array - self._write(knowledge) - return True - - def dist(self, student_program, losses): - """Building the distillation network - Args: - student_program(Program): student program. - losses(list, optional): The losses need to add. If set to None - does not add any loss. - Return: - (Program): Program for distillation. - (startup_program): Program for initializing distillation network. - (reader): Data reader for distillation training. - (Variable): Loss of distillation training - """ - - def loss(self, loss_func, *variables): - """User-defined loss - Args: - loss_func(func): Function used to define loss. - *variables(list): Variable name list. - Return: - (Variable): Distillation loss. - """ - pass - - def fsp_loss(self): - """fsp loss - """ - pass - - def l2_loss(self): - """l2 loss - """ - pass - - def softlabel_loss(self): - """softlabel_loss - """ - pass -- GitLab