diff --git a/PaddleSlim/quant_low_level_api/dygraph_quant/README.MD b/PaddleSlim/quant_low_level_api/dygraph_quant/README.MD new file mode 100644 index 0000000000000000000000000000000000000000..e312886b122b9b5ce053ab6713a9193336d99ea5 --- /dev/null +++ b/PaddleSlim/quant_low_level_api/dygraph_quant/README.MD @@ -0,0 +1,2 @@ +Train: sh run_quant.sh +Test: sh run_test.sh model_path diff --git a/PaddleSlim/quant_low_level_api/dygraph_quant/download.py b/PaddleSlim/quant_low_level_api/dygraph_quant/download.py new file mode 100644 index 0000000000000000000000000000000000000000..9d935e48995742ca8dfadce79cb2ce7395051a29 --- /dev/null +++ b/PaddleSlim/quant_low_level_api/dygraph_quant/download.py @@ -0,0 +1,340 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import sys +import os.path as osp +import shutil +import requests +import hashlib +import tarfile +import zipfile +import time +from collections import OrderedDict +from paddle.fluid.dygraph.parallel import ParallelEnv + +try: + from tqdm import tqdm +except: + + class tqdm(object): + def __init__(self, total=None): + self.total = total + self.n = 0 + + def update(self, n): + self.n += n + if self.total is None: + sys.stderr.write("\r{0:.1f} bytes".format(self.n)) + else: + sys.stderr.write("\r{0:.1f}%".format(100 * self.n / float( + self.total))) + sys.stderr.flush() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + sys.stderr.write('\n') + + +import logging +logger = logging.getLogger(__name__) + +__all__ = ['get_weights_path_from_url'] + +WEIGHTS_HOME = osp.expanduser("~/.cache/paddle/hapi/weights") + +DOWNLOAD_RETRY_LIMIT = 3 + +nlp_models = OrderedDict(( + ('RoBERTa-zh-base', + 'https://bert-models.bj.bcebos.com/chinese_roberta_wwm_ext_L-12_H-768_A-12.tar.gz' + ), + ('RoBERTa-zh-large', + 'https://bert-models.bj.bcebos.com/chinese_roberta_wwm_large_ext_L-24_H-1024_A-16.tar.gz' + ), + ('ERNIE-v2-en-base', + 'https://ernie.bj.bcebos.com/ERNIE_Base_en_stable-2.0.0.tar.gz'), + ('ERNIE-v2-en-large', + 'https://ernie.bj.bcebos.com/ERNIE_Large_en_stable-2.0.0.tar.gz'), + ('XLNet-cased-base', + 'https://xlnet.bj.bcebos.com/xlnet_cased_L-12_H-768_A-12.tgz'), + ('XLNet-cased-large', + 'https://xlnet.bj.bcebos.com/xlnet_cased_L-24_H-1024_A-16.tgz'), + ('ERNIE-v1-zh-base', + 'https://baidu-nlp.bj.bcebos.com/ERNIE_stable-1.0.1.tar.gz'), + ('ERNIE-v1-zh-base-max-len-512', + 'https://ernie.bj.bcebos.com/ERNIE_1.0_max-len-512.tar.gz'), + ('BERT-en-uncased-large-whole-word-masking', + 'https://bert-models.bj.bcebos.com/wwm_uncased_L-24_H-1024_A-16.tar.gz'), + ('BERT-en-cased-large-whole-word-masking', + 'https://bert-models.bj.bcebos.com/wwm_cased_L-24_H-1024_A-16.tar.gz'), + ('BERT-en-uncased-base', + 'https://bert-models.bj.bcebos.com/uncased_L-12_H-768_A-12.tar.gz'), + ('BERT-en-uncased-large', + 'https://bert-models.bj.bcebos.com/uncased_L-24_H-1024_A-16.tar.gz'), + ('BERT-en-cased-base', + 'https://bert-models.bj.bcebos.com/cased_L-12_H-768_A-12.tar.gz'), + ('BERT-en-cased-large', + 'https://bert-models.bj.bcebos.com/cased_L-24_H-1024_A-16.tar.gz'), + ('BERT-multilingual-uncased-base', + 'https://bert-models.bj.bcebos.com/multilingual_L-12_H-768_A-12.tar.gz'), + ('BERT-multilingual-cased-base', + 'https://bert-models.bj.bcebos.com/multi_cased_L-12_H-768_A-12.tar.gz'), + ('BERT-zh-base', + 'https://bert-models.bj.bcebos.com/chinese_L-12_H-768_A-12.tar.gz'), )) + + +def is_url(path): + """ + Whether path is URL. + Args: + path (string): URL string or not. + """ + return path.startswith('http://') or path.startswith('https://') + + +def get_weights_path_from_url(url, md5sum=None): + """Get weights path from WEIGHT_HOME, if not exists, + download it from url. + + Args: + url (str): download url + md5sum (str): md5 sum of download package + + Returns: + str: a local path to save downloaded weights. + + Examples: + .. code-block:: python + + from paddle.incubate.hapi.download import get_weights_path_from_url + + resnet18_pretrained_weight_url = 'https://paddle-hapi.bj.bcebos.com/models/resnet18.pdparams' + local_weight_path = get_weights_path_from_url(resnet18_pretrained_weight_url) + + """ + path = get_path_from_url(url, WEIGHTS_HOME, md5sum) + return path + + +def _map_path(url, root_dir): + # parse path after download under root_dir + fname = osp.split(url)[-1] + fpath = fname + return osp.join(root_dir, fpath) + + +def get_path_from_url(url, root_dir, md5sum=None, check_exist=True): + """ Download from given url to root_dir. + if file or directory specified by url is exists under + root_dir, return the path directly, otherwise download + from url and decompress it, return the path. + + Args: + url (str): download url + root_dir (str): root dir for downloading, it should be + WEIGHTS_HOME or DATASET_HOME + md5sum (str): md5 sum of download package + + Returns: + str: a local path to save downloaded models & weights & datasets. + """ + assert is_url(url), "downloading from {} not a url".format(url) + # parse path after download to decompress under root_dir + fullpath = _map_path(url, root_dir) + + if osp.exists(fullpath) and check_exist and _md5check(fullpath, md5sum): + logger.info("Found {}".format(fullpath)) + else: + if ParallelEnv().local_rank == 0: + fullpath = _download(url, root_dir, md5sum) + else: + while not os.path.exists(fullpath): + time.sleep(1) + + if ParallelEnv().local_rank == 0: + if tarfile.is_tarfile(fullpath) or zipfile.is_zipfile(fullpath): + fullpath = _decompress(fullpath) + + return fullpath + + +def _download(url, path, md5sum=None): + """ + Download from url, save to path. + + url (str): download url + path (str): download to given path + """ + if not osp.exists(path): + os.makedirs(path) + + fname = osp.split(url)[-1] + fullname = osp.join(path, fname) + retry_cnt = 0 + + while not (osp.exists(fullname) and _md5check(fullname, md5sum)): + if retry_cnt < DOWNLOAD_RETRY_LIMIT: + retry_cnt += 1 + else: + raise RuntimeError("Download from {} failed. " + "Retry limit reached".format(url)) + + logger.info("Downloading {} from {}".format(fname, url)) + + req = requests.get(url, stream=True) + if req.status_code != 200: + raise RuntimeError("Downloading from {} failed with code " + "{}!".format(url, req.status_code)) + + # For protecting download interupted, download to + # tmp_fullname firstly, move tmp_fullname to fullname + # after download finished + tmp_fullname = fullname + "_tmp" + total_size = req.headers.get('content-length') + with open(tmp_fullname, 'wb') as f: + if total_size: + with tqdm(total=(int(total_size) + 1023) // 1024) as pbar: + for chunk in req.iter_content(chunk_size=1024): + f.write(chunk) + pbar.update(1) + else: + for chunk in req.iter_content(chunk_size=1024): + if chunk: + f.write(chunk) + shutil.move(tmp_fullname, fullname) + + return fullname + + +def _md5check(fullname, md5sum=None): + if md5sum is None: + return True + + logger.info("File {} md5 checking...".format(fullname)) + md5 = hashlib.md5() + with open(fullname, 'rb') as f: + for chunk in iter(lambda: f.read(4096), b""): + md5.update(chunk) + calc_md5sum = md5.hexdigest() + + if calc_md5sum != md5sum: + logger.info("File {} md5 check failed, {}(calc) != " + "{}(base)".format(fullname, calc_md5sum, md5sum)) + return False + return True + + +def _decompress(fname): + """ + Decompress for zip and tar file + """ + logger.info("Decompressing {}...".format(fname)) + + # For protecting decompressing interupted, + # decompress to fpath_tmp directory firstly, if decompress + # successed, move decompress files to fpath and delete + # fpath_tmp and remove download compress file. + + if tarfile.is_tarfile(fname): + uncompressed_path = _uncompress_file_tar(fname) + elif zipfile.is_zipfile(fname): + uncompressed_path = _uncompress_file_zip(fname) + else: + raise TypeError("Unsupport compress file type {}".format(fname)) + + return uncompressed_path + + +def _uncompress_file_zip(filepath): + files = zipfile.ZipFile(filepath, 'r') + file_list = files.namelist() + + file_dir = os.path.dirname(filepath) + + if _is_a_single_file(file_list): + rootpath = file_list[0] + uncompressed_path = os.path.join(file_dir, rootpath) + + for item in file_list: + files.extract(item, file_dir) + + elif _is_a_single_dir(file_list): + rootpath = os.path.splitext(file_list[0])[0].split(os.sep)[-1] + uncompressed_path = os.path.join(file_dir, rootpath) + + for item in file_list: + files.extract(item, file_dir) + + else: + rootpath = os.path.splitext(filepath)[0].split(os.sep)[-1] + uncompressed_path = os.path.join(file_dir, rootpath) + if not os.path.exists(uncompressed_path): + os.makedirs(uncompressed_path) + for item in file_list: + files.extract(item, os.path.join(file_dir, rootpath)) + + files.close() + + return uncompressed_path + + +def _uncompress_file_tar(filepath, mode="r:*"): + files = tarfile.open(filepath, mode) + file_list = files.getnames() + + file_dir = os.path.dirname(filepath) + + if _is_a_single_file(file_list): + rootpath = file_list[0] + uncompressed_path = os.path.join(file_dir, rootpath) + for item in file_list: + files.extract(item, file_dir) + elif _is_a_single_dir(file_list): + rootpath = os.path.splitext(file_list[0])[0].split(os.sep)[-1] + uncompressed_path = os.path.join(file_dir, rootpath) + for item in file_list: + files.extract(item, file_dir) + else: + rootpath = os.path.splitext(filepath)[0].split(os.sep)[-1] + uncompressed_path = os.path.join(file_dir, rootpath) + if not os.path.exists(uncompressed_path): + os.makedirs(uncompressed_path) + + for item in file_list: + files.extract(item, os.path.join(file_dir, rootpath)) + + files.close() + + return uncompressed_path + + +def _is_a_single_file(file_list): + if len(file_list) == 1 and file_list[0].find(os.sep) < -1: + return True + return False + + +def _is_a_single_dir(file_list): + file_name = file_list[0].split(os.sep)[0] + for i in range(1, len(file_list)): + if file_name != file_list[i].split(os.sep)[0]: + return False + return True diff --git a/PaddleSlim/quant_low_level_api/dygraph_quant/eval.py b/PaddleSlim/quant_low_level_api/dygraph_quant/eval.py new file mode 100644 index 0000000000000000000000000000000000000000..ccb50acfd1dc22833d10260c0aecb89e6592cd2a --- /dev/null +++ b/PaddleSlim/quant_low_level_api/dygraph_quant/eval.py @@ -0,0 +1,116 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +import os +import numpy as np +import time +import sys +import paddle +import paddle.fluid as fluid +import reader as reader +import argparse +import functools +from utility import add_arguments, print_arguments +import math + +parser = argparse.ArgumentParser(description=__doc__) +add_arg = functools.partial(add_arguments, argparser=parser) +# yapf: disable +add_arg('use_gpu', bool, True, "Whether to use GPU or not.") +add_arg('class_dim', int, 1000, "Class number.") +add_arg('image_shape', str, "3,224,224", "Input image size") +add_arg('data_dir', str, "/dataset/ILSVRC2012/", "The ImageNet dataset root dir.") +add_arg('inference_model', str, "", "The inference model path.") +add_arg('test_samples', int, -1, "Test samples. if set as -1, eval all test sample") +add_arg('batch_size', int, 20, "Batch size.") +# yapf: enable + + +def eval(args): + class_dim = args.class_dim + inference_model = args.inference_model + data_dir = args.data_dir + image_shape = [int(m) for m in args.image_shape.split(",")] + + place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() + exe = fluid.Executor(place) + [inference_program, feed_target_names, fetch_targets] = \ + fluid.io.load_inference_model(dirname=inference_model, executor=exe) + feed_vars = [fluid.framework._get_var(str(var_name), inference_program) \ + for var_name in feed_target_names] + print('--------------------------input--------------------------') + for i in feed_vars: + print(i.name) + print('--------------------------output--------------------------') + for o in fetch_targets: + print(o.name) + val_reader = paddle.batch( + reader.val(data_dir=data_dir), batch_size=args.batch_size) + feeder = fluid.DataFeeder(place=place, feed_list=feed_vars) + + total = 0 + correct = 0 + correct_5 = 0 + for batch_id, data in enumerate(val_reader()): + labels = [] + in_data = [] + for dd in data: + labels.append(dd[1]) + in_data.append([np.array(dd[0])]) + t1 = time.time() + fetch_out = exe.run(inference_program, + fetch_list=fetch_targets, + feed=feeder.feed(in_data)) + t2 = time.time() + for i in range(len(labels)): + label = labels[i] + result = np.array(fetch_out[0][i]) + index = result.argsort() + top_1_index = index[-1] + top_5_index = index[-5:] + total += 1 + if top_1_index == label: + correct += 1 + if label in top_5_index: + correct_5 += 1 + + if batch_id % 10 == 0: + acc1 = float(correct) / float(total) + acc5 = float(correct_5) / float(total) + period = t2 - t1 + print("Testbatch {0}, " + "acc1 {1}, acc5 {2}, time {3}".format(batch_id, \ + acc1, acc5, "%2.2f sec" % period)) + sys.stdout.flush() + if args.test_samples > 0 and \ + batch_id * args.batch_size > args.test_samples: + break + total_acc1 = float(correct) / float(total) + total_acc5 = float(correct_5) / float(total) + print("End test: test_acc1 {0}, test_acc5 {1}".format(total_acc1, + total_acc5)) + sys.stdout.flush() + + +def main(): + args = parser.parse_args() + print_arguments(args) + eval(args) + + +if __name__ == '__main__': + main() diff --git a/PaddleSlim/quant_low_level_api/dygraph_quant/models/__init__.py b/PaddleSlim/quant_low_level_api/dygraph_quant/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..60d8c246ae10e2bcb2a6576ce13a99e5e984c5bc --- /dev/null +++ b/PaddleSlim/quant_low_level_api/dygraph_quant/models/__init__.py @@ -0,0 +1,31 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve. +# +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. + +from . import resnet +from . import vgg +from . import mobilenetv1 +from . import mobilenetv2 +from . import lenet + +from .resnet import * +from .mobilenetv1 import * +from .mobilenetv2 import * +from .vgg import * +from .lenet import * + +__all__ = resnet.__all__ \ + + vgg.__all__ \ + + mobilenetv1.__all__ \ + + mobilenetv2.__all__ \ + + lenet.__all__ diff --git a/PaddleSlim/quant_low_level_api/dygraph_quant/models/lenet.py b/PaddleSlim/quant_low_level_api/dygraph_quant/models/lenet.py new file mode 100644 index 0000000000000000000000000000000000000000..a2f2d335548d298aa63e803ae9911170ed5cba1e --- /dev/null +++ b/PaddleSlim/quant_low_level_api/dygraph_quant/models/lenet.py @@ -0,0 +1,63 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve. +# +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. + +import paddle.fluid as fluid +from paddle.fluid.dygraph.nn import Conv2D, BatchNorm, Pool2D, Linear +from paddle.fluid.dygraph.container import Sequential + +__all__ = ['LeNet'] + + +class LeNet(fluid.dygraph.Layer): + """LeNet model from + `"LeCun Y, Bottou L, Bengio Y, et al. Gradient-based learning applied to document recognition[J]. Proceedings of the IEEE, 1998, 86(11): 2278-2324.`_ + + Args: + num_classes (int): output dim of last fc layer. If num_classes <=0, last fc layer + will not be defined. Default: 10. + classifier_activation (str): activation for the last fc layer. Default: 'softmax'. + + Examples: + .. code-block:: python + + from paddle.incubate.hapi.vision.models import LeNet + + model = LeNet() + """ + + def __init__(self, num_classes=10, classifier_activation='softmax'): + super(LeNet, self).__init__() + self.num_classes = num_classes + self.features = Sequential( + Conv2D( + 1, 6, 3, stride=1, padding=1), + Pool2D(2, 'max', 2), + Conv2D( + 6, 16, 5, stride=1, padding=0), + Pool2D(2, 'max', 2)) + + if num_classes > 0: + self.fc = Sequential( + Linear(400, 120), + Linear(120, 84), + Linear( + 84, 10, act=classifier_activation)) + + def forward(self, inputs): + x = self.features(inputs) + + if self.num_classes > 0: + x = fluid.layers.flatten(x, 1) + x = self.fc(x) + return x diff --git a/PaddleSlim/quant_low_level_api/dygraph_quant/models/mobilenetv1.py b/PaddleSlim/quant_low_level_api/dygraph_quant/models/mobilenetv1.py new file mode 100644 index 0000000000000000000000000000000000000000..306866994734e029f7acf3afde89c6fa17ba393e --- /dev/null +++ b/PaddleSlim/quant_low_level_api/dygraph_quant/models/mobilenetv1.py @@ -0,0 +1,342 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import paddle +import paddle.fluid as fluid +from paddle.fluid.initializer import MSRA +from paddle.fluid.param_attr import ParamAttr +from paddle.fluid.dygraph.nn import Conv2D, Pool2D, BatchNorm, Linear +from paddle.fluid.dygraph import declarative + +from download import get_weights_path_from_url +import re + +__all__ = ['MobileNetV1', 'mobilenet_v1'] + +model_urls = { + 'mobilenetv1_1.0': + ('https://paddle-hapi.bj.bcebos.com/models/mobilenet_v1_x1.0.pdparams', + 'bf0d25cb0bed1114d9dac9384ce2b4a6') +} + + +class ConvBNLayer(fluid.dygraph.Layer): + def __init__(self, + num_channels, + filter_size, + num_filters, + stride, + padding, + channels=None, + num_groups=1, + act='relu', + use_cudnn=True, + name=None): + super(ConvBNLayer, self).__init__() + + self._conv = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=padding, + groups=num_groups, + act=None, + use_cudnn=use_cudnn, + param_attr=ParamAttr( + initializer=MSRA(), name=self.full_name() + "_weights"), + bias_attr=False) + + self._batch_norm = BatchNorm( + num_filters, + act=act, + param_attr=ParamAttr(name=self.full_name() + "_bn" + "_scale"), + bias_attr=ParamAttr(name=self.full_name() + "_bn" + "_offset"), + moving_mean_name=self.full_name() + "_bn" + '_mean', + moving_variance_name=self.full_name() + "_bn" + '_variance') + + def forward(self, inputs): + y = self._conv(inputs) + y = self._batch_norm(y) + return y + + +class DepthwiseSeparable(fluid.dygraph.Layer): + def __init__(self, + num_channels, + num_filters1, + num_filters2, + num_groups, + stride, + scale, + name=None): + super(DepthwiseSeparable, self).__init__() + + self._depthwise_conv = ConvBNLayer( + num_channels=num_channels, + num_filters=int(num_filters1 * scale), + filter_size=3, + stride=stride, + padding=1, + num_groups=int(num_groups * scale), + use_cudnn=False) + + self._pointwise_conv = ConvBNLayer( + num_channels=int(num_filters1 * scale), + filter_size=1, + num_filters=int(num_filters2 * scale), + stride=1, + padding=0) + + def forward(self, inputs): + y = self._depthwise_conv(inputs) + y = self._pointwise_conv(y) + return y + + +class MobileNetV1(fluid.dygraph.Layer): + """MobileNetV1 model from + `"MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications" `_. + + Args: + scale (float): scale of channels in each layer. Default: 1.0. + num_classes (int): output dim of last fc layer. If num_classes <=0, last fc layer + will not be defined. Default: 1000. + with_pool (bool): use pool before the last fc layer or not. Default: True. + classifier_activation (str): activation for the last fc layer. Default: 'softmax'. + + Examples: + .. code-block:: python + + from paddle.incubate.hapi.vision.models import MobileNetV1 + + model = MobileNetV1() + """ + + def __init__(self, + scale=1.0, + num_classes=1000, + with_pool=True, + classifier_activation='softmax'): + super(MobileNetV1, self).__init__() + self.scale = scale + self.dwsl = [] + self.num_classes = num_classes + self.with_pool = with_pool + + self.conv1 = ConvBNLayer( + num_channels=3, + filter_size=3, + channels=3, + num_filters=int(32 * scale), + stride=2, + padding=1) + + dws21 = self.add_sublayer( + sublayer=DepthwiseSeparable( + num_channels=int(32 * scale), + num_filters1=32, + num_filters2=64, + num_groups=32, + stride=1, + scale=scale), + name="conv2_1") + self.dwsl.append(dws21) + + dws22 = self.add_sublayer( + sublayer=DepthwiseSeparable( + num_channels=int(64 * scale), + num_filters1=64, + num_filters2=128, + num_groups=64, + stride=2, + scale=scale), + name="conv2_2") + self.dwsl.append(dws22) + + dws31 = self.add_sublayer( + sublayer=DepthwiseSeparable( + num_channels=int(128 * scale), + num_filters1=128, + num_filters2=128, + num_groups=128, + stride=1, + scale=scale), + name="conv3_1") + self.dwsl.append(dws31) + + dws32 = self.add_sublayer( + sublayer=DepthwiseSeparable( + num_channels=int(128 * scale), + num_filters1=128, + num_filters2=256, + num_groups=128, + stride=2, + scale=scale), + name="conv3_2") + self.dwsl.append(dws32) + + dws41 = self.add_sublayer( + sublayer=DepthwiseSeparable( + num_channels=int(256 * scale), + num_filters1=256, + num_filters2=256, + num_groups=256, + stride=1, + scale=scale), + name="conv4_1") + self.dwsl.append(dws41) + + dws42 = self.add_sublayer( + sublayer=DepthwiseSeparable( + num_channels=int(256 * scale), + num_filters1=256, + num_filters2=512, + num_groups=256, + stride=2, + scale=scale), + name="conv4_2") + self.dwsl.append(dws42) + + for i in range(5): + tmp = self.add_sublayer( + sublayer=DepthwiseSeparable( + num_channels=int(512 * scale), + num_filters1=512, + num_filters2=512, + num_groups=512, + stride=1, + scale=scale), + name="conv5_" + str(i + 1)) + self.dwsl.append(tmp) + + dws56 = self.add_sublayer( + sublayer=DepthwiseSeparable( + num_channels=int(512 * scale), + num_filters1=512, + num_filters2=1024, + num_groups=512, + stride=2, + scale=scale), + name="conv5_6") + self.dwsl.append(dws56) + + dws6 = self.add_sublayer( + sublayer=DepthwiseSeparable( + num_channels=int(1024 * scale), + num_filters1=1024, + num_filters2=1024, + num_groups=1024, + stride=1, + scale=scale), + name="conv6") + self.dwsl.append(dws6) + + if with_pool: + self.pool2d_avg = Pool2D(pool_type='avg', global_pooling=True) + + if num_classes > -1: + self.out = Linear( + int(1024 * scale), + num_classes, + act=classifier_activation, + param_attr=ParamAttr( + initializer=MSRA(), name=self.full_name() + "fc7_weights"), + bias_attr=ParamAttr(name="fc7_offset")) + + @declarative + def forward(self, inputs): + y = self.conv1(inputs) + for dws in self.dwsl: + y = dws(y) + + if self.with_pool: + y = self.pool2d_avg(y) + + if self.num_classes > 0: + y = fluid.layers.reshape(y, shape=[-1, 1024]) + y = self.out(y) + return y + + +def _mobilenet(arch, pretrained=False, **kwargs): + model = MobileNetV1(**kwargs) + if pretrained: + assert arch in model_urls, "{} model do not have a pretrained model now, you should set pretrained=False".format( + arch) + weight_path = get_weights_path_from_url(model_urls[arch][0], + model_urls[arch][1]) + assert weight_path.endswith( + '.pdparams'), "suffix of weight must be .pdparams" + print("Load pretrained weights from " + weight_path + " ...") + #model.load(weight_path) + #paddle.imperative.save(model.state_dict(), "weight_path") + model_dict, _ = fluid.load_dygraph(weight_path) + # model.set_dict(model_dict) + dks = [v for v in model_dict.keys()] + new_dks = [] + d_map = {} + for v in dks: + nv = re.sub("\.", "_", v) + nv = re.sub("__", "_", nv) + new_dks.append(nv) + d_map[nv] = v + new_dks = sorted(new_dks) + + model_path = '/work/Develop/sync_work/models/PaddleSlim/pretrain/MobileNetV1_pretrained' + load_state = fluid.load_program_state(model_path) + sks = [v for v in load_state.keys()] + new_sks = [] + s_map = {} + for v in sks: + nv = re.sub("offset", "bias", v) + nv = re.sub("scale", "weight", nv) + nv = re.sub("bn", "batch_norm", nv) + nv = re.sub("dw", "depthwise_conv", nv) + nv = re.sub("sep", "pointwise_conv", nv) + s_map[nv] = v + new_sks.append(nv) + new_sks = sorted(new_sks) + final_state = {} + for d, s in zip(new_dks, new_sks): + final_state[d_map[d]] = load_state[s_map[s]] + model.set_dict(final_state) + + return model + + +def mobilenet_v1(pretrained=False, scale=1.0, **kwargs): + """MobileNetV1 + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet. Default: False. + scale: (float): scale of channels in each layer. Default: 1.0. + + Examples: + .. code-block:: python + + from paddle.incubate.hapi.vision.models import mobilenet_v1 + + # build model + model = mobilenet_v1() + + # build model and load imagenet pretrained weight + # model = mobilenet_v1(pretrained=True) + + # build mobilenet v1 with scale=0.5 + model = mobilenet_v1(scale=0.5) + """ + model = _mobilenet( + 'mobilenetv1_' + str(scale), pretrained, scale=scale, **kwargs) + return model diff --git a/PaddleSlim/quant_low_level_api/dygraph_quant/models/mobilenetv2.py b/PaddleSlim/quant_low_level_api/dygraph_quant/models/mobilenetv2.py new file mode 100644 index 0000000000000000000000000000000000000000..26449041b733392349194c36aa4b5ba7965e2755 --- /dev/null +++ b/PaddleSlim/quant_low_level_api/dygraph_quant/models/mobilenetv2.py @@ -0,0 +1,287 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import paddle +import paddle.fluid as fluid +from paddle.fluid.param_attr import ParamAttr +from paddle.fluid.dygraph.nn import Conv2D, Pool2D, BatchNorm, Linear +from paddle.fluid.dygraph import declarative + +from download import get_weights_path_from_url + +__all__ = ['MobileNetV2', 'mobilenet_v2'] + +model_urls = { + 'mobilenetv2_1.0': + ('https://paddle-hapi.bj.bcebos.com/models/mobilenet_v2_x1.0.pdparams', + '8ff74f291f72533f2a7956a4efff9d88') +} + + +class ConvBNLayer(fluid.dygraph.Layer): + def __init__(self, + num_channels, + filter_size, + num_filters, + stride, + padding, + channels=None, + num_groups=1, + use_cudnn=True): + super(ConvBNLayer, self).__init__() + + tmp_param = ParamAttr(name=self.full_name() + "_weights") + self._conv = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=padding, + groups=num_groups, + act=None, + use_cudnn=use_cudnn, + param_attr=tmp_param, + bias_attr=False) + + self._batch_norm = BatchNorm( + num_filters, + param_attr=ParamAttr(name=self.full_name() + "_bn" + "_scale"), + bias_attr=ParamAttr(name=self.full_name() + "_bn" + "_offset"), + moving_mean_name=self.full_name() + "_bn" + '_mean', + moving_variance_name=self.full_name() + "_bn" + '_variance') + + def forward(self, inputs, if_act=True): + y = self._conv(inputs) + y = self._batch_norm(y) + if if_act: + y = fluid.layers.relu6(y) + return y + + +class InvertedResidualUnit(fluid.dygraph.Layer): + def __init__( + self, + num_channels, + num_in_filter, + num_filters, + stride, + filter_size, + padding, + expansion_factor, ): + super(InvertedResidualUnit, self).__init__() + num_expfilter = int(round(num_in_filter * expansion_factor)) + self._expand_conv = ConvBNLayer( + num_channels=num_channels, + num_filters=num_expfilter, + filter_size=1, + stride=1, + padding=0, + num_groups=1) + + self._bottleneck_conv = ConvBNLayer( + num_channels=num_expfilter, + num_filters=num_expfilter, + filter_size=filter_size, + stride=stride, + padding=padding, + num_groups=num_expfilter, + use_cudnn=False) + + self._linear_conv = ConvBNLayer( + num_channels=num_expfilter, + num_filters=num_filters, + filter_size=1, + stride=1, + padding=0, + num_groups=1) + + def forward(self, inputs, ifshortcut): + y = self._expand_conv(inputs, if_act=True) + y = self._bottleneck_conv(y, if_act=True) + y = self._linear_conv(y, if_act=False) + if ifshortcut: + y = fluid.layers.elementwise_add(inputs, y) + return y + + +class InvresiBlocks(fluid.dygraph.Layer): + def __init__(self, in_c, t, c, n, s): + super(InvresiBlocks, self).__init__() + + self._first_block = InvertedResidualUnit( + num_channels=in_c, + num_in_filter=in_c, + num_filters=c, + stride=s, + filter_size=3, + padding=1, + expansion_factor=t) + + self._inv_blocks = [] + for i in range(1, n): + tmp = self.add_sublayer( + sublayer=InvertedResidualUnit( + num_channels=c, + num_in_filter=c, + num_filters=c, + stride=1, + filter_size=3, + padding=1, + expansion_factor=t), + name=self.full_name() + "_" + str(i + 1)) + self._inv_blocks.append(tmp) + + def forward(self, inputs): + y = self._first_block(inputs, ifshortcut=False) + for inv_block in self._inv_blocks: + y = inv_block(y, ifshortcut=True) + return y + + +class MobileNetV2(fluid.dygraph.Layer): + """MobileNetV2 model from + `"MobileNetV2: Inverted Residuals and Linear Bottlenecks" `_. + + Args: + scale (float): scale of channels in each layer. Default: 1.0. + num_classes (int): output dim of last fc layer. If num_classes <=0, last fc layer + will not be defined. Default: 1000. + with_pool (bool): use pool before the last fc layer or not. Default: True. + classifier_activation (str): activation for the last fc layer. Default: 'softmax'. + + Examples: + .. code-block:: python + + from paddle.incubate.hapi.vision.models import MobileNetV2 + + model = MobileNetV2() + """ + + def __init__(self, + scale=1.0, + num_classes=1000, + with_pool=True, + classifier_activation='softmax'): + super(MobileNetV2, self).__init__() + self.scale = scale + self.num_classes = num_classes + self.with_pool = with_pool + + bottleneck_params_list = [ + (1, 16, 1, 1), + (6, 24, 2, 2), + (6, 32, 3, 2), + (6, 64, 4, 2), + (6, 96, 3, 1), + (6, 160, 3, 2), + (6, 320, 1, 1), + ] + + self._conv1 = ConvBNLayer( + num_channels=3, + num_filters=int(32 * scale), + filter_size=3, + stride=2, + padding=1) + + self._invl = [] + i = 1 + in_c = int(32 * scale) + for layer_setting in bottleneck_params_list: + t, c, n, s = layer_setting + i += 1 + tmp = self.add_sublayer( + sublayer=InvresiBlocks( + in_c=in_c, t=t, c=int(c * scale), n=n, s=s), + name='conv' + str(i)) + self._invl.append(tmp) + in_c = int(c * scale) + + self._out_c = int(1280 * scale) if scale > 1.0 else 1280 + self._conv9 = ConvBNLayer( + num_channels=in_c, + num_filters=self._out_c, + filter_size=1, + stride=1, + padding=0) + + if with_pool: + self._pool2d_avg = Pool2D(pool_type='avg', global_pooling=True) + + if num_classes > 0: + tmp_param = ParamAttr(name=self.full_name() + "fc10_weights") + self._fc = Linear( + self._out_c, + num_classes, + act=classifier_activation, + param_attr=tmp_param, + bias_attr=ParamAttr(name="fc10_offset")) + + @declarative + def forward(self, inputs): + y = self._conv1(inputs, if_act=True) + for inv in self._invl: + y = inv(y) + y = self._conv9(y, if_act=True) + + if self.with_pool: + y = self._pool2d_avg(y) + if self.num_classes > 0: + y = fluid.layers.reshape(y, shape=[-1, self._out_c]) + y = self._fc(y) + return y + + +def _mobilenet(arch, pretrained=False, **kwargs): + model = MobileNetV2(**kwargs) + if pretrained: + assert arch in model_urls, "{} model do not have a pretrained model now, you should set pretrained=False".format( + arch) + weight_path = get_weights_path_from_url(model_urls[arch][0], + model_urls[arch][1]) + assert weight_path.endswith( + '.pdparams'), "suffix of weight must be .pdparams" + #model.load(weight_path) + print("Load pretrained weights from " + weight_path + " ...") + model_dict, _ = fluid.load_dygraph(weight_path) + model.set_dict(model_dict) + + return model + + +def mobilenet_v2(pretrained=False, scale=1.0, **kwargs): + """MobileNetV2 + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet. Default: False. + scale: (float): scale of channels in each layer. Default: 1.0. + + Examples: + .. code-block:: python + + from paddle.incubate.hapi.vision.models import mobilenet_v2 + + # build model + model = mobilenet_v2() + + # build model and load imagenet pretrained weight + # model = mobilenet_v2(pretrained=True) + + # build mobilenet v2 with scale=0.5 + model = mobilenet_v2(scale=0.5) + """ + model = _mobilenet( + 'mobilenetv2_' + str(scale), pretrained, scale=scale, **kwargs) + return model diff --git a/PaddleSlim/quant_low_level_api/dygraph_quant/models/resnet.py b/PaddleSlim/quant_low_level_api/dygraph_quant/models/resnet.py new file mode 100644 index 0000000000000000000000000000000000000000..25c504bd972c633bede3b63bdd4ba007643374e0 --- /dev/null +++ b/PaddleSlim/quant_low_level_api/dygraph_quant/models/resnet.py @@ -0,0 +1,386 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import division +from __future__ import print_function + +import math +import paddle.fluid as fluid + +from paddle.fluid.dygraph.nn import Conv2D, Pool2D, BatchNorm, Linear +from paddle.fluid.dygraph.container import Sequential +from paddle.fluid.dygraph import declarative + +from download import get_weights_path_from_url + +__all__ = [ + 'ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', 'resnet152' +] + +model_urls = { + 'resnet18': ('https://paddle-hapi.bj.bcebos.com/models/resnet18.pdparams', + '0ba53eea9bc970962d0ef96f7b94057e'), + 'resnet34': ('https://paddle-hapi.bj.bcebos.com/models/resnet34.pdparams', + '46bc9f7c3dd2e55b7866285bee91eff3'), + 'resnet50': ('https://paddle-hapi.bj.bcebos.com/models/resnet50.pdparams', + '5ce890a9ad386df17cf7fe2313dca0a1'), + 'resnet101': ('https://paddle-hapi.bj.bcebos.com/models/resnet101.pdparams', + 'fb07a451df331e4b0bb861ed97c3a9b9'), + 'resnet152': ('https://paddle-hapi.bj.bcebos.com/models/resnet152.pdparams', + 'f9c700f26d3644bb76ad2226ed5f5713'), +} + + +class ConvBNLayer(fluid.dygraph.Layer): + def __init__(self, + num_channels, + num_filters, + filter_size, + stride=1, + groups=1, + act=None): + super(ConvBNLayer, self).__init__() + + self._conv = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + act=None, + bias_attr=False) + + self._batch_norm = BatchNorm(num_filters, act=act) + + def forward(self, inputs): + x = self._conv(inputs) + x = self._batch_norm(x) + + return x + + +class BasicBlock(fluid.dygraph.Layer): + """residual block of resnet18 and resnet34 + """ + expansion = 1 + + def __init__(self, num_channels, num_filters, stride, shortcut=True): + super(BasicBlock, self).__init__() + + self.conv0 = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters, + filter_size=3, + act='relu') + self.conv1 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters, + filter_size=3, + stride=stride, + act='relu') + + if not shortcut: + self.short = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters, + filter_size=1, + stride=stride) + + self.shortcut = shortcut + + def forward(self, inputs): + y = self.conv0(inputs) + conv1 = self.conv1(y) + + if self.shortcut: + short = inputs + else: + short = self.short(inputs) + + y = short + conv1 + + return fluid.layers.relu(y) + + +class BottleneckBlock(fluid.dygraph.Layer): + """residual block of resnet50, resnet101 amd resnet152 + """ + + expansion = 4 + + def __init__(self, num_channels, num_filters, stride, shortcut=True): + super(BottleneckBlock, self).__init__() + + self.conv0 = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters, + filter_size=1, + act='relu') + self.conv1 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters, + filter_size=3, + stride=stride, + act='relu') + self.conv2 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters * self.expansion, + filter_size=1, + act=None) + + if not shortcut: + self.short = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters * self.expansion, + filter_size=1, + stride=stride) + + self.shortcut = shortcut + + self._num_channels_out = num_filters * self.expansion + + def forward(self, inputs): + x = self.conv0(inputs) + conv1 = self.conv1(x) + conv2 = self.conv2(conv1) + + if self.shortcut: + short = inputs + else: + short = self.short(inputs) + + x = fluid.layers.elementwise_add(x=short, y=conv2) + + return fluid.layers.relu(x) + + +class ResNet(fluid.dygraph.Layer): + """ResNet model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + Block (BasicBlock|BottleneckBlock): block module of model. + depth (int): layers of resnet, default: 50. + num_classes (int): output dim of last fc layer. If num_classes <=0, last fc layer + will not be defined. Default: 1000. + with_pool (bool): use pool before the last fc layer or not. Default: True. + classifier_activation (str): activation for the last fc layer. Default: 'softmax'. + + Examples: + .. code-block:: python + + from paddle.incubate.hapi.vision.models import ResNet + from paddle.incubate.hapi.vision.models.resnet import BottleneckBlock, BasicBlock + + resnet50 = ResNet(BottleneckBlock, 50) + + resnet18 = ResNet(BasicBlock, 18) + + """ + + def __init__(self, + Block, + depth=50, + num_classes=1000, + with_pool=True, + classifier_activation='softmax'): + super(ResNet, self).__init__() + + self.num_classes = num_classes + self.with_pool = with_pool + + layer_config = { + 18: [2, 2, 2, 2], + 34: [3, 4, 6, 3], + 50: [3, 4, 6, 3], + 101: [3, 4, 23, 3], + 152: [3, 8, 36, 3], + } + assert depth in layer_config.keys(), \ + "supported depth are {} but input layer is {}".format( + layer_config.keys(), depth) + + layers = layer_config[depth] + + in_channels = 64 + out_channels = [64, 128, 256, 512] + + self.conv = ConvBNLayer( + num_channels=3, num_filters=64, filter_size=7, stride=2, act='relu') + self.pool = Pool2D( + pool_size=3, pool_stride=2, pool_padding=1, pool_type='max') + + self.layers = [] + for idx, num_blocks in enumerate(layers): + blocks = [] + shortcut = False + for b in range(num_blocks): + if b == 1: + in_channels = out_channels[idx] * Block.expansion + block = Block( + num_channels=in_channels, + num_filters=out_channels[idx], + stride=2 if b == 0 and idx != 0 else 1, + shortcut=shortcut) + blocks.append(block) + shortcut = True + layer = self.add_sublayer("layer_{}".format(idx), + Sequential(*blocks)) + self.layers.append(layer) + + if with_pool: + self.global_pool = Pool2D( + pool_size=7, pool_type='avg', global_pooling=True) + + if num_classes > 0: + stdv = 1.0 / math.sqrt(out_channels[-1] * Block.expansion * 1.0) + self.fc_input_dim = out_channels[-1] * Block.expansion * 1 * 1 + self.fc = Linear( + self.fc_input_dim, + num_classes, + act=classifier_activation, + param_attr=fluid.param_attr.ParamAttr( + initializer=fluid.initializer.Uniform(-stdv, stdv))) + + @declarative + def forward(self, inputs): + x = self.conv(inputs) + x = self.pool(x) + for layer in self.layers: + x = layer(x) + + if self.with_pool: + x = self.global_pool(x) + + if self.num_classes > -1: + x = fluid.layers.reshape(x, shape=[-1, self.fc_input_dim]) + x = self.fc(x) + return x + + +def _resnet(arch, Block, depth, pretrained, **kwargs): + model = ResNet(Block, depth, **kwargs) + if pretrained: + assert arch in model_urls, "{} model do not have a pretrained model now, you should set pretrained=False".format( + arch) + weight_path = get_weights_path_from_url(model_urls[arch][0], + model_urls[arch][1]) + assert weight_path.endswith( + '.pdparams'), "suffix of weight must be .pdparams" + #model.load(weight_path) + print("Load pretrained weights from " + weight_path + " ...") + model_dict, _ = fluid.load_dygraph(weight_path) + model.set_dict(model_dict) + return model + + +def resnet18(pretrained=False, **kwargs): + """ResNet 18-layer model + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + + Examples: + .. code-block:: python + + from paddle.incubate.hapi.vision.models import resnet18 + + # build model + model = resnet18() + + # build model and load imagenet pretrained weight + # model = resnet18(pretrained=True) + """ + return _resnet('resnet18', BasicBlock, 18, pretrained, **kwargs) + + +def resnet34(pretrained=False, **kwargs): + """ResNet 34-layer model + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + + Examples: + .. code-block:: python + + from paddle.incubate.hapi.vision.models import resnet34 + + # build model + model = resnet34() + + # build model and load imagenet pretrained weight + # model = resnet34(pretrained=True) + """ + return _resnet('resnet34', BasicBlock, 34, pretrained, **kwargs) + + +def resnet50(pretrained=False, **kwargs): + """ResNet 50-layer model + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + + Examples: + .. code-block:: python + + from paddle.incubate.hapi.vision.models import resnet50 + + # build model + model = resnet50() + + # build model and load imagenet pretrained weight + # model = resnet50(pretrained=True) + """ + return _resnet('resnet50', BottleneckBlock, 50, pretrained, **kwargs) + + +def resnet101(pretrained=False, **kwargs): + """ResNet 101-layer model + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + + Examples: + .. code-block:: python + + from paddle.incubate.hapi.vision.models import resnet101 + + # build model + model = resnet101() + + # build model and load imagenet pretrained weight + # model = resnet101(pretrained=True) + """ + return _resnet('resnet101', BottleneckBlock, 101, pretrained, **kwargs) + + +def resnet152(pretrained=False, **kwargs): + """ResNet 152-layer model + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + + Examples: + .. code-block:: python + + from paddle.incubate.hapi.vision.models import resnet152 + + # build model + model = resnet152() + + # build model and load imagenet pretrained weight + # model = resnet152(pretrained=True) + """ + return _resnet('resnet152', BottleneckBlock, 152, pretrained, **kwargs) diff --git a/PaddleSlim/quant_low_level_api/dygraph_quant/models/vgg.py b/PaddleSlim/quant_low_level_api/dygraph_quant/models/vgg.py new file mode 100644 index 0000000000000000000000000000000000000000..ef3c494f01dbbbc8f4a4104db63c647871905312 --- /dev/null +++ b/PaddleSlim/quant_low_level_api/dygraph_quant/models/vgg.py @@ -0,0 +1,246 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import paddle.fluid as fluid +from paddle.fluid.dygraph.nn import Conv2D, Pool2D, BatchNorm, Linear +from paddle.fluid.dygraph.container import Sequential +from paddle.fluid.dygraph import declarative + +from download import get_weights_path_from_url + +__all__ = [ + 'VGG', + 'vgg11', + 'vgg13', + 'vgg16', + 'vgg19', +] + +model_urls = { + 'vgg16': ('https://paddle-hapi.bj.bcebos.com/models/vgg16.pdparams', + 'c788f453a3b999063e8da043456281ee') +} + + +class Classifier(fluid.dygraph.Layer): + def __init__(self, num_classes, classifier_activation='softmax'): + super(Classifier, self).__init__() + self.linear1 = Linear(512 * 7 * 7, 4096) + self.linear2 = Linear(4096, 4096) + self.linear3 = Linear(4096, num_classes, act=classifier_activation) + + def forward(self, x): + x = self.linear1(x) + x = fluid.layers.relu(x) + x = fluid.layers.dropout(x, 0.5) + x = self.linear2(x) + x = fluid.layers.relu(x) + x = fluid.layers.dropout(x, 0.5) + out = self.linear3(x) + return out + + +class VGG(fluid.dygraph.Layer): + """VGG model from + `"Very Deep Convolutional Networks For Large-Scale Image Recognition" `_ + + Args: + features (fluid.dygraph.Layer): vgg features create by function make_layers. + num_classes (int): output dim of last fc layer. If num_classes <=0, last fc layer + will not be defined. Default: 1000. + classifier_activation (str): activation for the last fc layer. Default: 'softmax'. + + Examples: + .. code-block:: python + + from paddle.incubate.hapi.vision.models import VGG + from paddle.incubate.hapi.vision.models.vgg import make_layers + + vgg11_cfg = [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'] + + features = make_layers(vgg11_cfg) + + vgg11 = VGG(features) + + """ + + def __init__(self, + features, + num_classes=1000, + classifier_activation='softmax'): + super(VGG, self).__init__() + self.features = features + self.num_classes = num_classes + + if num_classes > 0: + classifier = Classifier(num_classes, classifier_activation) + self.classifier = self.add_sublayer("classifier", + Sequential(classifier)) + + @declarative + def forward(self, x): + x = self.features(x) + + if self.num_classes > 0: + x = fluid.layers.flatten(x, 1) + x = self.classifier(x) + return x + + +def make_layers(cfg, batch_norm=False): + layers = [] + in_channels = 3 + + for v in cfg: + if v == 'M': + layers += [Pool2D(pool_size=2, pool_stride=2)] + else: + if batch_norm: + conv2d = Conv2D(in_channels, v, filter_size=3, padding=1) + layers += [conv2d, BatchNorm(v, act='relu')] + else: + conv2d = Conv2D( + in_channels, v, filter_size=3, padding=1, act='relu') + layers += [conv2d] + in_channels = v + return Sequential(*layers) + + +cfgs = { + 'A': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], + 'B': + [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], + 'D': [ + 64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, + 512, 512, 'M' + ], + 'E': [ + 64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, + 'M', 512, 512, 512, 512, 'M' + ], +} + + +def _vgg(arch, cfg, batch_norm, pretrained, **kwargs): + model = VGG(make_layers( + cfgs[cfg], batch_norm=batch_norm), + num_classes=1000, + **kwargs) + + if pretrained: + assert arch in model_urls, "{} model do not have a pretrained model now, you should set pretrained=False".format( + arch) + weight_path = get_weights_path_from_url(model_urls[arch][0], + model_urls[arch][1]) + assert weight_path.endswith( + '.pdparams'), "suffix of weight must be .pdparams" + model.load(weight_path) + + return model + + +def vgg11(pretrained=False, batch_norm=False, **kwargs): + """VGG 11-layer model + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet. Default: False. + batch_norm (bool): If True, returns a model with batch_norm layer. Default: False. + + Examples: + .. code-block:: python + + from paddle.incubate.hapi.vision.models import vgg11 + + # build model + model = vgg11() + + # build vgg11 model with batch_norm + model = vgg11(batch_norm=True) + """ + model_name = 'vgg11' + if batch_norm: + model_name += ('_bn') + return _vgg(model_name, 'A', batch_norm, pretrained, **kwargs) + + +def vgg13(pretrained=False, batch_norm=False, **kwargs): + """VGG 13-layer model + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet. Default: False. + batch_norm (bool): If True, returns a model with batch_norm layer. Default: False. + + Examples: + .. code-block:: python + + from paddle.incubate.hapi.vision.models import vgg13 + + # build model + model = vgg13() + + # build vgg13 model with batch_norm + model = vgg13(batch_norm=True) + """ + model_name = 'vgg13' + if batch_norm: + model_name += ('_bn') + return _vgg(model_name, 'B', batch_norm, pretrained, **kwargs) + + +def vgg16(pretrained=False, batch_norm=False, **kwargs): + """VGG 16-layer model + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet. Default: False. + batch_norm (bool): If True, returns a model with batch_norm layer. Default: False. + + Examples: + .. code-block:: python + + from paddle.incubate.hapi.vision.models import vgg16 + + # build model + model = vgg16() + + # build vgg16 model with batch_norm + model = vgg16(batch_norm=True) + """ + model_name = 'vgg16' + if batch_norm: + model_name += ('_bn') + return _vgg(model_name, 'D', batch_norm, pretrained, **kwargs) + + +def vgg19(pretrained=False, batch_norm=False, **kwargs): + """VGG 19-layer model + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet. Default: False. + batch_norm (bool): If True, returns a model with batch_norm layer. Default: False. + + Examples: + .. code-block:: python + + from paddle.incubate.hapi.vision.models import vgg19 + + # build model + model = vgg19() + + # build vgg19 model with batch_norm + model = vgg19(batch_norm=True) + """ + model_name = 'vgg19' + if batch_norm: + model_name += ('_bn') + return _vgg(model_name, 'E', batch_norm, pretrained, **kwargs) diff --git a/PaddleSlim/quant_low_level_api/dygraph_quant/quant_dygraph.py b/PaddleSlim/quant_low_level_api/dygraph_quant/quant_dygraph.py new file mode 100644 index 0000000000000000000000000000000000000000..2cc03c0da51f5d16637a5d40e28acbe855539a12 --- /dev/null +++ b/PaddleSlim/quant_low_level_api/dygraph_quant/quant_dygraph.py @@ -0,0 +1,225 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import division +from __future__ import print_function + +import argparse +import os + +import time +import math +import numpy as np + +import reader +import models +import paddle +import paddle.fluid as fluid +from paddle.fluid.contrib.slim.quantization import DygraphQuantAware +from paddle.fluid.optimizer import AdamOptimizer, SGD + + +def make_optimizer(step_per_epoch, parameter_list=None): + base_lr = FLAGS.lr + lr_scheduler = FLAGS.lr_scheduler + momentum = FLAGS.momentum + weight_decay = FLAGS.weight_decay + + if lr_scheduler == 'piecewise': + milestones = FLAGS.milestones + boundaries = [step_per_epoch * e for e in milestones] + values = [base_lr * (0.1**i) for i in range(len(boundaries) + 1)] + print("lr value:" + str(values)) + learning_rate = fluid.layers.piecewise_decay( + boundaries=boundaries, values=values) + elif lr_scheduler == 'cosine': + learning_rate = fluid.layers.cosine_decay(base_lr, step_per_epoch, + FLAGS.epoch) + else: + raise ValueError( + "Expected lr_scheduler in ['piecewise', 'cosine'], but got {}". + format(lr_scheduler)) + + learning_rate = fluid.layers.linear_lr_warmup( + learning_rate=learning_rate, + warmup_steps=5 * step_per_epoch, + start_lr=0., + end_lr=base_lr) + + # optimizer = fluid.optimizer.Momentum( + # learning_rate=learning_rate, + # momentum=momentum, + # regularization=fluid.regularizer.L2Decay(weight_decay), + # parameter_list=parameter_list) + + optimizer = fluid.optimizer.AdamOptimizer( + learning_rate=0.001, parameter_list=parameter_list) + + return optimizer + + +def main(): + dygraph_qat = DygraphQuantAware() + paddle.enable_imperative() + + print("Load model ...") + model_list = [x for x in models.__dict__["__all__"]] + assert FLAGS.model_name in model_list, "Expected FLAGS.model_name in {}, but received {}".format( + model_list, FLAGS.model_name) + model = models.__dict__[FLAGS.model_name](pretrained=True) # load weights + + print("Quantize model ...") + dygraph_qat.quantize(model) + + print("Prepare train ...") + adam = SGD(learning_rate=0.1, parameter_list=model.parameters()) + train_reader = paddle.batch( + reader.train(data_dir=FLAGS.data_path), + batch_size=FLAGS.batch_size, + drop_last=True) + test_reader = paddle.batch( + reader.val(data_dir=FLAGS.data_path), batch_size=128) + + print("Train and test ...") + for epoch in range(FLAGS.epoch): + if not FLAGS.action_only_eval: + # Train + model.train() + for batch_id, data in enumerate(test_reader()): + x_data = np.array( + [x[0].reshape(3, 224, 224) for x in data]).astype('float32') + # x_data = np.ones_like(np.array(x_data)) * batch_id + # print(x_data[0][0][0][:10]) + y_data = np.array([x[1] for x in data]).astype('int64').reshape( + -1, 1) + for p in model.parameters(): + if p.name == 'conv_bn_layer_0_weights': + # print("weight check----------------", p.numpy()[0][0][0][:10]) + pass + img = fluid.dygraph.to_variable(x_data) + label = fluid.dygraph.to_variable(y_data) + out = model(img) + acc = fluid.layers.accuracy(out, label) + loss = fluid.layers.cross_entropy(out, label) + avg_loss = fluid.layers.mean(loss) + avg_loss.backward() + adam.minimize(avg_loss) + model.clear_gradients() + if batch_id % 1 == 0: + print("Train | At epoch {} step {}: loss = {:}, acc= {:}". + format(epoch, batch_id, avg_loss.numpy()[0], acc.numpy( + )[0])) + if FLAGS.action_fast_test and batch_id > 20: + break + + # Test + model.eval() + all_acc_top1 = 0 + all_acc_top5 = 0 + for batch_id, data in enumerate(test_reader()): + x_data = np.array([x[0].reshape(3, 224, 224) + for x in data]).astype('float32') + y_data = np.array( + [x[1] for x in data]).astype('int64').reshape(-1, 1) + + img = fluid.dygraph.to_variable(x_data) + label = fluid.dygraph.to_variable(y_data) + + out = model(img) + acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1) + acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5) + all_acc_top1 += acc_top1.numpy() + all_acc_top5 += acc_top5.numpy() + + if batch_id % 10 == 0: + print( + "Test | At epoch {} step {}: avg_acc1 = {:}, avg_acc5 = {:}". + format(epoch, batch_id, all_acc_top1 / (batch_id + 1), + all_acc_top5 / (batch_id + 1))) + if FLAGS.action_fast_test and batch_id > 20: + break + print( + "Finish Test | At epoch {} step {}: avg_acc1 = {:}, avg_acc5 = {:}". + format(epoch, batch_id, all_acc_top1 / (batch_id + 1), all_acc_top5 + / (batch_id + 1))) + + # save inference quantized model + print("Save quantized model ...") + output_dir = os.path.join(FLAGS.output_dir, FLAGS.model_name) + if not os.path.exists(output_dir): + os.makedirs(output_dir) + save_path = output_dir + "_epoch" + str(epoch) + dygraph_qat.save_infer_quant_model( + dirname=save_path, + model=model, + input_shape=[(3, 224, 224)], + input_dtype=['float32'], + feed=[0], + fetch=[0]) + print("Finish quantization, and save quantized model to " + save_path + + "\n\n") + + +if __name__ == '__main__': + parser = argparse.ArgumentParser("Training on ImageNet") + parser.add_argument( + '--data_path', + default="/work/datasets/ILSVRC2012/", + help='path to dataset ' + '(should have subdirectories named "train" and "val"') + parser.add_argument( + "--output_dir", type=str, default='output', help="save dir") + parser.add_argument( + "--model_name", type=str, default='mobilenet_v1', help="model name") + parser.add_argument( + "--device", type=str, default='gpu', help="device to run, cpu or gpu") + parser.add_argument( + "-e", "--epoch", default=3, type=int, help="number of epoch") + parser.add_argument( + "-b", "--batch_size", default=64, type=int, help="batch size") + parser.add_argument( + "--action_only_eval", action="store_true", help="not train, only eval") + parser.add_argument( + "--action_fast_test", + action="store_true", + help="fast train and test a model") + + parser.add_argument( + "--image_size", default=224, type=int, help="intput image size") + parser.add_argument( + '--lr', + '--learning_rate', + default=0.0001, + type=float, + metavar='LR', + help='initial learning rate') + parser.add_argument( + "--lr_scheduler", + default='piecewise', + type=str, + help="learning rate scheduler") + parser.add_argument( + "--milestones", + nargs='+', + type=int, + default=[30, 60, 80], + help="piecewise decay milestones") + parser.add_argument( + "--weight_decay", default=1e-4, type=float, help="weight decay") + parser.add_argument("--momentum", default=0.9, type=float, help="momentum") + + FLAGS = parser.parse_args() + print("Input params:") + print(FLAGS) + main() diff --git a/PaddleSlim/quant_low_level_api/dygraph_quant/reader.py b/PaddleSlim/quant_low_level_api/dygraph_quant/reader.py new file mode 100644 index 0000000000000000000000000000000000000000..62e14ef68f319b0f12f8d6a9d93d7224f0607b90 --- /dev/null +++ b/PaddleSlim/quant_low_level_api/dygraph_quant/reader.py @@ -0,0 +1,205 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import math +import random +import functools +import numpy as np +import paddle +from PIL import Image, ImageEnhance + +random.seed(0) +np.random.seed(0) + +DATA_DIM = 224 + +THREAD = 1 +BUF_SIZE = 10240 + +DATA_DIR = 'data/ILSVRC2012' + +img_mean = np.array([0.485, 0.456, 0.406]).reshape((3, 1, 1)) +img_std = np.array([0.229, 0.224, 0.225]).reshape((3, 1, 1)) + + +def resize_short(img, target_size): + percent = float(target_size) / min(img.size[0], img.size[1]) + resized_width = int(round(img.size[0] * percent)) + resized_height = int(round(img.size[1] * percent)) + img = img.resize((resized_width, resized_height), Image.LANCZOS) + return img + + +def crop_image(img, target_size, center): + width, height = img.size + size = target_size + if center == True: + w_start = (width - size) / 2 + h_start = (height - size) / 2 + else: + w_start = np.random.randint(0, width - size + 1) + h_start = np.random.randint(0, height - size + 1) + w_end = w_start + size + h_end = h_start + size + img = img.crop((w_start, h_start, w_end, h_end)) + return img + + +def random_crop(img, size, scale=[0.08, 1.0], ratio=[3. / 4., 4. / 3.]): + aspect_ratio = math.sqrt(np.random.uniform(*ratio)) + w = 1. * aspect_ratio + h = 1. / aspect_ratio + + bound = min((float(img.size[0]) / img.size[1]) / (w**2), + (float(img.size[1]) / img.size[0]) / (h**2)) + scale_max = min(scale[1], bound) + scale_min = min(scale[0], bound) + + target_area = img.size[0] * img.size[1] * np.random.uniform(scale_min, + scale_max) + target_size = math.sqrt(target_area) + w = int(target_size * w) + h = int(target_size * h) + + i = np.random.randint(0, img.size[0] - w + 1) + j = np.random.randint(0, img.size[1] - h + 1) + + img = img.crop((i, j, i + w, j + h)) + img = img.resize((size, size), Image.LANCZOS) + return img + + +def rotate_image(img): + angle = np.random.randint(-10, 11) + img = img.rotate(angle) + return img + + +def distort_color(img): + def random_brightness(img, lower=0.5, upper=1.5): + e = np.random.uniform(lower, upper) + return ImageEnhance.Brightness(img).enhance(e) + + def random_contrast(img, lower=0.5, upper=1.5): + e = np.random.uniform(lower, upper) + return ImageEnhance.Contrast(img).enhance(e) + + def random_color(img, lower=0.5, upper=1.5): + e = np.random.uniform(lower, upper) + return ImageEnhance.Color(img).enhance(e) + + ops = [random_brightness, random_contrast, random_color] + np.random.shuffle(ops) + + img = ops[0](img) + img = ops[1](img) + img = ops[2](img) + + return img + + +def process_image(sample, mode, color_jitter, rotate): + img_path = sample[0] + + img = Image.open(img_path) + if mode == 'train': + if rotate: img = rotate_image(img) + img = random_crop(img, DATA_DIM) + else: + img = resize_short(img, target_size=256) + img = crop_image(img, target_size=DATA_DIM, center=True) + if mode == 'train': + if color_jitter: + img = distort_color(img) + if np.random.randint(0, 2) == 1: + img = img.transpose(Image.FLIP_LEFT_RIGHT) + + if img.mode != 'RGB': + img = img.convert('RGB') + + img = np.array(img).astype('float32').transpose((2, 0, 1)) / 255 + img -= img_mean + img /= img_std + + if mode == 'train' or mode == 'val': + return img, sample[1] + elif mode == 'test': + return [img] + + +def _reader_creator(file_list, + mode, + shuffle=False, + color_jitter=False, + rotate=False, + data_dir=DATA_DIR, + batch_size=1): + def reader(): + try: + with open(file_list) as flist: + full_lines = [line.strip() for line in flist] + if shuffle: + np.random.shuffle(full_lines) + if mode == 'train' and os.getenv('PADDLE_TRAINING_ROLE'): + # distributed mode if the env var `PADDLE_TRAINING_ROLE` exits + trainer_id = int(os.getenv("PADDLE_TRAINER_ID", "0")) + trainer_count = int(os.getenv("PADDLE_TRAINERS", "1")) + per_node_lines = len(full_lines) // trainer_count + lines = full_lines[trainer_id * per_node_lines:( + trainer_id + 1) * per_node_lines] + print( + "read images from %d, length: %d, lines length: %d, total: %d" + % (trainer_id * per_node_lines, per_node_lines, + len(lines), len(full_lines))) + else: + lines = full_lines + + for line in lines: + if mode == 'train' or mode == 'val': + img_path, label = line.split() + img_path = os.path.join(data_dir, img_path) + yield img_path, int(label) + elif mode == 'test': + img_path = os.path.join(data_dir, line) + yield [img_path] + except Exception as e: + print("Reader failed!\n{}".format(str(e))) + os._exit(1) + + mapper = functools.partial( + process_image, mode=mode, color_jitter=color_jitter, rotate=rotate) + + return paddle.reader.xmap_readers(mapper, reader, THREAD, BUF_SIZE) + + +def train(data_dir=DATA_DIR): + file_list = os.path.join(data_dir, 'train_list.txt') + return _reader_creator( + file_list, + 'train', + shuffle=False, + color_jitter=False, + rotate=False, + data_dir=data_dir) + + +def val(data_dir=DATA_DIR): + file_list = os.path.join(data_dir, 'val_list.txt') + return _reader_creator(file_list, 'val', shuffle=False, data_dir=data_dir) + + +def test(data_dir=DATA_DIR): + file_list = os.path.join(data_dir, 'val_list.txt') + return _reader_creator(file_list, 'test', shuffle=False, data_dir=data_dir) diff --git a/PaddleSlim/quant_low_level_api/dygraph_quant/run_quant.sh b/PaddleSlim/quant_low_level_api/dygraph_quant/run_quant.sh new file mode 100644 index 0000000000000000000000000000000000000000..a7ed93bb76ef029e945106b2ebc5a6cb30f7b23e --- /dev/null +++ b/PaddleSlim/quant_low_level_api/dygraph_quant/run_quant.sh @@ -0,0 +1,36 @@ +export CUDA_VISIBLE_DEVICES=2 +export FLAGS_fraction_of_gpu_memory_to_use=0.8 +export FLAGS_cudnn_deterministic=1 + +data_dir='/work/datasets/ILSVRC2012/' +out_dir='output_0610' +epoch=10 +batch_size=128 +lr=0.0001 +is_fast_test=false + +#for model_name in mobilenet_v1 resnet50 +for model_name in mobilenet_v1 +do + if [ $is_fast_test = true ]; + then + echo "is_fast_test=true" + python -u quant_dygraph.py \ + --model_name=${model_name} \ + --data_path=${data_dir} \ + --output_dir=${out_dir} \ + --epoch=${epoch} \ + --batch_size=${batch_size} \ + --lr=${lr} \ + --action_fast_test + else + echo "is_fast_test=false" + python -u quant_dygraph.py \ + --model_name=${model_name} \ + --data_path=${data_dir} \ + --output_dir=${out_dir} \ + --epoch=${epoch} \ + --batch_size=${batch_size} \ + --lr=${lr} + fi +done diff --git a/PaddleSlim/quant_low_level_api/dygraph_quant/run_test.sh b/PaddleSlim/quant_low_level_api/dygraph_quant/run_test.sh new file mode 100644 index 0000000000000000000000000000000000000000..e28f87170aab455b45ddfdb120a2ffa2a909e0a4 --- /dev/null +++ b/PaddleSlim/quant_low_level_api/dygraph_quant/run_test.sh @@ -0,0 +1,16 @@ +export CUDA_VISIBLE_DEVICES=6 + +load_model_path=$1 +data_dir='/work/datasets/ILSVRC2012/' +eval_test_samples=-1 # if set as -1, eval all test samples + +echo "--------eval model: ${model_name}-------------" +python -u eval.py \ + --use_gpu=True \ + --class_dim=1000 \ + --image_shape=3,224,224 \ + --data_dir=${data_dir} \ + --test_samples=${eval_test_samples} \ + --inference_model=$load_model_path + +echo "\n\n" diff --git a/PaddleSlim/quant_low_level_api/dygraph_quant/utility.py b/PaddleSlim/quant_low_level_api/dygraph_quant/utility.py new file mode 100644 index 0000000000000000000000000000000000000000..439fa1c24c917ea8c5a3dc29934e4f7aca8c7d38 --- /dev/null +++ b/PaddleSlim/quant_low_level_api/dygraph_quant/utility.py @@ -0,0 +1,169 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains common utility functions.""" +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +import distutils.util +import os +import numpy as np +import six +import logging +import paddle.fluid as fluid +import paddle.compat as cpt +from paddle.fluid import core +from paddle.fluid.framework import Program + +logging.basicConfig(format='%(asctime)s-%(levelname)s: %(message)s') +_logger = logging.getLogger(__name__) +_logger.setLevel(logging.INFO) + + +def print_arguments(args): + """Print argparse's arguments. + + Usage: + + .. code-block:: python + + parser = argparse.ArgumentParser() + parser.add_argument("name", default="Jonh", type=str, help="User name.") + args = parser.parse_args() + print_arguments(args) + + :param args: Input argparse.Namespace for printing. + :type args: argparse.Namespace + """ + print("----------- Configuration Arguments -----------") + for arg, value in sorted(six.iteritems(vars(args))): + print("%s: %s" % (arg, value)) + print("------------------------------------------------") + + +def add_arguments(argname, type, default, help, argparser, **kwargs): + """Add argparse's argument. + + Usage: + + .. code-block:: python + + parser = argparse.ArgumentParser() + add_argument("name", str, "Jonh", "User name.", parser) + args = parser.parse_args() + """ + type = distutils.util.strtobool if type == bool else type + argparser.add_argument( + "--" + argname, + default=default, + type=type, + help=help + ' Default: %(default)s.', + **kwargs) + + +def save_persistable_nodes(executor, dirname, graph): + """ + Save persistable nodes to the given directory by the executor. + + Args: + executor(Executor): The executor to run for saving node values. + dirname(str): The directory path. + graph(IrGraph): All the required persistable nodes in the graph will be saved. + """ + persistable_node_names = set() + persistable_nodes = [] + all_persistable_nodes = graph.all_persistable_nodes() + for node in all_persistable_nodes: + name = cpt.to_text(node.name()) + if name not in persistable_node_names: + persistable_node_names.add(name) + persistable_nodes.append(node) + program = Program() + var_list = [] + for node in persistable_nodes: + var_desc = node.var() + if var_desc.type() == core.VarDesc.VarType.RAW or \ + var_desc.type() == core.VarDesc.VarType.READER: + continue + var = program.global_block().create_var( + name=var_desc.name(), + shape=var_desc.shape(), + dtype=var_desc.dtype(), + type=var_desc.type(), + lod_level=var_desc.lod_level(), + persistable=var_desc.persistable()) + var_list.append(var) + fluid.io.save_vars(executor=executor, dirname=dirname, vars=var_list) + + +def load_persistable_nodes(executor, dirname, graph): + """ + Load persistable node values from the given directory by the executor. + + Args: + executor(Executor): The executor to run for loading node values. + dirname(str): The directory path. + graph(IrGraph): All the required persistable nodes in the graph will be loaded. + """ + persistable_node_names = set() + persistable_nodes = [] + all_persistable_nodes = graph.all_persistable_nodes() + for node in all_persistable_nodes: + name = cpt.to_text(node.name()) + if name not in persistable_node_names: + persistable_node_names.add(name) + persistable_nodes.append(node) + program = Program() + var_list = [] + + def _exist(var): + return os.path.exists(os.path.join(dirname, var.name)) + + def _load_var(name, scope): + return np.array(scope.find_var(name).get_tensor()) + + def _store_var(name, array, scope, place): + tensor = scope.find_var(name).get_tensor() + tensor.set(array, place) + + for node in persistable_nodes: + var_desc = node.var() + if var_desc.type() == core.VarDesc.VarType.RAW or \ + var_desc.type() == core.VarDesc.VarType.READER: + continue + var = program.global_block().create_var( + name=var_desc.name(), + shape=var_desc.shape(), + dtype=var_desc.dtype(), + type=var_desc.type(), + lod_level=var_desc.lod_level(), + persistable=var_desc.persistable()) + if _exist(var): + var_list.append(var) + else: + _logger.info("Cannot find the var %s!!!" % (node.name())) + fluid.io.load_vars(executor=executor, dirname=dirname, vars=var_list)