# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserve. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import logging import platform from ..common import get_logger from .utils.predict import predict_compressed_model, with_variable_shape from .strategy_config import * _logger = get_logger(__name__, level=logging.INFO) __all__ = [ "prepare_strategy", "create_strategy_config", "get_final_quant_config" ] # config tester to test the loss of quant_post hpo_config_tester = { "ptq_algo": ["avg", "mse", "KL"], "weight_quantize_type": ['channel_wise_abs_max', 'abs_max'], "bias_correct": [False], "batch_num": [5], "max_quant_count": 1, } # default hpo config default_hpo_config = { "ptq_algo": ["KL", "hist", "avg", "mse"], "weight_quantize_type": ['channel_wise_abs_max', 'abs_max'], "bias_correct": [True, False], "hist_percent": [0.98, 0.999], "batch_num": [10, 30], "max_quant_count": 20, } # default quant config, can be used by ptq&hpo and qat&distillation default_quant_config = { 'quantize_op_types': ['conv2d', 'depthwise_conv2d', 'mul', 'matmul'], 'weight_bits': 8, 'activation_bits': 8, "is_full_quantize": False, "activation_quantize_type": 'range_abs_max', "weight_quantize_type": 'abs_max', "not_quant_pattern": ["skip_quant"], } # default train config DefaultTrainConfig = { "epochs": 1, "eval_iter": 500, "learning_rate": 0.0001, "optimizer": "Momentum", "optim_args": { "weight_decay": 4.0e-05 }, } EXPERIENCE_STRATEGY_WITHOUT_LOSS = [ 'sparse_0.75_fp32', 'prune_0.3_fp32', 'origin_int8', 'sparse_0.75_int8', 'prune_0.3_int8' ] MAGIC_SPARSE_RATIO = 0.75 ### TODO: 0.02 threshold maybe not suitable, need to check MAGIC_MAX_EMD_DISTANCE = 0.02 MAGIC_MIN_EMD_DISTANCE = 0.01 DEFAULT_TRANSFORMER_STRATEGY = 'prune_0.25_int8' DEFAULT_STRATEGY = 'origin_int8' DEFAULT_QUANT_SPEEDUP = 0.7 def create_strategy_config(strategy_str, model_type): """ create config according to string""" tmp_s = strategy_str.split('_') configs = [] dis_config = Distillation() if len(tmp_s) == 3: ### TODO(ceci3): choose prune algo automatically if 'prune' in tmp_s[0]: ### default prune config default_prune_config = { 'pruned_ratio': float(tmp_s[1]), 'prune_algo': 'prune', 'criterion': 'l1_norm' } else: ### default unstruture prune config default_prune_config = { 'prune_strategy': 'gmp', ### default unstruture prune strategy is gmp 'prune_mode': 'ratio', 'pruned_ratio': float(tmp_s[1]), 'local_sparsity': True, 'prune_params_type': 'conv1x1_only' } tmp_s[0] = tmp_s[0].replace('prune', 'Prune') tmp_s[0] = tmp_s[0].replace('sparse', 'UnstructurePrune') if model_type == 'transformer' and tmp_s[0] == 'Prune': default_prune_config['prune_algo'] = 'transformer_pruner' prune_config = eval(tmp_s[0])(**default_prune_config) configs.append({tmp_s[0]: prune_config, 'Distillation': dis_config}) ### TODO(ceci3): support skip some layer and full quant if tmp_s[-1] == 'int8': ### only platform is linux can use smac to do hyperparameter optimization ### choose quant_aware to do quantization in other platform if platform.system().lower() == 'linux': quant_config = Quantization(**default_quant_config) hpo_config = HyperParameterOptimization(**hpo_config_tester) configs.append({ 'Quantization': quant_config, 'HyperParameterOptimization': hpo_config }) else: quant_config = Quantization(**default_quant_config) dis_config = Distillation() configs.append({ 'Quantization': quant_config, 'Distillation': dis_config }) return configs def create_train_config(strategy_str, model_type): # TDOD: support more strategy and model_type train_config = TrainConfig(**DefaultTrainConfig) return train_config def prepare_strategy(executor, places, model_dir, model_filename, params_filename, target_speedup=None, deploy_hardware=None, model_type=None): """ prepare compression config automatically """ final_strategy = None if with_variable_shape( model_dir, model_filename=model_filename, params_filename=params_filename): deploy_hardware = None _logger.warning( "The model's inputs have variable shape. " "And the latency predictor doesn't support variable shape. " "So auto tuning will be skipped and a default strategy will be chosen." ) ### use hardware latency tabel if support if deploy_hardware is not None: compressed_time_dict = predict_compressed_model( executor, places, model_dir, model_filename, params_filename, hardware=deploy_hardware) baseline = compressed_time_dict['origin_fp32'] speedup_ratio = {} for strategy, latency in compressed_time_dict.items(): speedup_ratio[strategy] = 1.0 - float(latency) / baseline sorted_speedup_ratio = sorted(speedup_ratio.items(), key=lambda x: x[1]) ### if target speedup is None, choose strategy by experience. if target_speedup is None: max_speedup = -1.0 for s in EXPERIENCE_STRATEGY_WITHOUT_LOSS: if s not in speedup_ratio: _logger.info(f"cannot get the speed up of strategy {s}") continue if speedup_ratio[s] > max_speedup: max_speedup = speedup_ratio[s] final_strategy = s else: candidate_s = [] pre_s = None for strategy, ratio in sorted_speedup_ratio: if abs(ratio - target_speedup) <= 0.1: candidate_s.append(strategy) ### if there is no strategy satisfy target speedup ### choose the most recent speedup if ratio > target_speedup and len(candidate_s) == 0: if pre_s is not None: candidate_s.append(pre_s) candidate_s.append(strategy) pre_s = strategy if 'origin_int8' in candidate_s: final_strategy = candidate_s else: candidate_s = sorted(candidate_s, key=lambda x: x.split('_')[1]) for c in candidate_s: if c.startswith('sparse') and float(c.split('_')[ 1]) <= MAGIC_SPARSE_RATIO: final_strategy = c if final_strategy is None: final_strategy = candidate_s[0] ### if deploy_hardware is not None else: ### default speedup ratio of quantization is 70% compare to fp32 ### TODO(ceci3): full quant or skip some layer later if target_speedup is None: if model_type == 'transformer': final_strategy = DEFAULT_TRANSFORMER_STRATEGY else: final_strategy = DEFAULT_STRATEGY elif target_speedup > DEFAULT_QUANT_SPEEDUP: prune_ratio = target_speedup - DEFAULT_QUANT_SPEEDUP if prune_ratio > 1.0: raise NotImplementedError( "target_speedup {} is improper".format(target_speedup)) final_strategy = 'prune_{}_int8'.format(str(prune_ratio)) else: raise NotImplementedError("target_speedup {} is improper".format( target_speedup)) strategy_config = create_strategy_config(final_strategy, model_type) return strategy_config def get_final_quant_config(ptq_loss): """ transform quantization tester config to real quantization config """ ### if emd loss less than MAGIC_MIN_EMD_DISTANCE, final compress. if ptq_loss < MAGIC_MIN_EMD_DISTANCE: return None ### if emd loss less than MAGIC_MAX_EMD_DISTANCE, select quant_post & hpo. elif ptq_loss < MAGIC_MAX_EMD_DISTANCE: quant_config = Quantization(**default_quant_config) hpo_config = HyperParameterOptimization(**default_hpo_config) configs = [{ 'Quantization': quant_config, 'HyperParameterOptimization': hpo_config }] ### if emd loss greater than MAGIC_MAX_EMD_DISTANCE, select qat & dist. else: quant_config = Quantization(**default_quant_config) dis_config = Distillation() configs = [{'Quantization': quant_config, 'Distillation': dis_config}] _logger.info("Start Quantization and Distillation Training.") return configs if __name__ == '__main__': create_strategy_config('sparse_0.75_int8', 'transformer')