From 294d594450c9168995e1cc27caf86dddf98993f3 Mon Sep 17 00:00:00 2001 From: Haihao Shen Date: Wed, 30 Jan 2019 14:20:22 +0800 Subject: [PATCH] Enable performance measurement in INT8 calibration unit test (#15560) * Enable performance measurement in INT8 calibration unit test --- .../fluid/contrib/tests/test_calibration.py | 144 +++++++++++++----- 1 file changed, 106 insertions(+), 38 deletions(-) diff --git a/python/paddle/fluid/contrib/tests/test_calibration.py b/python/paddle/fluid/contrib/tests/test_calibration.py index f07fefe7e09..cd6b7ba1661 100644 --- a/python/paddle/fluid/contrib/tests/test_calibration.py +++ b/python/paddle/fluid/contrib/tests/test_calibration.py @@ -19,10 +19,8 @@ import sys import random import paddle import paddle.fluid as fluid -import argparse import functools import contextlib -import paddle.fluid.profiler as profiler from paddle.dataset.common import download from PIL import Image, ImageEnhance import math @@ -43,7 +41,7 @@ img_mean = np.array([0.485, 0.456, 0.406]).reshape((3, 1, 1)) img_std = np.array([0.229, 0.224, 0.225]).reshape((3, 1, 1)) -# TODO(guomingz): Remove duplicated code from line 45 ~ line 114 +# TODO(guomingz): Remove duplicated code from resize_short, crop_image, process_image, _reader_creator def resize_short(img, target_size): percent = float(target_size) / min(img.size[0], img.size[1]) resized_width = int(round(img.size[0] * percent)) @@ -123,16 +121,37 @@ class TestCalibrationForResnet50(unittest.TestCase): self.cache_folder = os.path.expanduser('~/.cache/paddle/dataset/' + self.int8_download) - data_url = 'http://paddle-inference-dist.cdn.bcebos.com/int8/calibration_test_data.tar.gz' - data_md5 = '1b6c1c434172cca1bf9ba1e4d7a3157d' - self.data_cache_folder = self.download_data(data_url, data_md5, "data") + data_urls = [] + data_md5s = [] + self.data_cache_folder = '' + if os.environ.get('DATASET') == 'full': + data_urls.append( + 'https://paddle-inference-dist.bj.bcebos.com/int8/ILSVRC2012_img_val.tar.gz.partaa' + ) + data_md5s.append('60f6525b0e1d127f345641d75d41f0a8') + data_urls.append( + 'https://paddle-inference-dist.bj.bcebos.com/int8/ILSVRC2012_img_val.tar.gz.partab' + ) + data_md5s.append('1e9f15f64e015e58d6f9ec3210ed18b5') + self.data_cache_folder = self.download_data(data_urls, data_md5s, + "full_data", False) + else: + data_urls.append( + 'http://paddle-inference-dist.cdn.bcebos.com/int8/calibration_test_data.tar.gz' + ) + data_md5s.append('1b6c1c434172cca1bf9ba1e4d7a3157d') + self.data_cache_folder = self.download_data(data_urls, data_md5s, + "small_data", False) # reader/decorator.py requires the relative path to the data folder cmd = 'rm -rf {0} && ln -s {1} {0}'.format("data", self.data_cache_folder) os.system(cmd) - self.iterations = 50 + self.batch_size = 1 + self.sample_iterations = 50 + self.infer_iterations = 50000 if os.environ.get( + 'DATASET') == 'full' else 50 def cache_unzipping(self, target_folder, zip_path): if not os.path.exists(target_folder): @@ -140,20 +159,44 @@ class TestCalibrationForResnet50(unittest.TestCase): zip_path) os.system(cmd) - def download_data(self, data_url, data_md5, folder_name): - download(data_url, self.int8_download, data_md5) + def download_data(self, data_urls, data_md5s, folder_name, is_model=True): data_cache_folder = os.path.join(self.cache_folder, folder_name) - file_name = data_url.split('/')[-1] - zip_path = os.path.join(self.cache_folder, file_name) + zip_path = '' + if os.environ.get('DATASET') == 'full': + file_names = [] + for i in range(0, len(data_urls)): + download(data_urls[i], self.int8_download, data_md5s[i]) + file_names.append(data_urls[i].split('/')[-1]) + + zip_path = os.path.join(self.cache_folder, + 'full_imagenet_val.tar.gz') + if not os.path.exists(zip_path): + cat_command = 'cat' + for file_name in file_names: + cat_command += ' ' + os.path.join(self.cache_folder, + file_name) + cat_command += ' > ' + zip_path + os.system(cat_command) + + if os.environ.get('DATASET') != 'full' or is_model: + download(data_urls[0], self.int8_download, data_md5s[0]) + file_name = data_urls[0].split('/')[-1] + zip_path = os.path.join(self.cache_folder, file_name) + + print('Data is downloaded at {0}').format(zip_path) self.cache_unzipping(data_cache_folder, zip_path) return data_cache_folder - def download_resnet50_model(self): + def download_model(self): # resnet50 fp32 data - data_url = 'http://paddle-inference-dist.cdn.bcebos.com/int8/resnet50_int8_model.tar.gz' - data_md5 = '4a5194524823d9b76da6e738e1367881' - self.model_cache_folder = self.download_data(data_url, data_md5, + data_urls = [ + 'http://paddle-inference-dist.cdn.bcebos.com/int8/resnet50_int8_model.tar.gz' + ] + data_md5s = ['4a5194524823d9b76da6e738e1367881'] + self.model_cache_folder = self.download_data(data_urls, data_md5s, "resnet50_fp32") + self.model = "ResNet-50" + self.algo = "direct" def run_program(self, model_path, generate_int8=False, algo='direct'): image_shape = [3, 224, 224] @@ -169,17 +212,17 @@ class TestCalibrationForResnet50(unittest.TestCase): t = fluid.transpiler.InferenceTranspiler() t.transpile(infer_program, fluid.CPUPlace()) - val_reader = paddle.batch(val(), batch_size=1) + val_reader = paddle.batch(val(), self.batch_size) + iterations = self.infer_iterations if generate_int8: int8_model = os.path.join(os.getcwd(), "calibration_out") + iterations = self.sample_iterations if os.path.exists(int8_model): os.system("rm -rf " + int8_model) os.system("mkdir " + int8_model) - print("Start calibration ...") - calibrator = int8_utility.Calibrator( program=infer_program, pretrained_model=model_path, @@ -191,6 +234,7 @@ class TestCalibrationForResnet50(unittest.TestCase): test_info = [] cnt = 0 + periods = [] for batch_id, data in enumerate(val_reader()): image = np.array( [x[0].reshape(image_shape) for x in data]).astype("float32") @@ -202,21 +246,28 @@ class TestCalibrationForResnet50(unittest.TestCase): if op.has_attr("use_mkldnn"): op._set_attr("use_mkldnn", True) + t1 = time.time() _, acc1, _ = exe.run( running_program, feed={feed_dict[0]: image, feed_dict[1]: label}, fetch_list=fetch_targets) + t2 = time.time() + period = t2 - t1 + periods.append(period) + if generate_int8: calibrator.sample_data() test_info.append(np.mean(acc1) * len(data)) cnt += len(data) - if batch_id != self.iterations - 1: - continue + if (batch_id + 1) % 100 == 0: + print("{0} images,".format(batch_id + 1)) + sys.stdout.flush() - break + if (batch_id + 1) == iterations: + break if generate_int8: calibrator.save_int8_model() @@ -225,32 +276,49 @@ class TestCalibrationForResnet50(unittest.TestCase): "Calibration is done and the corresponding files are generated at {}". format(os.path.abspath("calibration_out"))) else: - return np.sum(test_info) / cnt + throughput = cnt / np.sum(periods) + latency = np.average(periods) + acc1 = np.sum(test_info) / cnt + return (throughput, latency, acc1) def test_calibration(self): - self.download_resnet50_model() - fp32_acc1 = self.run_program(self.model_cache_folder + "/model") - self.run_program(self.model_cache_folder + "/model", True) - int8_acc1 = self.run_program("calibration_out") + self.download_model() + print("Start FP32 inference for {0} on {1} images ...").format( + self.model, self.infer_iterations) + (fp32_throughput, fp32_latency, + fp32_acc1) = self.run_program(self.model_cache_folder + "/model") + print("Start INT8 calibration for {0} on {1} images ...").format( + self.model, self.sample_iterations) + self.run_program( + self.model_cache_folder + "/model", True, algo=self.algo) + print("Start INT8 inference for {0} on {1} images ...").format( + self.model, self.infer_iterations) + (int8_throughput, int8_latency, + int8_acc1) = self.run_program("calibration_out") delta_value = np.abs(fp32_acc1 - int8_acc1) self.assertLess(delta_value, 0.01) + print( + "FP32 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}". + format(self.model, self.batch_size, fp32_throughput, fp32_latency, + fp32_acc1)) + print( + "INT8 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}". + format(self.model, self.batch_size, int8_throughput, int8_latency, + int8_acc1)) + sys.stdout.flush() class TestCalibrationForMobilenetv1(TestCalibrationForResnet50): - def download_mobilenetv1_model(self): + def download_model(self): # mobilenetv1 fp32 data - data_url = 'http://paddle-inference-dist.cdn.bcebos.com/int8/mobilenetv1_int8_model.tar.gz' - data_md5 = '13892b0716d26443a8cdea15b3c6438b' - self.model_cache_folder = self.download_data(data_url, data_md5, + data_urls = [ + 'http://paddle-inference-dist.cdn.bcebos.com/int8/mobilenetv1_int8_model.tar.gz' + ] + data_md5s = ['13892b0716d26443a8cdea15b3c6438b'] + self.model_cache_folder = self.download_data(data_urls, data_md5s, "mobilenetv1_fp32") - - def test_calibration(self): - self.download_mobilenetv1_model() - fp32_acc1 = self.run_program(self.model_cache_folder + "/model") - self.run_program(self.model_cache_folder + "/model", True, algo='KL') - int8_acc1 = self.run_program("calibration_out") - delta_value = np.abs(fp32_acc1 - int8_acc1) - self.assertLess(delta_value, 0.01) + self.model = "MobileNet-V1" + self.algo = "KL" if __name__ == '__main__': -- GitLab