From 45ebafc33e81156970f94dd25e31a4dd5745234c Mon Sep 17 00:00:00 2001 From: "Wang, Chuanqi" Date: Sat, 20 Apr 2019 23:11:00 +0800 Subject: [PATCH] Split test_calibration test to two tests (#16977) * Split test_calibration test to two tests test=develop * Modify CMakeLists file test=develop --- .../paddle/fluid/contrib/tests/CMakeLists.txt | 5 +- .../tests/test_calibration_mobilenetv1.py | 59 +++++++++++++++++++ ...ration.py => test_calibration_resnet50.py} | 38 +++++------- 3 files changed, 77 insertions(+), 25 deletions(-) create mode 100644 python/paddle/fluid/contrib/tests/test_calibration_mobilenetv1.py rename python/paddle/fluid/contrib/tests/{test_calibration.py => test_calibration_resnet50.py} (95%) diff --git a/python/paddle/fluid/contrib/tests/CMakeLists.txt b/python/paddle/fluid/contrib/tests/CMakeLists.txt index 73e9ff10f..b538e38ab 100644 --- a/python/paddle/fluid/contrib/tests/CMakeLists.txt +++ b/python/paddle/fluid/contrib/tests/CMakeLists.txt @@ -2,11 +2,12 @@ file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py") string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}") if(APPLE OR WIN32 OR NOT WITH_MKL) - list(REMOVE_ITEM TEST_OPS test_calibration) + list(REMOVE_ITEM TEST_OPS test_calibration_resnet50) + list(REMOVE_ITEM TEST_OPS test_calibration_mobilenetv1) endif() foreach(src ${TEST_OPS}) - if(src MATCHES "test_calibration") + if(src MATCHES "test_calibration_*") py_test(${src} SRCS ${src}.py ENVS FLAGS_use_mkldnn=true FLAGS_OMP_NUM_THREADS=${CPU_NUM_THREADS_ON_CI}) else() py_test(${src} SRCS ${src}.py) diff --git a/python/paddle/fluid/contrib/tests/test_calibration_mobilenetv1.py b/python/paddle/fluid/contrib/tests/test_calibration_mobilenetv1.py new file mode 100644 index 000000000..2efcc1cfa --- /dev/null +++ b/python/paddle/fluid/contrib/tests/test_calibration_mobilenetv1.py @@ -0,0 +1,59 @@ +# copyright (c) 2018 paddlepaddle authors. all rights reserved. +# +# licensed under the apache license, version 2.0 (the "license"); +# you may not use this file except in compliance with the license. +# you may obtain a copy of the license at +# +# http://www.apache.org/licenses/license-2.0 +# +# unless required by applicable law or agreed to in writing, software +# distributed under the license is distributed on an "as is" basis, +# without warranties or conditions of any kind, either express or implied. +# see the license for the specific language governing permissions and +# limitations under the license. +import unittest +import sys +from test_calibration_resnet50 import TestCalibration + + +class TestCalibrationForMobilenetv1(TestCalibration): + def download_model(self): + # mobilenetv1 fp32 data + data_urls = [ + 'http://paddle-inference-dist.bj.bcebos.com/int8/mobilenetv1_int8_model.tar.gz' + ] + data_md5s = ['13892b0716d26443a8cdea15b3c6438b'] + self.model_cache_folder = self.download_data(data_urls, data_md5s, + "mobilenetv1_fp32") + self.model = "MobileNet-V1" + self.algo = "KL" + + def test_calibration(self): + self.download_model() + print("Start FP32 inference for {0} on {1} images ...").format( + self.model, self.infer_iterations * self.batch_size) + (fp32_throughput, fp32_latency, + fp32_acc1) = self.run_program(self.model_cache_folder + "/model") + print("Start INT8 calibration for {0} on {1} images ...").format( + self.model, self.sample_iterations * self.batch_size) + self.run_program( + self.model_cache_folder + "/model", True, algo=self.algo) + print("Start INT8 inference for {0} on {1} images ...").format( + self.model, self.infer_iterations * self.batch_size) + (int8_throughput, int8_latency, + int8_acc1) = self.run_program("calibration_out") + delta_value = fp32_acc1 - int8_acc1 + self.assertLess(delta_value, 0.01) + print( + "FP32 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}". + format(self.model, self.batch_size, fp32_throughput, fp32_latency, + fp32_acc1)) + print( + "INT8 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}". + format(self.model, self.batch_size, int8_throughput, int8_latency, + int8_acc1)) + sys.stdout.flush() + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/fluid/contrib/tests/test_calibration.py b/python/paddle/fluid/contrib/tests/test_calibration_resnet50.py similarity index 95% rename from python/paddle/fluid/contrib/tests/test_calibration.py rename to python/paddle/fluid/contrib/tests/test_calibration_resnet50.py index 16de60440..83ed2d4f8 100644 --- a/python/paddle/fluid/contrib/tests/test_calibration.py +++ b/python/paddle/fluid/contrib/tests/test_calibration_resnet50.py @@ -114,7 +114,7 @@ def val(data_dir=DATA_DIR): return _reader_creator(file_list, 'val', shuffle=False, data_dir=data_dir) -class TestCalibrationForResnet50(unittest.TestCase): +class TestCalibration(unittest.TestCase): def setUp(self): self.int8_download = 'int8/download' self.cache_folder = os.path.expanduser('~/.cache/paddle/dataset/' + @@ -188,15 +188,7 @@ class TestCalibrationForResnet50(unittest.TestCase): return data_cache_folder def download_model(self): - # resnet50 fp32 data - data_urls = [ - 'http://paddle-inference-dist.bj.bcebos.com/int8/resnet50_int8_model.tar.gz' - ] - data_md5s = ['4a5194524823d9b76da6e738e1367881'] - self.model_cache_folder = self.download_data(data_urls, data_md5s, - "resnet50_fp32") - self.model = "ResNet-50" - self.algo = "direct" + pass def run_program(self, model_path, generate_int8=False, algo='direct'): image_shape = [3, 224, 224] @@ -277,6 +269,19 @@ class TestCalibrationForResnet50(unittest.TestCase): acc1 = np.sum(test_info) / cnt return (throughput, latency, acc1) + +class TestCalibrationForResnet50(TestCalibration): + def download_model(self): + # resnet50 fp32 data + data_urls = [ + 'http://paddle-inference-dist.bj.bcebos.com/int8/resnet50_int8_model.tar.gz' + ] + data_md5s = ['4a5194524823d9b76da6e738e1367881'] + self.model_cache_folder = self.download_data(data_urls, data_md5s, + "resnet50_fp32") + self.model = "ResNet-50" + self.algo = "direct" + def test_calibration(self): self.download_model() print("Start FP32 inference for {0} on {1} images ...").format( @@ -304,18 +309,5 @@ class TestCalibrationForResnet50(unittest.TestCase): sys.stdout.flush() -class TestCalibrationForMobilenetv1(TestCalibrationForResnet50): - def download_model(self): - # mobilenetv1 fp32 data - data_urls = [ - 'http://paddle-inference-dist.bj.bcebos.com/int8/mobilenetv1_int8_model.tar.gz' - ] - data_md5s = ['13892b0716d26443a8cdea15b3c6438b'] - self.model_cache_folder = self.download_data(data_urls, data_md5s, - "mobilenetv1_fp32") - self.model = "MobileNet-V1" - self.algo = "KL" - - if __name__ == '__main__': unittest.main() -- GitLab