提交 45ebafc3 编写于 作者: W Wang, Chuanqi 提交者: Tao Luo

Split test_calibration test to two tests (#16977)

* Split test_calibration test to two tests

test=develop

* Modify CMakeLists file
test=develop
上级 45352983
...@@ -2,11 +2,12 @@ file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py") ...@@ -2,11 +2,12 @@ file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py")
string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}") string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}")
if(APPLE OR WIN32 OR NOT WITH_MKL) if(APPLE OR WIN32 OR NOT WITH_MKL)
list(REMOVE_ITEM TEST_OPS test_calibration) list(REMOVE_ITEM TEST_OPS test_calibration_resnet50)
list(REMOVE_ITEM TEST_OPS test_calibration_mobilenetv1)
endif() endif()
foreach(src ${TEST_OPS}) foreach(src ${TEST_OPS})
if(src MATCHES "test_calibration") if(src MATCHES "test_calibration_*")
py_test(${src} SRCS ${src}.py ENVS FLAGS_use_mkldnn=true FLAGS_OMP_NUM_THREADS=${CPU_NUM_THREADS_ON_CI}) py_test(${src} SRCS ${src}.py ENVS FLAGS_use_mkldnn=true FLAGS_OMP_NUM_THREADS=${CPU_NUM_THREADS_ON_CI})
else() else()
py_test(${src} SRCS ${src}.py) py_test(${src} SRCS ${src}.py)
......
# copyright (c) 2018 paddlepaddle authors. all rights reserved.
#
# licensed under the apache license, version 2.0 (the "license");
# you may not use this file except in compliance with the license.
# you may obtain a copy of the license at
#
# http://www.apache.org/licenses/license-2.0
#
# unless required by applicable law or agreed to in writing, software
# distributed under the license is distributed on an "as is" basis,
# without warranties or conditions of any kind, either express or implied.
# see the license for the specific language governing permissions and
# limitations under the license.
import unittest
import sys
from test_calibration_resnet50 import TestCalibration
class TestCalibrationForMobilenetv1(TestCalibration):
def download_model(self):
# mobilenetv1 fp32 data
data_urls = [
'http://paddle-inference-dist.bj.bcebos.com/int8/mobilenetv1_int8_model.tar.gz'
]
data_md5s = ['13892b0716d26443a8cdea15b3c6438b']
self.model_cache_folder = self.download_data(data_urls, data_md5s,
"mobilenetv1_fp32")
self.model = "MobileNet-V1"
self.algo = "KL"
def test_calibration(self):
self.download_model()
print("Start FP32 inference for {0} on {1} images ...").format(
self.model, self.infer_iterations * self.batch_size)
(fp32_throughput, fp32_latency,
fp32_acc1) = self.run_program(self.model_cache_folder + "/model")
print("Start INT8 calibration for {0} on {1} images ...").format(
self.model, self.sample_iterations * self.batch_size)
self.run_program(
self.model_cache_folder + "/model", True, algo=self.algo)
print("Start INT8 inference for {0} on {1} images ...").format(
self.model, self.infer_iterations * self.batch_size)
(int8_throughput, int8_latency,
int8_acc1) = self.run_program("calibration_out")
delta_value = fp32_acc1 - int8_acc1
self.assertLess(delta_value, 0.01)
print(
"FP32 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}".
format(self.model, self.batch_size, fp32_throughput, fp32_latency,
fp32_acc1))
print(
"INT8 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}".
format(self.model, self.batch_size, int8_throughput, int8_latency,
int8_acc1))
sys.stdout.flush()
if __name__ == '__main__':
unittest.main()
...@@ -114,7 +114,7 @@ def val(data_dir=DATA_DIR): ...@@ -114,7 +114,7 @@ def val(data_dir=DATA_DIR):
return _reader_creator(file_list, 'val', shuffle=False, data_dir=data_dir) return _reader_creator(file_list, 'val', shuffle=False, data_dir=data_dir)
class TestCalibrationForResnet50(unittest.TestCase): class TestCalibration(unittest.TestCase):
def setUp(self): def setUp(self):
self.int8_download = 'int8/download' self.int8_download = 'int8/download'
self.cache_folder = os.path.expanduser('~/.cache/paddle/dataset/' + self.cache_folder = os.path.expanduser('~/.cache/paddle/dataset/' +
...@@ -188,15 +188,7 @@ class TestCalibrationForResnet50(unittest.TestCase): ...@@ -188,15 +188,7 @@ class TestCalibrationForResnet50(unittest.TestCase):
return data_cache_folder return data_cache_folder
def download_model(self): def download_model(self):
# resnet50 fp32 data pass
data_urls = [
'http://paddle-inference-dist.bj.bcebos.com/int8/resnet50_int8_model.tar.gz'
]
data_md5s = ['4a5194524823d9b76da6e738e1367881']
self.model_cache_folder = self.download_data(data_urls, data_md5s,
"resnet50_fp32")
self.model = "ResNet-50"
self.algo = "direct"
def run_program(self, model_path, generate_int8=False, algo='direct'): def run_program(self, model_path, generate_int8=False, algo='direct'):
image_shape = [3, 224, 224] image_shape = [3, 224, 224]
...@@ -277,6 +269,19 @@ class TestCalibrationForResnet50(unittest.TestCase): ...@@ -277,6 +269,19 @@ class TestCalibrationForResnet50(unittest.TestCase):
acc1 = np.sum(test_info) / cnt acc1 = np.sum(test_info) / cnt
return (throughput, latency, acc1) return (throughput, latency, acc1)
class TestCalibrationForResnet50(TestCalibration):
def download_model(self):
# resnet50 fp32 data
data_urls = [
'http://paddle-inference-dist.bj.bcebos.com/int8/resnet50_int8_model.tar.gz'
]
data_md5s = ['4a5194524823d9b76da6e738e1367881']
self.model_cache_folder = self.download_data(data_urls, data_md5s,
"resnet50_fp32")
self.model = "ResNet-50"
self.algo = "direct"
def test_calibration(self): def test_calibration(self):
self.download_model() self.download_model()
print("Start FP32 inference for {0} on {1} images ...").format( print("Start FP32 inference for {0} on {1} images ...").format(
...@@ -304,18 +309,5 @@ class TestCalibrationForResnet50(unittest.TestCase): ...@@ -304,18 +309,5 @@ class TestCalibrationForResnet50(unittest.TestCase):
sys.stdout.flush() sys.stdout.flush()
class TestCalibrationForMobilenetv1(TestCalibrationForResnet50):
def download_model(self):
# mobilenetv1 fp32 data
data_urls = [
'http://paddle-inference-dist.bj.bcebos.com/int8/mobilenetv1_int8_model.tar.gz'
]
data_md5s = ['13892b0716d26443a8cdea15b3c6438b']
self.model_cache_folder = self.download_data(data_urls, data_md5s,
"mobilenetv1_fp32")
self.model = "MobileNet-V1"
self.algo = "KL"
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册