Split test_calibration test to two tests (#16977)

* Split test_calibration test to two tests

test=develop

* Modify CMakeLists file
test=develop

python/paddle/fluid/contrib/tests/CMakeLists.txt

@@ -2,11 +2,12 @@ file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py")
 string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}")
 
 if(APPLE OR WIN32 OR NOT WITH_MKL)
-    list(REMOVE_ITEM TEST_OPS test_calibration)
+    list(REMOVE_ITEM TEST_OPS test_calibration_resnet50)
+    list(REMOVE_ITEM TEST_OPS test_calibration_mobilenetv1)
 endif()
 
 foreach(src ${TEST_OPS})
-    if(src MATCHES "test_calibration")
+    if(src MATCHES "test_calibration_*")
         py_test(${src} SRCS ${src}.py ENVS FLAGS_use_mkldnn=true FLAGS_OMP_NUM_THREADS=${CPU_NUM_THREADS_ON_CI})
     else()
         py_test(${src} SRCS ${src}.py)

python/paddle/fluid/contrib/tests/test_calibration_mobilenetv1.py

+#   copyright (c) 2018 paddlepaddle authors. all rights reserved.
+#
+# licensed under the apache license, version 2.0 (the "license");
+# you may not use this file except in compliance with the license.
+# you may obtain a copy of the license at
+#
+#     http://www.apache.org/licenses/license-2.0
+#
+# unless required by applicable law or agreed to in writing, software
+# distributed under the license is distributed on an "as is" basis,
+# without warranties or conditions of any kind, either express or implied.
+# see the license for the specific language governing permissions and
+# limitations under the license.
+import unittest
+import sys
+from test_calibration_resnet50 import TestCalibration
+
+
+class TestCalibrationForMobilenetv1(TestCalibration):
+    def download_model(self):
+        # mobilenetv1 fp32 data
+        data_urls = [
+            'http://paddle-inference-dist.bj.bcebos.com/int8/mobilenetv1_int8_model.tar.gz'
+        ]
+        data_md5s = ['13892b0716d26443a8cdea15b3c6438b']
+        self.model_cache_folder = self.download_data(data_urls, data_md5s,
+                                                     "mobilenetv1_fp32")
+        self.model = "MobileNet-V1"
+        self.algo = "KL"
+
+    def test_calibration(self):
+        self.download_model()
+        print("Start FP32 inference for {0} on {1} images ...").format(
+            self.model, self.infer_iterations * self.batch_size)
+        (fp32_throughput, fp32_latency,
+         fp32_acc1) = self.run_program(self.model_cache_folder + "/model")
+        print("Start INT8 calibration for {0} on {1} images ...").format(
+            self.model, self.sample_iterations * self.batch_size)
+        self.run_program(
+            self.model_cache_folder + "/model", True, algo=self.algo)
+        print("Start INT8 inference for {0} on {1} images ...").format(
+            self.model, self.infer_iterations * self.batch_size)
+        (int8_throughput, int8_latency,
+         int8_acc1) = self.run_program("calibration_out")
+        delta_value = fp32_acc1 - int8_acc1
+        self.assertLess(delta_value, 0.01)
+        print(
+            "FP32 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}".
+            format(self.model, self.batch_size, fp32_throughput, fp32_latency,
+                   fp32_acc1))
+        print(
+            "INT8 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}".
+            format(self.model, self.batch_size, int8_throughput, int8_latency,
+                   int8_acc1))
+        sys.stdout.flush()
+
+
+if __name__ == '__main__':
+    unittest.main()

python/paddle/fluid/contrib/tests/test_calibration.py → python/paddle/fluid/contrib/tests/test_calibration_resnet50.py

@@ -114,7 +114,7 @@ def val(data_dir=DATA_DIR):
     return _reader_creator(file_list, 'val', shuffle=False, data_dir=data_dir)
 
 
-class TestCalibrationForResnet50(unittest.TestCase):
+class TestCalibration(unittest.TestCase):
     def setUp(self):
         self.int8_download = 'int8/download'
         self.cache_folder = os.path.expanduser('~/.cache/paddle/dataset/' +
@@ -188,15 +188,7 @@ class TestCalibrationForResnet50(unittest.TestCase):
         return data_cache_folder
 
     def download_model(self):
-        # resnet50 fp32 data
-        data_urls = [
-            'http://paddle-inference-dist.bj.bcebos.com/int8/resnet50_int8_model.tar.gz'
-        ]
-        data_md5s = ['4a5194524823d9b76da6e738e1367881']
-        self.model_cache_folder = self.download_data(data_urls, data_md5s,
-                                                     "resnet50_fp32")
-        self.model = "ResNet-50"
-        self.algo = "direct"
+        pass
 
     def run_program(self, model_path, generate_int8=False, algo='direct'):
         image_shape = [3, 224, 224]
@@ -277,6 +269,19 @@ class TestCalibrationForResnet50(unittest.TestCase):
             acc1 = np.sum(test_info) / cnt
             return (throughput, latency, acc1)
 
+
+class TestCalibrationForResnet50(TestCalibration):
+    def download_model(self):
+        # resnet50 fp32 data
+        data_urls = [
+            'http://paddle-inference-dist.bj.bcebos.com/int8/resnet50_int8_model.tar.gz'
+        ]
+        data_md5s = ['4a5194524823d9b76da6e738e1367881']
+        self.model_cache_folder = self.download_data(data_urls, data_md5s,
+                                                     "resnet50_fp32")
+        self.model = "ResNet-50"
+        self.algo = "direct"
+
     def test_calibration(self):
         self.download_model()
         print("Start FP32 inference for {0} on {1} images ...").format(
@@ -304,18 +309,5 @@ class TestCalibrationForResnet50(unittest.TestCase):
         sys.stdout.flush()
 
 
-class TestCalibrationForMobilenetv1(TestCalibrationForResnet50):
-    def download_model(self):
-        # mobilenetv1 fp32 data
-        data_urls = [
-            'http://paddle-inference-dist.bj.bcebos.com/int8/mobilenetv1_int8_model.tar.gz'
-        ]
-        data_md5s = ['13892b0716d26443a8cdea15b3c6438b']
-        self.model_cache_folder = self.download_data(data_urls, data_md5s,
-                                                     "mobilenetv1_fp32")
-        self.model = "MobileNet-V1"
-        self.algo = "KL"
-
-
 if __name__ == '__main__':
     unittest.main()