CMakeLists.txt 13.8 KB
Newer Older
W
WangZhen 已提交
1 2 3
file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py")
string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}")

4
function(_inference_analysis_python_api_int8_test target model_dir data_path filename use_mkldnn)
5
    py_test(${target} SRCS ${filename}
6
        ENVS CPU_NUM_THREADS=${CPU_NUM_THREADS_ON_CI}
7
             FLAGS_use_mkldnn=${use_mkldnn}
8
        ARGS --infer_model ${model_dir}/model
9
             --infer_data ${data_path}
10
             --int8_model_save_path int8_models/${target}
11
             --warmup_batch_size ${WARMUP_BATCH_SIZE}
12 13 14
             --batch_size 50)
endfunction()

15 16
function(inference_analysis_python_api_int8_test target model_dir data_path filename)
    _inference_analysis_python_api_int8_test(${target} ${model_dir} ${data_path} ${filename} False)
17 18
endfunction()

19 20 21 22 23
function(inference_analysis_python_api_int8_test_custom_warmup_batch_size target model_dir data_dir filename warmup_batch_size)
    set(WARMUP_BATCH_SIZE ${warmup_batch_size})
    inference_analysis_python_api_int8_test(${target} ${model_dir} ${data_dir} ${filename})
endfunction()

24 25 26 27
function(inference_analysis_python_api_int8_test_mkldnn target model_dir data_path filename)
    _inference_analysis_python_api_int8_test(${target} ${model_dir} ${data_path} ${filename} True)
endfunction()

W
Wojciech Uss 已提交
28
function(download_quant_data install_dir data_file)
29 30 31 32 33
    if (NOT EXISTS ${install_dir}/${data_file})
	    inference_download_and_uncompress(${install_dir} ${INFERENCE_URL}/int8 ${data_file})
    endif()
endfunction()

W
Wojciech Uss 已提交
34
function(download_quant_model install_dir data_file)
35 36 37 38 39
    if (NOT EXISTS ${install_dir}/${data_file})
	    inference_download_and_uncompress(${install_dir} ${INFERENCE_URL}/int8/QAT_models ${data_file})
    endif()
endfunction()

W
Wojciech Uss 已提交
40
function(download_quant_fp32_model install_dir data_file)
41 42 43 44 45
    if (NOT EXISTS ${install_dir}/${data_file})
	    inference_download_and_uncompress(${install_dir} ${INFERENCE_URL}/int8/QAT_models/fp32 ${data_file})
    endif()
endfunction()

W
Wojciech Uss 已提交
46
function(inference_quant_int8_image_classification_test target quant_model_dir dataset_path)
47
    py_test(${target} SRCS "${CMAKE_CURRENT_SOURCE_DIR}/quant_int8_image_classification_comparison.py"
48 49
            ENVS FLAGS_OMP_NUM_THREADS=${CPU_NUM_THREADS_ON_CI}
                 OMP_NUM_THREADS=${CPU_NUM_THREADS_ON_CI}
50
                 FLAGS_use_mkldnn=true
W
Wojciech Uss 已提交
51
            ARGS --quant_model ${quant_model_dir}
52
                 --infer_data ${dataset_path}
53 54 55 56 57
                 --batch_size 25
                 --batch_num 2
                 --acc_diff_threshold 0.1)
endfunction()

58 59

# set batch_size 10 for UT only (avoid OOM). For whole dataset, use batch_size 25 
W
Wojciech Uss 已提交
60
function(inference_quant2_int8_image_classification_test target quant_model_dir fp32_model_dir dataset_path)
61
    py_test(${target} SRCS "${CMAKE_CURRENT_SOURCE_DIR}/quant2_int8_image_classification_comparison.py"
62 63
            ENVS FLAGS_OMP_NUM_THREADS=${CPU_NUM_THREADS_ON_CI}
                 OMP_NUM_THREADS=${CPU_NUM_THREADS_ON_CI}
64
                 FLAGS_use_mkldnn=true
W
Wojciech Uss 已提交
65
            ARGS --quant_model ${quant_model_dir}
66 67
                 --fp32_model ${fp32_model_dir}
                 --infer_data ${dataset_path}
68
                 --batch_size 10
69
                 --batch_num 2
W
Wojciech Uss 已提交
70
                 --acc_diff_threshold 0.1)
71 72 73
endfunction()

# set batch_size 10 for UT only (avoid OOM). For whole dataset, use batch_size 20 
W
Wojciech Uss 已提交
74
function(inference_quant2_int8_nlp_test target quant_model_dir fp32_model_dir dataset_path labels_path ops_to_quantize)
75
    py_test(${target} SRCS "${CMAKE_CURRENT_SOURCE_DIR}/quant2_int8_nlp_comparison.py"
76 77 78
            ENVS FLAGS_OMP_NUM_THREADS=${CPU_NUM_THREADS_ON_CI}
                 OMP_NUM_THREADS=${CPU_NUM_THREADS_ON_CI}
                 FLAGS_use_mkldnn=true
W
Wojciech Uss 已提交
79
            ARGS --quant_model ${quant_model_dir}
80 81
		 --fp32_model ${fp32_model_dir}
                 --infer_data ${dataset_path}
82 83 84
		 --labels ${labels_path}
                 --batch_size 10
                 --batch_num 2
W
Wojciech Uss 已提交
85 86
                 --acc_diff_threshold 0.1
		 --ops_to_quantize ${ops_to_quantize})
87 88
endfunction()

W
Wojciech Uss 已提交
89
function(download_quant_data install_dir data_file)
90
    if (NOT EXISTS ${install_dir}/${data_file})
91
           inference_download_and_uncompress(${install_dir} ${INFERENCE_URL}/int8 ${data_file})
92 93 94
    endif()
endfunction()

W
Wojciech Uss 已提交
95
function(download_quant_model install_dir data_file)
96
    if (NOT EXISTS ${install_dir}/${data_file})
97
           inference_download_and_uncompress(${install_dir} ${INFERENCE_URL}/int8/QAT_models ${data_file})
98
    endif()
99 100
endfunction()

W
Wojciech Uss 已提交
101
function(save_quant_ic_model_test target quant_model_dir fp32_model_save_path int8_model_save_path)
102
    py_test(${target} SRCS ${CMAKE_CURRENT_SOURCE_DIR}/save_quant_model.py
W
Wojciech Uss 已提交
103
            ARGS --quant_model_path ${quant_model_dir}
104 105
	         --fp32_model_save_path ${fp32_model_save_path}
	         --int8_model_save_path ${int8_model_save_path}
W
Wojciech Uss 已提交
106
		 --debug)
107 108
endfunction()

W
Wojciech Uss 已提交
109
function(save_quant_nlp_model_test target quant_model_dir fp32_model_save_path int8_model_save_path ops_to_quantize)
110
    py_test(${target} SRCS ${CMAKE_CURRENT_SOURCE_DIR}/save_quant_model.py
W
Wojciech Uss 已提交
111
            ARGS --quant_model_path ${quant_model_dir}
112
	         --fp32_model_save_path ${fp32_model_save_path}
W
Wojciech Uss 已提交
113 114
	         --int8_model_save_path ${int8_model_save_path}
		 --ops_to_quantize ${ops_to_quantize})
115
endfunction()
116

117 118 119 120 121 122 123
function(convert_model2dot_test target model_path save_graph_dir save_graph_name)
    py_test(${target} SRCS ${CMAKE_CURRENT_SOURCE_DIR}/convert_model2dot.py
            ARGS --model_path ${model_path}
	         --save_graph_dir ${save_graph_dir}
	         --save_graph_name ${save_graph_name})
endfunction()

W
whs 已提交
124
if(WIN32)
125 126
	list(REMOVE_ITEM TEST_OPS test_light_nas)
	list(REMOVE_ITEM TEST_OPS test_post_training_quantization_mobilenetv1)
127
    list(REMOVE_ITEM TEST_OPS test_post_training_quantization_resnet50)
128
    list(REMOVE_ITEM TEST_OPS test_weight_quantization_mobilenetv1)
W
whs 已提交
129 130
endif()

131 132
if(LINUX AND WITH_MKLDNN)

133 134 135
	#### Image classification dataset: ImageNet (small)
	# The dataset should already be downloaded for INT8v2 unit tests
	set(IMAGENET_DATA_PATH "${INFERENCE_DEMO_INSTALL_DIR}/imagenet/data.bin")
136

137 138 139 140
	#### INT8 image classification python api test
	# Models should be already downloaded for INT8v2 unit tests

	set(INT8_INSTALL_DIR "${INFERENCE_DEMO_INSTALL_DIR}/int8v2")
141

W
Wojciech Uss 已提交
142
	#### QUANT & INT8 comparison python api tests
143

W
Wojciech Uss 已提交
144
	set(QUANT_INSTALL_DIR "${INFERENCE_DEMO_INSTALL_DIR}/quant")
145

W
Wojciech Uss 已提交
146
	### Quant1 for image classification
147

W
Wojciech Uss 已提交
148 149 150 151 152
	# Quant ResNet50
	set(QUANT_RESNET50_MODEL_DIR "${QUANT_INSTALL_DIR}/ResNet50_quant")
	set(QUANT_RESNET50_MODEL_ARCHIVE "ResNet50_qat_model.tar.gz")
	download_quant_model(${QUANT_RESNET50_MODEL_DIR} ${QUANT_RESNET50_MODEL_ARCHIVE})
	inference_quant_int8_image_classification_test(test_quant_int8_resnet50_mkldnn ${QUANT_RESNET50_MODEL_DIR}/model ${IMAGENET_DATA_PATH})
153

W
Wojciech Uss 已提交
154 155 156 157 158
	# Quant ResNet101
	set(QUANT_RESNET101_MODEL_DIR "${QUANT_INSTALL_DIR}/ResNet101_quant")
	set(QUANT_RESNET101_MODEL_ARCHIVE "ResNet101_qat_model.tar.gz")
	download_quant_model(${QUANT_RESNET101_MODEL_DIR} ${QUANT_RESNET101_MODEL_ARCHIVE})
	# inference_quant_int8_image_classification_test(test_quant_int8_resnet101_mkldnn ${QUANT_RESNET101_MODEL_DIR}/model ${IMAGENET_DATA_PATH})
159

W
Wojciech Uss 已提交
160 161 162 163 164
	# Quant GoogleNet
	set(QUANT_GOOGLENET_MODEL_DIR "${QUANT_INSTALL_DIR}/GoogleNet_quant")
	set(QUANT_GOOGLENET_MODEL_ARCHIVE "GoogleNet_qat_model.tar.gz")
	download_quant_model(${QUANT_GOOGLENET_MODEL_DIR} ${QUANT_GOOGLENET_MODEL_ARCHIVE})
	inference_quant_int8_image_classification_test(test_quant_int8_googlenet_mkldnn ${QUANT_GOOGLENET_MODEL_DIR}/model ${IMAGENET_DATA_PATH})
165

W
Wojciech Uss 已提交
166 167 168 169 170
	# Quant MobileNetV1
	set(QUANT_MOBILENETV1_MODEL_DIR "${QUANT_INSTALL_DIR}/MobileNetV1_quant")
	set(QUANT_MOBILENETV1_MODEL_ARCHIVE "MobileNetV1_qat_model.tar.gz")
	download_quant_model(${QUANT_MOBILENETV1_MODEL_DIR} ${QUANT_MOBILENETV1_MODEL_ARCHIVE})
	inference_quant_int8_image_classification_test(test_quant_int8_mobilenetv1_mkldnn ${QUANT_MOBILENETV1_MODEL_DIR}/model ${IMAGENET_DATA_PATH})
171

W
Wojciech Uss 已提交
172 173 174 175 176
	# Quant MobileNetV2
	set(QUANT_MOBILENETV2_MODEL_DIR "${QUANT_INSTALL_DIR}/MobileNetV2_quant")
	set(QUANT_MOBILENETV2_MODEL_ARCHIVE "MobileNetV2_qat_model.tar.gz")
	download_quant_model(${QUANT_MOBILENETV2_MODEL_DIR} ${QUANT_MOBILENETV2_MODEL_ARCHIVE})
	inference_quant_int8_image_classification_test(test_quant_int8_mobilenetv2_mkldnn ${QUANT_MOBILENETV2_MODEL_DIR}/model ${IMAGENET_DATA_PATH})
177

W
Wojciech Uss 已提交
178 179 180 181 182
	# Quant VGG16
	set(QUANT_VGG16_MODEL_DIR "${QUANT_INSTALL_DIR}/VGG16_quant")
	set(QUANT_VGG16_MODEL_ARCHIVE "VGG16_qat_model.tar.gz")
	download_quant_model(${QUANT_VGG16_MODEL_DIR} ${QUANT_VGG16_MODEL_ARCHIVE})
	# inference_quant_int8_image_classification_test(test_quant_int8_vgg16_mkldnn ${QUANT_VGG16_MODEL_DIR}/model ${IMAGENET_DATA_PATH})
183

W
Wojciech Uss 已提交
184 185 186 187 188
	# Quant VGG19
	set(QUANT_VGG19_MODEL_DIR "${QUANT_INSTALL_DIR}/VGG19_quant")
	set(QUANT_VGG19_MODEL_ARCHIVE "VGG19_qat_model.tar.gz")
	download_quant_model(${QUANT_VGG19_MODEL_DIR} ${QUANT_VGG19_MODEL_ARCHIVE})
	# inference_quant_int8_image_classification_test(test_quant_int8_vgg19_mkldnn ${QUANT_VGG19_MODEL_DIR}/model ${IMAGENET_DATA_PATH})
189

W
Wojciech Uss 已提交
190
	### Quant2 for image classification
191

W
Wojciech Uss 已提交
192
	# Quant2 ResNet50 with input/output scales in `fake_quantize_moving_average_abs_max` operators,
193
	# with weight scales in `fake_dequantize_max_abs` operators
W
Wojciech Uss 已提交
194 195 196
        set(QUANT2_RESNET50_MODEL_DIR "${QUANT_INSTALL_DIR}/ResNet50_quant2")
	set(QUANT2_RESNET50_MODEL_ARCHIVE "ResNet50_qat_perf.tar.gz")
	download_quant_model(${QUANT2_RESNET50_MODEL_DIR} ${QUANT2_RESNET50_MODEL_ARCHIVE})
197
	set(FP32_RESNET50_MODEL_DIR "${INT8_INSTALL_DIR}/resnet50")
W
Wojciech Uss 已提交
198
	inference_quant2_int8_image_classification_test(test_quant2_int8_resnet50_mkldnn ${QUANT2_RESNET50_MODEL_DIR}/ResNet50_qat_perf/float ${FP32_RESNET50_MODEL_DIR}/model ${IMAGENET_DATA_PATH})
199

W
Wojciech Uss 已提交
200
	# Quant2 ResNet50 with input/output scales in `fake_quantize_range_abs_max` operators and the `out_threshold` attributes,
201
	# with weight scales in `fake_dequantize_max_abs` operators
W
Wojciech Uss 已提交
202 203 204
	set(QUANT2_RESNET50_RANGE_MODEL_DIR "${QUANT_INSTALL_DIR}/ResNet50_quant2_range")
	set(QUANT2_RESNET50_RANGE_MODEL_ARCHIVE "ResNet50_qat_range.tar.gz")
	download_quant_model(${QUANT2_RESNET50_RANGE_MODEL_DIR} ${QUANT2_RESNET50_RANGE_MODEL_ARCHIVE})
W
Wojciech Uss 已提交
205
	inference_quant2_int8_image_classification_test(test_quant2_int8_resnet50_range_mkldnn ${QUANT2_RESNET50_RANGE_MODEL_DIR}/ResNet50_qat_range ${FP32_RESNET50_MODEL_DIR}/model ${IMAGENET_DATA_PATH})
206

W
Wojciech Uss 已提交
207
	# Quant2 ResNet50 with input/output scales in `fake_quantize_range_abs_max` operators and the `out_threshold` attributes,
208
	# with weight scales in `fake_channel_wise_dequantize_max_abs` operators
W
Wojciech Uss 已提交
209 210 211
	set(QUANT2_RESNET50_CHANNELWISE_MODEL_DIR "${QUANT_INSTALL_DIR}/ResNet50_quant2_channelwise")
	set(QUANT2_RESNET50_CHANNELWISE_MODEL_ARCHIVE "ResNet50_qat_channelwise.tar.gz")
	download_quant_model(${QUANT2_RESNET50_CHANNELWISE_MODEL_DIR} ${QUANT2_RESNET50_CHANNELWISE_MODEL_ARCHIVE})
W
Wojciech Uss 已提交
212
	inference_quant2_int8_image_classification_test(test_quant2_int8_resnet50_channelwise_mkldnn ${QUANT2_RESNET50_CHANNELWISE_MODEL_DIR}/ResNet50_qat_channelwise ${FP32_RESNET50_MODEL_DIR}/model ${IMAGENET_DATA_PATH})
W
Wojciech Uss 已提交
213 214 215 216 217

	# Quant2 MobileNetV1
        set(QUANT2_MOBILENETV1_MODEL_DIR "${QUANT_INSTALL_DIR}/MobileNetV1_quant2")
	set(QUANT2_MOBILENETV1_MODEL_ARCHIVE "MobileNet_qat_perf.tar.gz")
	download_quant_model(${QUANT2_MOBILENETV1_MODEL_DIR} ${QUANT2_MOBILENETV1_MODEL_ARCHIVE})
218
	set(FP32_MOBILENETV1_MODEL_DIR "${INT8_INSTALL_DIR}/mobilenetv1")
W
Wojciech Uss 已提交
219
	inference_quant2_int8_image_classification_test(test_quant2_int8_mobilenetv1_mkldnn ${QUANT2_MOBILENETV1_MODEL_DIR}/MobileNet_qat_perf/float ${FP32_MOBILENETV1_MODEL_DIR}/model ${IMAGENET_DATA_PATH})
220
	
W
Wojciech Uss 已提交
221
	### Quant2 for NLP
222 223 224 225 226

	set(NLP_DATA_ARCHIVE "Ernie_dataset.tar.gz")
	set(NLP_DATA_DIR "${INFERENCE_DEMO_INSTALL_DIR}/Ernie_dataset")
	set(NLP_DATA_PATH "${NLP_DATA_DIR}/Ernie_dataset/1.8w.bs1")
	set(NLP_LABLES_PATH "${NLP_DATA_DIR}/Ernie_dataset/label.xnli.dev")
W
Wojciech Uss 已提交
227
	download_quant_data(${NLP_DATA_DIR} ${NLP_DATA_ARCHIVE})
228

W
Wojciech Uss 已提交
229 230
	set(QUANT2_NLP_OPS_TO_QUANTIZE "fc,reshape2,transpose2,matmul,elementwise_add")

W
Wojciech Uss 已提交
231 232 233 234
	# Quant2 Ernie
	set(QUANT2_ERNIE_MODEL_ARCHIVE "ernie_qat.tar.gz")
	set(QUANT2_ERNIE_MODEL_DIR "${QUANT_INSTALL_DIR}/Ernie_quant2")
	download_quant_model(${QUANT2_ERNIE_MODEL_DIR} ${QUANT2_ERNIE_MODEL_ARCHIVE})
235
	set(FP32_ERNIE_MODEL_ARCHIVE "ernie_fp32_model.tar.gz")
W
Wojciech Uss 已提交
236 237
	set(FP32_ERNIE_MODEL_DIR "${QUANT_INSTALL_DIR}/Ernie_float")
	download_quant_fp32_model(${FP32_ERNIE_MODEL_DIR} ${FP32_ERNIE_MODEL_ARCHIVE})
W
Wojciech Uss 已提交
238
	inference_quant2_int8_nlp_test(test_quant2_int8_ernie_mkldnn ${QUANT2_ERNIE_MODEL_DIR}/Ernie_qat/float ${FP32_ERNIE_MODEL_DIR}/ernie_fp32_model ${NLP_DATA_PATH} ${NLP_LABLES_PATH} ${QUANT2_NLP_OPS_TO_QUANTIZE})
239

W
Wojciech Uss 已提交
240
	### Save FP32 model or INT8 model from Quant model
241
        
W
Wojciech Uss 已提交
242 243
	set(QUANT2_INT8_RESNET50_SAVE_PATH "${QUANT_INSTALL_DIR}/ResNet50_quant2_int8")
	set(QUANT2_FP32_RESNET50_SAVE_PATH "${QUANT_INSTALL_DIR}/ResNet50_quant2_fp32")
W
Wojciech Uss 已提交
244
	save_quant_ic_model_test(save_quant2_model_resnet50 ${QUANT2_RESNET50_MODEL_DIR}/ResNet50_qat_perf/float ${QUANT2_FP32_RESNET50_SAVE_PATH} ${QUANT2_INT8_RESNET50_SAVE_PATH})
245

W
Wojciech Uss 已提交
246 247
	set(QUANT2_INT8_ERNIE_SAVE_PATH "${QUANT_INSTALL_DIR}/Ernie_quant2_int8")
	set(QUANT2_FP32_ERNIE_SAVE_PATH "${QUANT_INSTALL_DIR}/Ernie_quant2_fp32")
W
Wojciech Uss 已提交
248
	save_quant_nlp_model_test(save_quant2_model_ernie ${QUANT2_ERNIE_MODEL_DIR}/Ernie_qat/float ${QUANT2_FP32_ERNIE_SAVE_PATH} ${QUANT2_INT8_ERNIE_SAVE_PATH} ${QUANT2_NLP_OPS_TO_QUANTIZE})
249

W
Wojciech Uss 已提交
250 251 252
	# Convert Quant2 model to dot and pdf files 
	set(QUANT2_INT8_ERNIE_DOT_SAVE_PATH "${QUANT_INSTALL_DIR}/Ernie_quant2_int8_dot_file")
	convert_model2dot_test(convert_model2dot_ernie ${QUANT2_ERNIE_MODEL_DIR}/Ernie_qat/float ${QUANT2_INT8_ERNIE_DOT_SAVE_PATH} "Ernie_quant2_int8")
253

254 255
endif()

W
Wojciech Uss 已提交
256
# Since the tests for Quant & INT8 comparison support only testing on Linux 
257
# with MKL-DNN, we remove it here to not test it on other systems.
258 259
list(REMOVE_ITEM TEST_OPS
	test_mkldnn_int8_quantization_strategy
W
Wojciech Uss 已提交
260 261
	quant_int8_image_classification_comparison
	quant_int8_nlp_comparison)
262

I
iducn 已提交
263 264 265
#TODO(wanghaoshuang): Fix this unitest failed on GCC8.
LIST(REMOVE_ITEM TEST_OPS test_auto_pruning)
LIST(REMOVE_ITEM TEST_OPS test_filter_pruning)
W
WangZhen 已提交
266 267 268
foreach(src ${TEST_OPS})
    py_test(${src} SRCS ${src}.py)
endforeach()