From 6c5f928a3e099eb787111c8fe5120118ef2e5155 Mon Sep 17 00:00:00 2001
From: tensor-tang <jian.j.tang@intel.com>
Date: Mon, 27 Nov 2017 14:27:25 +0800
Subject: [PATCH] enable inference benchmark

---
 benchmark/paddle/image/googlenet.py  |  2 +-
 benchmark/paddle/image/resnet.py     |  2 +-
 benchmark/paddle/image/run_mkldnn.sh | 69 ++++++++++++++++++++++++++--
 benchmark/paddle/image/vgg.py        |  2 +-
 4 files changed, 68 insertions(+), 7 deletions(-)

diff --git a/benchmark/paddle/image/googlenet.py b/benchmark/paddle/image/googlenet.py
index a88ecac67d9..5b1f0ca0068 100644
--- a/benchmark/paddle/image/googlenet.py
+++ b/benchmark/paddle/image/googlenet.py
@@ -9,7 +9,7 @@ use_gpu = get_config_arg('use_gpu', bool, True)
 
 args = {'height': height, 'width': width, 'color': True, 'num_class': num_class}
 define_py_data_sources2(
-    "train.list", None, module="provider", obj="process", args=args)
+    "train.list", "test.list", module="provider", obj="process", args=args)
 
 settings(
     batch_size=batch_size,
diff --git a/benchmark/paddle/image/resnet.py b/benchmark/paddle/image/resnet.py
index 6ae1857642e..f8c1c2df885 100644
--- a/benchmark/paddle/image/resnet.py
+++ b/benchmark/paddle/image/resnet.py
@@ -10,7 +10,7 @@ is_test = get_config_arg("is_test", bool, False)
 
 args = {'height': height, 'width': width, 'color': True, 'num_class': num_class}
 define_py_data_sources2(
-    "train.list", None, module="provider", obj="process", args=args)
+    "train.list", "test.list", module="provider", obj="process", args=args)
 
 settings(
     batch_size=batch_size,
diff --git a/benchmark/paddle/image/run_mkldnn.sh b/benchmark/paddle/image/run_mkldnn.sh
index f768f6c29a8..c78079fa45d 100755
--- a/benchmark/paddle/image/run_mkldnn.sh
+++ b/benchmark/paddle/image/run_mkldnn.sh
@@ -8,13 +8,13 @@ function train() {
   use_mkldnn=$4
   if [ $4 == "True" ]; then
     thread=1
-    log="logs/${topology}-${layer_num}-mkldnn-${bs}.log"
+    log="logs/train-${topology}-${layer_num}-mkldnn-${bs}.log"
   elif [ $4 == "False" ]; then
     thread=`nproc`
     # each trainer_count use only 1 core to avoid conflict
-    log="logs/${topology}-${layer_num}-${thread}mklml-${bs}.log"
+    log="logs/train-${topology}-${layer_num}-${thread}mklml-${bs}.log"
   else
-    echo "Wrong input $3, use True or False."
+    echo "Wrong input $4, use True or False."
     exit 0
   fi
   args="batch_size=${bs},layer_num=${layer_num}"
@@ -30,13 +30,74 @@ function train() {
     2>&1 | tee ${log} 
 }
 
-if [ ! -d "train.list" ]; then
+function test() {
+  unset OMP_NUM_THREADS MKL_NUM_THREADS OMP_DYNAMIC KMP_AFFINITY
+  topology=$1
+  layer_num=$2
+  bs=$3
+  use_mkldnn=$4
+  if [ $4 == "True" ]; then
+    thread=1
+    log="logs/test-${topology}-${layer_num}-mkldnn-${bs}.log"
+  elif [ $4 == "False" ]; then
+    thread=`nproc`
+    if [ $thread -gt $bs ]; then
+      thread=$bs
+    fi
+    log="logs/test-${topology}-${layer_num}-${thread}mklml-${bs}.log"
+  else
+    echo "Wrong input $4, use True or False."
+    exit 0
+  fi
+
+  models_in="models/${topology}-${layer_num}/pass-00000/"
+  if [ ! -d $models_in ]; then
+    echo "Training model ${topology}_${layer_num}"
+    paddle train --job=train \
+      --config="${topology}.py" \
+      --use_mkldnn=True \
+      --use_gpu=False \
+      --trainer_count=1 \
+      --num_passes=1 \
+      --save_dir="models/${topology}-${layer_num}" \
+      --config_args="batch_size=128,layer_num=${layer_num}" \
+      > /dev/null 2>&1
+    echo "Done"
+  fi
+  paddle train --job=test \
+    --config="${topology}.py" \
+    --use_mkldnn=$use_mkldnn \
+    --use_gpu=False \
+    --trainer_count=$thread \
+    --log_period=10 \
+    --config_args="batch_size=${bs},layer_num=${layer_num},is_test=True" \
+    --init_model_path=$models_in \
+    2>&1 | tee ${log} 
+}
+
+if [ ! -f "train.list" ]; then
   echo " " > train.list
 fi
+if [ ! -f "test.list" ]; then
+  echo " " > test.list
+fi
 if [ ! -d "logs" ]; then
   mkdir logs
 fi
+if [ ! -d "models" ]; then
+  mkdir -p models
+fi
+
+# inference benchmark
+for use_mkldnn in True False; do
+  for batchsize in 1 2 4 8 16; do
+    test googlenet v1 $batchsize $use_mkldnn
+    test resnet 50 $batchsize $use_mkldnn
+    test vgg 19 $batchsize $use_mkldnn
+  done
+done
 
+# training benchmark
 for use_mkldnn in True False; do
   for batchsize in 64 128 256; do
     train vgg 19 $batchsize $use_mkldnn
diff --git a/benchmark/paddle/image/vgg.py b/benchmark/paddle/image/vgg.py
index 420884ed8e1..97f4dbe0e1a 100644
--- a/benchmark/paddle/image/vgg.py
+++ b/benchmark/paddle/image/vgg.py
@@ -9,7 +9,7 @@ layer_num = get_config_arg('layer_num', int, 19)
 
 args = {'height': height, 'width': width, 'color': True, 'num_class': num_class}
 define_py_data_sources2(
-    "train.list", None, module="provider", obj="process", args=args)
+    "train.list", "test.list", module="provider", obj="process", args=args)
 
 settings(
     batch_size=batch_size,
-- 
GitLab