diff.sh

#!/bin/bash

#
#function:
#   a tool used to check the difference of models' results generated by caffe model and paddle model
#
#howto:
#   bash diff.sh resnet50 #when this has been finished, you can get the difference in precision
#
#notes:
#   0, in order to infer using caffe, we need pycaffe installed
#   1, prepare your caffe model in 'models.caffe/', eg: 'model.caffe/resnet101/resnet101.[prototxt|caffemodel]'
#   2, converted paddle model will be in 'models'
#   3, results of layers will be stored in 'results/${model_name}.[paddle|caffe]'
#   4, only the last layer will be checked by default

model_name="resnet50"
results_root="results/"

if [[ -n $1 ]];then
    if [ $1 = "-h" ];then
        echo "usage:"
        echo "  bash $0 [model_name]"
        echo "  eg:bash $0 resnet50"
        exit 0
    fi
    model_name=$1
fi

mkdir -p $results_root

prototxt="$2/${model_name}.prototxt"
caffemodel="$2/${model_name}.caffemodel"

#1, dump layers' results from paddle
paddle_results="$results_root/${model_name}.paddle"
rm -rf $paddle_results
rm -rf "results.paddle"
bash ./tools/run.sh $model_name $2 $3
if [[ $? -ne 0 ]] || [[ ! -e "results.paddle" ]];then
    echo "not found paddle's results, maybe failed to convert"
    exit 1
fi
mv results.paddle $paddle_results

#2, dump layers' results from caffe
caffe_results="$results_root/${model_name}.caffe"
rm -rf $caffe_results
rm -rf "results.caffe"
PYTHON=`which python`
if [[ -z $PYTHON ]];then
    PYTHON=`which python`
fi
$PYTHON ./infer.py caffe $prototxt $caffemodel $paddle_results/data.npy
if [[ $? -ne 0 ]] || [[ ! -e "results.caffe" ]];then
    echo "not found caffe's results, maybe failed to do inference with caffe"
    exit 1
fi
mv results.caffe $caffe_results

#3, extract layer names
cat $prototxt | grep name | perl -ne 'if(/^\s*name\s*:\s+\"([^\"]+)/){ print $1."\n";}' >.layer_names

final_layer=$(cat $prototxt | perl -ne 'if(/^\s*top\s*:\s+\"([^\"]+)/){ print $1."\n";}' | tail -n1)
ret=$(grep "^$final_layer$" .layer_names | wc -l)
if [[ $ret -eq 0 ]];then
    echo $final_layer >>.layer_names
fi

#4, compare one by one
#for i in $(cat .layer_names);do
for i in $(cat .layer_names | tail -n1);do
    i=${i//\//_}
    echo "process $i"
    pd_npy=$(find $paddle_results/ -iname "${i}.*npy" | grep deleted -v | head -n1)
    #pd_npy="$paddle_results/${i}.npy"
    if [[ -f $pd_npy ]];then
        $PYTHON compare.py $caffe_results/${i}.npy $pd_npy
    else
        echo "not found npy file[${i}.*npy] for layer[$i]"
        exit 1
    fi
done