From d7bb6da1bd24ffe0f73f5acaa485eccf8b878d78 Mon Sep 17 00:00:00 2001 From: sunyanfang01 Date: Thu, 28 Mar 2019 16:44:00 +0800 Subject: [PATCH] compatible with python2 and python3 --- caffe2fluid/convert.py | 2 +- caffe2fluid/examples/imagenet/tools/diff.sh | 123 ++++++++++---------- caffe2fluid/examples/imagenet/tools/run.sh | 30 +++-- caffe2fluid/kaffe/net_template.py | 27 +++-- caffe2fluid/kaffe/protobuf_to_dict.py | 12 +- 5 files changed, 101 insertions(+), 93 deletions(-) diff --git a/caffe2fluid/convert.py b/caffe2fluid/convert.py index adce7d0..145d365 100755 --- a/caffe2fluid/convert.py +++ b/caffe2fluid/convert.py @@ -64,7 +64,7 @@ def main(): """ main """ parser = argparse.ArgumentParser() - parser.add_argument('def_path', help='Model definition (.prototxt) path') + parser.add_argument('--def_path', help='Model definition (.prototxt) path') parser.add_argument('--caffemodel', help='Model data (.caffemodel) path') parser.add_argument('--data-output-path', help='Converted data output path') parser.add_argument( diff --git a/caffe2fluid/examples/imagenet/tools/diff.sh b/caffe2fluid/examples/imagenet/tools/diff.sh index 45250d2..c1756e8 100755 --- a/caffe2fluid/examples/imagenet/tools/diff.sh +++ b/caffe2fluid/examples/imagenet/tools/diff.sh @@ -1,83 +1,86 @@ #!/bin/bash -# #function: -# a tool used to check the difference of models' results generated by caffe model and paddle model +# a tool used to: +# 1, convert a caffe model +# 2, do inference(only in fluid) using this model # -#howto: -# bash diff.sh resnet50 #when this has been finished, you can get the difference in precision +#usage: +# cd caffe2fluid/examples/imagenet && bash run.sh alexnet ./models/alexnet.prototxt ./models/alexnet.caffemodel ./models/alexnet.py ./models/alexnet.npy # -#notes: -# 0, in order to infer using caffe, we need pycaffe installed -# 1, prepare your caffe model in 'models.caffe/', eg: 'model.caffe/resnet101/resnet101.[prototxt|caffemodel]' -# 2, converted paddle model will be in 'models' -# 3, results of layers will be stored in 'results/${model_name}.[paddle|caffe]' -# 4, only the last layer will be checked by default -model_name="resnet50" -results_root="results/" +#set -x -if [[ -n $1 ]];then - if [ $1 = "-h" ];then - echo "usage:" - echo " bash $0 [model_name]" - echo " eg:bash $0 resnet50" - exit 0 - fi + +if [[ $# -lt 5 ]];then + echo "usage:" + echo " bash $0 [model_name] [cf_prototxt_path] [cf_model_path] [pd_py_path] [pd_npy_path] [imagfile] [only_convert]" + echo " eg: bash $0 alexnet ./models/alexnet.prototxt ./models/alexnet.caffemodel ./models/alexnet.py ./models/alexnet.npy" + exit 1 +else model_name=$1 + cf_prototxt_path=$2 + cf_model_path=$3 + pd_py_path=$4 + pd_npy_path=$5 + only_convert=$7 fi +proto_file=$cf_prototxt_path +caffemodel_file=$cf_model_path +weight_file=$pd_npy_path +net_file=$pd_py_path -mkdir -p $results_root - -prototxt="$2/${model_name}.prototxt" -caffemodel="$2/${model_name}.caffemodel" +if [[ ! -e $proto_file ]];then + echo "not found prototxt[$proto_file]" + exit 1 +fi -#1, dump layers' results from paddle -paddle_results="$results_root/${model_name}.paddle" -rm -rf $paddle_results -rm -rf "results.paddle" -bash ./tools/run.sh $model_name $2 $3 -if [[ $? -ne 0 ]] || [[ ! -e "results.paddle" ]];then - echo "not found paddle's results, maybe failed to convert" +if [[ ! -e $caffemodel_file ]];then + echo "not found caffemodel[$caffemodel_file]" exit 1 fi -mv results.paddle $paddle_results -#2, dump layers' results from caffe -caffe_results="$results_root/${model_name}.caffe" -rm -rf $caffe_results -rm -rf "results.caffe" +if [[ ! -e $pd_model_path ]];then + mkdir $pd_model_path +fi + PYTHON=`which python` if [[ -z $PYTHON ]];then PYTHON=`which python` fi -$PYTHON ./infer.py caffe $prototxt $caffemodel $paddle_results/data.npy -if [[ $? -ne 0 ]] || [[ ! -e "results.caffe" ]];then - echo "not found caffe's results, maybe failed to do inference with caffe" - exit 1 -fi -mv results.caffe $caffe_results - -#3, extract layer names -cat $prototxt | grep name | perl -ne 'if(/^\s*name\s*:\s+\"([^\"]+)/){ print $1."\n";}' >.layer_names +$PYTHON ../../convert.py \ + $proto_file \ + --caffemodel $caffemodel_file \ + --data-output-path $weight_file\ + --code-output-path $net_file -final_layer=$(cat $prototxt | perl -ne 'if(/^\s*top\s*:\s+\"([^\"]+)/){ print $1."\n";}' | tail -n1) -ret=$(grep "^$final_layer$" .layer_names | wc -l) -if [[ $ret -eq 0 ]];then - echo $final_layer >>.layer_names +ret=$? +if [[ $ret -ne 0 ]];then + echo "failed to convert caffe model[$cf_model_path]" + exit $ret +else + echo "succeed to convert caffe model[$cf_model_path] to fluid model[$pd_model_path]" fi -#4, compare one by one -#for i in $(cat .layer_names);do -for i in $(cat .layer_names | tail -n1);do - i=${i//\//_} - echo "process $i" - pd_npy=$(find $paddle_results/ -iname "${i}.*npy" | grep deleted -v | head -n1) - #pd_npy="$paddle_results/${i}.npy" - if [[ -f $pd_npy ]];then - $PYTHON compare.py $caffe_results/${i}.npy $pd_npy +if [[ -z $only_convert ]];then + PYTHON=`which python` + if [[ -z $PYTHON ]];then + PYTHON=`which python` + fi + if [[ -n $6 ]];then + imgfile=$6 else - echo "not found npy file[${i}.*npy] for layer[$i]" - exit 1 + imgfile="data/65.jpeg" fi -done + #FIX ME: + # only look the first line in prototxt file for the name of this network, maybe not correct + net_name=`grep "name" $proto_file | head -n1 | perl -ne 'if(/^name\s*:\s*\"([^\"]+)\"/){ print $1."\n";}'` + if [[ -z $net_name ]];then + net_name="MyNet" + fi + cmd="$PYTHON ./infer.py dump $net_file $weight_file $imgfile $net_name" + echo $cmd + eval $cmd + ret=$? +fi +exit $ret diff --git a/caffe2fluid/examples/imagenet/tools/run.sh b/caffe2fluid/examples/imagenet/tools/run.sh index fc9fdc2..c1756e8 100755 --- a/caffe2fluid/examples/imagenet/tools/run.sh +++ b/caffe2fluid/examples/imagenet/tools/run.sh @@ -6,27 +6,29 @@ # 2, do inference(only in fluid) using this model # #usage: -# cd caffe2fluid/examples/imagenet && bash run.sh resnet50 ./models.caffe/resnet50 ./models/resnet50 +# cd caffe2fluid/examples/imagenet && bash run.sh alexnet ./models/alexnet.prototxt ./models/alexnet.caffemodel ./models/alexnet.py ./models/alexnet.npy # #set -x -if [[ $# -lt 3 ]];then +if [[ $# -lt 5 ]];then echo "usage:" - echo " bash $0 [model_name] [cf_model_path] [pd_model_path] [only_convert]" - echo " eg: bash $0 resnet50 ./models.caffe/resnet50 ./models/resnet50" + echo " bash $0 [model_name] [cf_prototxt_path] [cf_model_path] [pd_py_path] [pd_npy_path] [imagfile] [only_convert]" + echo " eg: bash $0 alexnet ./models/alexnet.prototxt ./models/alexnet.caffemodel ./models/alexnet.py ./models/alexnet.npy" exit 1 else model_name=$1 - cf_model_path=$2 - pd_model_path=$3 - only_convert=$4 + cf_prototxt_path=$2 + cf_model_path=$3 + pd_py_path=$4 + pd_npy_path=$5 + only_convert=$7 fi -proto_file=$cf_model_path/${model_name}.prototxt -caffemodel_file=$cf_model_path/${model_name}.caffemodel -weight_file=$pd_model_path/${model_name}.npy -net_file=$pd_model_path/${model_name}.py +proto_file=$cf_prototxt_path +caffemodel_file=$cf_model_path +weight_file=$pd_npy_path +net_file=$pd_py_path if [[ ! -e $proto_file ]];then echo "not found prototxt[$proto_file]" @@ -65,7 +67,11 @@ if [[ -z $only_convert ]];then if [[ -z $PYTHON ]];then PYTHON=`which python` fi - imgfile="data/65.jpeg" + if [[ -n $6 ]];then + imgfile=$6 + else + imgfile="data/65.jpeg" + fi #FIX ME: # only look the first line in prototxt file for the name of this network, maybe not correct net_name=`grep "name" $proto_file | head -n1 | perl -ne 'if(/^name\s*:\s*\"([^\"]+)\"/){ print $1."\n";}'` diff --git a/caffe2fluid/kaffe/net_template.py b/caffe2fluid/kaffe/net_template.py index c3a16da..f9387c9 100644 --- a/caffe2fluid/kaffe/net_template.py +++ b/caffe2fluid/kaffe/net_template.py @@ -88,21 +88,20 @@ def main(): import sys import os - filename = os.path.splitext(os.path.basename(sys.argv[0]))[0] - if len(sys.argv) < 3: - print('usage:') - print(' python %s %s.npy [save_dir] [layer names seperated by comma]' \ - % (sys.argv[0], filename)) - print(' eg: python %s %s.npy ./fluid' % (sys.argv[0], filename)) - print(' eg: python %s %s.npy ./fluid layer_name1,layer_name2' \ - % (sys.argv[0], filename)) - return 1 - - npy_weight = sys.argv[1] - fluid_model = sys.argv[2] + import argparse + filename = os.path.splitext(os.path.basename(sys.argv[0]))[0] + parser = argparse.ArgumentParser() + parser.add_argument('--npy_path', help='Model\'s parameters (.npy) path') + parser.add_argument('--model-param-path', help='The path of model and param which are convertd by .npy', + default='./fluid') + parser.add_argument( + '--need-layers-name', help='The layers need to save (split by ,)') + args = parser.parse_args() + npy_weight = args.npy_path + fluid_model = args.model_param_path outputs = None - if len(sys.argv) >= 4: - outputs = sys.argv[3].split(',') + if len(sys.argv) >= 6: + outputs = args.need_layers_name.split(',') ret = MyNet.convert(npy_weight, fluid_model, outputs) if ret == 0: diff --git a/caffe2fluid/kaffe/protobuf_to_dict.py b/caffe2fluid/kaffe/protobuf_to_dict.py index cdc8d44..5b65973 100644 --- a/caffe2fluid/kaffe/protobuf_to_dict.py +++ b/caffe2fluid/kaffe/protobuf_to_dict.py @@ -15,17 +15,17 @@ TYPE_CALLABLE_MAP = { FieldDescriptor.TYPE_DOUBLE: float, FieldDescriptor.TYPE_FLOAT: float, FieldDescriptor.TYPE_INT32: int, - FieldDescriptor.TYPE_INT64: long, + FieldDescriptor.TYPE_INT64: int, FieldDescriptor.TYPE_UINT32: int, - FieldDescriptor.TYPE_UINT64: long, + FieldDescriptor.TYPE_UINT64: int, FieldDescriptor.TYPE_SINT32: int, - FieldDescriptor.TYPE_SINT64: long, + FieldDescriptor.TYPE_SINT64: int, FieldDescriptor.TYPE_FIXED32: int, - FieldDescriptor.TYPE_FIXED64: long, + FieldDescriptor.TYPE_FIXED64: int, FieldDescriptor.TYPE_SFIXED32: int, - FieldDescriptor.TYPE_SFIXED64: long, + FieldDescriptor.TYPE_SFIXED64: int, FieldDescriptor.TYPE_BOOL: bool, - FieldDescriptor.TYPE_STRING: unicode, + FieldDescriptor.TYPE_STRING: str, FieldDescriptor.TYPE_BYTES: lambda b: b.encode("base64"), FieldDescriptor.TYPE_ENUM: int, } -- GitLab