提交 752cdcba 编写于 作者: S sunyanfang01 提交者: Macrobull

compatible with python2 and python3

上级 6fcef381
......@@ -64,7 +64,7 @@ def main():
""" main
"""
parser = argparse.ArgumentParser()
parser.add_argument('def_path', help='Model definition (.prototxt) path')
parser.add_argument('--def_path', help='Model definition (.prototxt) path')
parser.add_argument('--caffemodel', help='Model data (.caffemodel) path')
parser.add_argument('--data-output-path', help='Converted data output path')
parser.add_argument(
......
#!/bin/bash
#
#function:
# a tool used to check the difference of models' results generated by caffe model and paddle model
# a tool used to:
# 1, convert a caffe model
# 2, do inference(only in fluid) using this model
#
#howto:
# bash diff.sh resnet50 #when this has been finished, you can get the difference in precision
#usage:
# cd caffe2fluid/examples/imagenet && bash run.sh alexnet ./models/alexnet.prototxt ./models/alexnet.caffemodel ./models/alexnet.py ./models/alexnet.npy
#
#notes:
# 0, in order to infer using caffe, we need pycaffe installed
# 1, prepare your caffe model in 'models.caffe/', eg: 'model.caffe/resnet101/resnet101.[prototxt|caffemodel]'
# 2, converted paddle model will be in 'models'
# 3, results of layers will be stored in 'results/${model_name}.[paddle|caffe]'
# 4, only the last layer will be checked by default
model_name="resnet50"
results_root="results/"
#set -x
if [[ -n $1 ]];then
if [ $1 = "-h" ];then
echo "usage:"
echo " bash $0 [model_name]"
echo " eg:bash $0 resnet50"
exit 0
fi
if [[ $# -lt 5 ]];then
echo "usage:"
echo " bash $0 [model_name] [cf_prototxt_path] [cf_model_path] [pd_py_path] [pd_npy_path] [imagfile] [only_convert]"
echo " eg: bash $0 alexnet ./models/alexnet.prototxt ./models/alexnet.caffemodel ./models/alexnet.py ./models/alexnet.npy"
exit 1
else
model_name=$1
cf_prototxt_path=$2
cf_model_path=$3
pd_py_path=$4
pd_npy_path=$5
only_convert=$7
fi
proto_file=$cf_prototxt_path
caffemodel_file=$cf_model_path
weight_file=$pd_npy_path
net_file=$pd_py_path
mkdir -p $results_root
prototxt="$2/${model_name}.prototxt"
caffemodel="$2/${model_name}.caffemodel"
if [[ ! -e $proto_file ]];then
echo "not found prototxt[$proto_file]"
exit 1
fi
#1, dump layers' results from paddle
paddle_results="$results_root/${model_name}.paddle"
rm -rf $paddle_results
rm -rf "results.paddle"
bash ./tools/run.sh $model_name $2 $3
if [[ $? -ne 0 ]] || [[ ! -e "results.paddle" ]];then
echo "not found paddle's results, maybe failed to convert"
if [[ ! -e $caffemodel_file ]];then
echo "not found caffemodel[$caffemodel_file]"
exit 1
fi
mv results.paddle $paddle_results
#2, dump layers' results from caffe
caffe_results="$results_root/${model_name}.caffe"
rm -rf $caffe_results
rm -rf "results.caffe"
if [[ ! -e $pd_model_path ]];then
mkdir $pd_model_path
fi
PYTHON=`which python`
if [[ -z $PYTHON ]];then
PYTHON=`which python`
fi
$PYTHON ./infer.py caffe $prototxt $caffemodel $paddle_results/data.npy
if [[ $? -ne 0 ]] || [[ ! -e "results.caffe" ]];then
echo "not found caffe's results, maybe failed to do inference with caffe"
exit 1
fi
mv results.caffe $caffe_results
#3, extract layer names
cat $prototxt | grep name | perl -ne 'if(/^\s*name\s*:\s+\"([^\"]+)/){ print $1."\n";}' >.layer_names
$PYTHON ../../convert.py \
$proto_file \
--caffemodel $caffemodel_file \
--data-output-path $weight_file\
--code-output-path $net_file
final_layer=$(cat $prototxt | perl -ne 'if(/^\s*top\s*:\s+\"([^\"]+)/){ print $1."\n";}' | tail -n1)
ret=$(grep "^$final_layer$" .layer_names | wc -l)
if [[ $ret -eq 0 ]];then
echo $final_layer >>.layer_names
ret=$?
if [[ $ret -ne 0 ]];then
echo "failed to convert caffe model[$cf_model_path]"
exit $ret
else
echo "succeed to convert caffe model[$cf_model_path] to fluid model[$pd_model_path]"
fi
#4, compare one by one
#for i in $(cat .layer_names);do
for i in $(cat .layer_names | tail -n1);do
i=${i//\//_}
echo "process $i"
pd_npy=$(find $paddle_results/ -iname "${i}.*npy" | grep deleted -v | head -n1)
#pd_npy="$paddle_results/${i}.npy"
if [[ -f $pd_npy ]];then
$PYTHON compare.py $caffe_results/${i}.npy $pd_npy
if [[ -z $only_convert ]];then
PYTHON=`which python`
if [[ -z $PYTHON ]];then
PYTHON=`which python`
fi
if [[ -n $6 ]];then
imgfile=$6
else
echo "not found npy file[${i}.*npy] for layer[$i]"
exit 1
imgfile="data/65.jpeg"
fi
done
#FIX ME:
# only look the first line in prototxt file for the name of this network, maybe not correct
net_name=`grep "name" $proto_file | head -n1 | perl -ne 'if(/^name\s*:\s*\"([^\"]+)\"/){ print $1."\n";}'`
if [[ -z $net_name ]];then
net_name="MyNet"
fi
cmd="$PYTHON ./infer.py dump $net_file $weight_file $imgfile $net_name"
echo $cmd
eval $cmd
ret=$?
fi
exit $ret
......@@ -6,27 +6,29 @@
# 2, do inference(only in fluid) using this model
#
#usage:
# cd caffe2fluid/examples/imagenet && bash run.sh resnet50 ./models.caffe/resnet50 ./models/resnet50
# cd caffe2fluid/examples/imagenet && bash run.sh alexnet ./models/alexnet.prototxt ./models/alexnet.caffemodel ./models/alexnet.py ./models/alexnet.npy
#
#set -x
if [[ $# -lt 3 ]];then
if [[ $# -lt 5 ]];then
echo "usage:"
echo " bash $0 [model_name] [cf_model_path] [pd_model_path] [only_convert]"
echo " eg: bash $0 resnet50 ./models.caffe/resnet50 ./models/resnet50"
echo " bash $0 [model_name] [cf_prototxt_path] [cf_model_path] [pd_py_path] [pd_npy_path] [imagfile] [only_convert]"
echo " eg: bash $0 alexnet ./models/alexnet.prototxt ./models/alexnet.caffemodel ./models/alexnet.py ./models/alexnet.npy"
exit 1
else
model_name=$1
cf_model_path=$2
pd_model_path=$3
only_convert=$4
cf_prototxt_path=$2
cf_model_path=$3
pd_py_path=$4
pd_npy_path=$5
only_convert=$7
fi
proto_file=$cf_model_path/${model_name}.prototxt
caffemodel_file=$cf_model_path/${model_name}.caffemodel
weight_file=$pd_model_path/${model_name}.npy
net_file=$pd_model_path/${model_name}.py
proto_file=$cf_prototxt_path
caffemodel_file=$cf_model_path
weight_file=$pd_npy_path
net_file=$pd_py_path
if [[ ! -e $proto_file ]];then
echo "not found prototxt[$proto_file]"
......@@ -65,7 +67,11 @@ if [[ -z $only_convert ]];then
if [[ -z $PYTHON ]];then
PYTHON=`which python`
fi
imgfile="data/65.jpeg"
if [[ -n $6 ]];then
imgfile=$6
else
imgfile="data/65.jpeg"
fi
#FIX ME:
# only look the first line in prototxt file for the name of this network, maybe not correct
net_name=`grep "name" $proto_file | head -n1 | perl -ne 'if(/^name\s*:\s*\"([^\"]+)\"/){ print $1."\n";}'`
......
......@@ -88,21 +88,20 @@ def main():
import sys
import os
filename = os.path.splitext(os.path.basename(sys.argv[0]))[0]
if len(sys.argv) < 3:
print('usage:')
print(' python %s %s.npy [save_dir] [layer names seperated by comma]' \
% (sys.argv[0], filename))
print(' eg: python %s %s.npy ./fluid' % (sys.argv[0], filename))
print(' eg: python %s %s.npy ./fluid layer_name1,layer_name2' \
% (sys.argv[0], filename))
return 1
npy_weight = sys.argv[1]
fluid_model = sys.argv[2]
import argparse
filename = os.path.splitext(os.path.basename(sys.argv[0]))[0]
parser = argparse.ArgumentParser()
parser.add_argument('--npy_path', help='Model\'s parameters (.npy) path')
parser.add_argument('--model-param-path', help='The path of model and param which are convertd by .npy',
default='./fluid')
parser.add_argument(
'--need-layers-name', help='The layers need to save (split by ,)')
args = parser.parse_args()
npy_weight = args.npy_path
fluid_model = args.model_param_path
outputs = None
if len(sys.argv) >= 4:
outputs = sys.argv[3].split(',')
if len(sys.argv) >= 6:
outputs = args.need_layers_name.split(',')
ret = MyNet.convert(npy_weight, fluid_model, outputs)
if ret == 0:
......
......@@ -15,17 +15,17 @@ TYPE_CALLABLE_MAP = {
FieldDescriptor.TYPE_DOUBLE: float,
FieldDescriptor.TYPE_FLOAT: float,
FieldDescriptor.TYPE_INT32: int,
FieldDescriptor.TYPE_INT64: long,
FieldDescriptor.TYPE_INT64: int,
FieldDescriptor.TYPE_UINT32: int,
FieldDescriptor.TYPE_UINT64: long,
FieldDescriptor.TYPE_UINT64: int,
FieldDescriptor.TYPE_SINT32: int,
FieldDescriptor.TYPE_SINT64: long,
FieldDescriptor.TYPE_SINT64: int,
FieldDescriptor.TYPE_FIXED32: int,
FieldDescriptor.TYPE_FIXED64: long,
FieldDescriptor.TYPE_FIXED64: int,
FieldDescriptor.TYPE_SFIXED32: int,
FieldDescriptor.TYPE_SFIXED64: long,
FieldDescriptor.TYPE_SFIXED64: int,
FieldDescriptor.TYPE_BOOL: bool,
FieldDescriptor.TYPE_STRING: unicode,
FieldDescriptor.TYPE_STRING: str,
FieldDescriptor.TYPE_BYTES: lambda b: b.encode("base64"),
FieldDescriptor.TYPE_ENUM: int,
}
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册