Merge pull request #14 from SunAhong1993/master

compatible with python2 and python3

caffe2fluid/convert.py

@@ -64,7 +64,7 @@ def main():
     """ main
     """
     parser = argparse.ArgumentParser()
-    parser.add_argument('def_path', help='Model definition (.prototxt) path')
+    parser.add_argument('--def_path', help='Model definition (.prototxt) path')
     parser.add_argument('--caffemodel', help='Model data (.caffemodel) path')
     parser.add_argument('--data-output-path', help='Converted data output path')
     parser.add_argument(

caffe2fluid/examples/imagenet/tools/diff.sh

 #!/bin/bash
 
-#
 #function:
-#   a tool used to check the difference of models' results generated by caffe model and paddle model
+#   a tool used to:
+#       1, convert a caffe model
+#       2, do inference(only in fluid) using this model
 #
-#howto:
-#   bash diff.sh resnet50 #when this has been finished, you can get the difference in precision
+#usage:
+#   cd caffe2fluid/examples/imagenet && bash run.sh alexnet ./models/alexnet.prototxt ./models/alexnet.caffemodel ./models/alexnet.py ./models/alexnet.npy
 #
-#notes:
-#   0, in order to infer using caffe, we need pycaffe installed
-#   1, prepare your caffe model in 'models.caffe/', eg: 'model.caffe/resnet101/resnet101.[prototxt|caffemodel]'
-#   2, converted paddle model will be in 'models'
-#   3, results of layers will be stored in 'results/${model_name}.[paddle|caffe]'
-#   4, only the last layer will be checked by default
 
-model_name="resnet50"
-results_root="results/"
+#set -x
 
-if [[ -n $1 ]];then
-    if [ $1 = "-h" ];then
-        echo "usage:"
-        echo "  bash $0 [model_name]"
-        echo "  eg:bash $0 resnet50"
-        exit 0
-    fi
+
+if [[ $# -lt 5 ]];then
+    echo "usage:"
+    echo "  bash $0 [model_name] [cf_prototxt_path] [cf_model_path] [pd_py_path] [pd_npy_path] [imagfile] [only_convert]"
+    echo "  eg: bash $0 alexnet ./models/alexnet.prototxt ./models/alexnet.caffemodel ./models/alexnet.py ./models/alexnet.npy"
+    exit 1
+else
     model_name=$1
+    cf_prototxt_path=$2
+    cf_model_path=$3
+    pd_py_path=$4
+    pd_npy_path=$5
+    only_convert=$7
 fi
+proto_file=$cf_prototxt_path
+caffemodel_file=$cf_model_path
+weight_file=$pd_npy_path
+net_file=$pd_py_path
 
-mkdir -p $results_root
-
-prototxt="$2/${model_name}.prototxt"
-caffemodel="$2/${model_name}.caffemodel"
+if [[ ! -e $proto_file ]];then
+    echo "not found prototxt[$proto_file]"
+    exit 1
+fi
 
-#1, dump layers' results from paddle
-paddle_results="$results_root/${model_name}.paddle"
-rm -rf $paddle_results
-rm -rf "results.paddle"
-bash ./tools/run.sh $model_name $2 $3
-if [[ $? -ne 0 ]] || [[ ! -e "results.paddle" ]];then
-    echo "not found paddle's results, maybe failed to convert"
+if [[ ! -e $caffemodel_file ]];then
+    echo "not found caffemodel[$caffemodel_file]"
     exit 1
 fi
-mv results.paddle $paddle_results
 
-#2, dump layers' results from caffe
-caffe_results="$results_root/${model_name}.caffe"
-rm -rf $caffe_results
-rm -rf "results.caffe"
+if [[ ! -e $pd_model_path ]];then
+    mkdir $pd_model_path
+fi
+
 PYTHON=`which python`
 if [[ -z $PYTHON ]];then
     PYTHON=`which python`
 fi
-$PYTHON ./infer.py caffe $prototxt $caffemodel $paddle_results/data.npy
-if [[ $? -ne 0 ]] || [[ ! -e "results.caffe" ]];then
-    echo "not found caffe's results, maybe failed to do inference with caffe"
-    exit 1
-fi
-mv results.caffe $caffe_results
-
-#3, extract layer names
-cat $prototxt | grep name | perl -ne 'if(/^\s*name\s*:\s+\"([^\"]+)/){ print $1."\n";}' >.layer_names
+$PYTHON ../../convert.py \
+        $proto_file \
+        --caffemodel $caffemodel_file \
+        --data-output-path $weight_file\
+        --code-output-path $net_file
 
-final_layer=$(cat $prototxt | perl -ne 'if(/^\s*top\s*:\s+\"([^\"]+)/){ print $1."\n";}' | tail -n1)
-ret=$(grep "^$final_layer$" .layer_names | wc -l)
-if [[ $ret -eq 0 ]];then
-    echo $final_layer >>.layer_names
+ret=$?
+if [[ $ret -ne 0 ]];then
+    echo "failed to convert caffe model[$cf_model_path]"
+    exit $ret
+else
+    echo "succeed to convert caffe model[$cf_model_path] to fluid model[$pd_model_path]"
 fi
 
-#4, compare one by one
-#for i in $(cat .layer_names);do
-for i in $(cat .layer_names | tail -n1);do
-    i=${i//\//_}
-    echo "process $i"
-    pd_npy=$(find $paddle_results/ -iname "${i}.*npy" | grep deleted -v | head -n1)
-    #pd_npy="$paddle_results/${i}.npy"
-    if [[ -f $pd_npy ]];then
-        $PYTHON compare.py $caffe_results/${i}.npy $pd_npy
+if [[ -z $only_convert ]];then
+    PYTHON=`which python`
+    if [[ -z $PYTHON ]];then
+        PYTHON=`which python`
+    fi
+    if [[ -n $6 ]];then
+        imgfile=$6
     else
-        echo "not found npy file[${i}.*npy] for layer[$i]"
-        exit 1
+        imgfile="data/65.jpeg"
     fi
-done
+    #FIX ME:
+    #   only look the first line in prototxt file for the name of this network, maybe not correct
+    net_name=`grep "name" $proto_file | head -n1 | perl -ne 'if(/^name\s*:\s*\"([^\"]+)\"/){ print $1."\n";}'`
+    if [[ -z $net_name ]];then
+        net_name="MyNet"
+    fi
+    cmd="$PYTHON ./infer.py dump $net_file $weight_file $imgfile $net_name"
+    echo $cmd
+    eval $cmd
+    ret=$?
+fi
+exit $ret

caffe2fluid/examples/imagenet/tools/run.sh

@@ -6,27 +6,29 @@
 #       2, do inference(only in fluid) using this model
 #
 #usage:
-#   cd caffe2fluid/examples/imagenet && bash run.sh resnet50 ./models.caffe/resnet50 ./models/resnet50
+#   cd caffe2fluid/examples/imagenet && bash run.sh alexnet ./models/alexnet.prototxt ./models/alexnet.caffemodel ./models/alexnet.py ./models/alexnet.npy
 #
 
 #set -x
 
 
-if [[ $# -lt 3 ]];then
+if [[ $# -lt 5 ]];then
     echo "usage:"
-    echo "  bash $0 [model_name] [cf_model_path] [pd_model_path] [only_convert]"
-    echo "  eg: bash $0 resnet50 ./models.caffe/resnet50 ./models/resnet50"
+    echo "  bash $0 [model_name] [cf_prototxt_path] [cf_model_path] [pd_py_path] [pd_npy_path] [imagfile] [only_convert]"
+    echo "  eg: bash $0 alexnet ./models/alexnet.prototxt ./models/alexnet.caffemodel ./models/alexnet.py ./models/alexnet.npy"
     exit 1
 else
     model_name=$1
-    cf_model_path=$2
-    pd_model_path=$3
-    only_convert=$4
+    cf_prototxt_path=$2
+    cf_model_path=$3
+    pd_py_path=$4
+    pd_npy_path=$5
+    only_convert=$7
 fi
-proto_file=$cf_model_path/${model_name}.prototxt
-caffemodel_file=$cf_model_path/${model_name}.caffemodel
-weight_file=$pd_model_path/${model_name}.npy
-net_file=$pd_model_path/${model_name}.py
+proto_file=$cf_prototxt_path
+caffemodel_file=$cf_model_path
+weight_file=$pd_npy_path
+net_file=$pd_py_path
 
 if [[ ! -e $proto_file ]];then
     echo "not found prototxt[$proto_file]"
@@ -65,7 +67,11 @@ if [[ -z $only_convert ]];then
     if [[ -z $PYTHON ]];then
         PYTHON=`which python`
     fi
-    imgfile="data/65.jpeg"
+    if [[ -n $6 ]];then
+        imgfile=$6
+    else
+        imgfile="data/65.jpeg"
+    fi
     #FIX ME:
     #   only look the first line in prototxt file for the name of this network, maybe not correct
     net_name=`grep "name" $proto_file | head -n1 | perl -ne 'if(/^name\s*:\s*\"([^\"]+)\"/){ print $1."\n";}'`

caffe2fluid/kaffe/net_template.py

@@ -88,21 +88,20 @@ def main():
 
     import sys
     import os
-    filename = os.path.splitext(os.path.basename(sys.argv[0]))[0]
-    if len(sys.argv) < 3:
-        print('usage:')
-        print('	python %s %s.npy [save_dir] [layer names seperated by comma]' \
-                % (sys.argv[0], filename))
-        print('	eg: python %s %s.npy ./fluid' % (sys.argv[0], filename))
-        print('	eg: python %s %s.npy ./fluid layer_name1,layer_name2' \
-                % (sys.argv[0], filename))
-        return 1
-
-    npy_weight = sys.argv[1]
-    fluid_model = sys.argv[2]
+    import argparse
+    filename = os.path.splitext(os.path.basename(sys.argv[0]))[0]    
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--npy_path', help='Model\'s parameters  (.npy) path')
+    parser.add_argument('--model-param-path', help='The path of model and param which are convertd by .npy',
+                       default='./fluid')
+    parser.add_argument(
+        '--need-layers-name', help='The layers need to save (split by ,)')
+    args = parser.parse_args()
+    npy_weight = args.npy_path
+    fluid_model = args.model_param_path
     outputs = None
-    if len(sys.argv) >= 4:
-        outputs = sys.argv[3].split(',')
+    if len(sys.argv) >= 6:
+        outputs = args.need_layers_name.split(',')
 
     ret = MyNet.convert(npy_weight, fluid_model, outputs)
     if ret == 0:

caffe2fluid/kaffe/protobuf_to_dict.py

@@ -15,17 +15,17 @@ TYPE_CALLABLE_MAP = {
     FieldDescriptor.TYPE_DOUBLE: float,
     FieldDescriptor.TYPE_FLOAT: float,
     FieldDescriptor.TYPE_INT32: int,
-    FieldDescriptor.TYPE_INT64: long,
+    FieldDescriptor.TYPE_INT64: int,
     FieldDescriptor.TYPE_UINT32: int,
-    FieldDescriptor.TYPE_UINT64: long,
+    FieldDescriptor.TYPE_UINT64: int,
     FieldDescriptor.TYPE_SINT32: int,
-    FieldDescriptor.TYPE_SINT64: long,
+    FieldDescriptor.TYPE_SINT64: int,
     FieldDescriptor.TYPE_FIXED32: int,
-    FieldDescriptor.TYPE_FIXED64: long,
+    FieldDescriptor.TYPE_FIXED64: int,
     FieldDescriptor.TYPE_SFIXED32: int,
-    FieldDescriptor.TYPE_SFIXED64: long,
+    FieldDescriptor.TYPE_SFIXED64: int,
     FieldDescriptor.TYPE_BOOL: bool,
-    FieldDescriptor.TYPE_STRING: unicode,
+    FieldDescriptor.TYPE_STRING: str,
     FieldDescriptor.TYPE_BYTES: lambda b: b.encode("base64"),
     FieldDescriptor.TYPE_ENUM: int,
 }