From 7167dd176b49f0f64edd84750b4e03a21e811637 Mon Sep 17 00:00:00 2001 From: guosheng Date: Fri, 26 May 2017 19:43:28 +0800 Subject: [PATCH] add caffe2paddle.py --- image_classification/caffe2paddle.py | 179 +++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 image_classification/caffe2paddle.py diff --git a/image_classification/caffe2paddle.py b/image_classification/caffe2paddle.py new file mode 100644 index 00000000..07769906 --- /dev/null +++ b/image_classification/caffe2paddle.py @@ -0,0 +1,179 @@ +import os +import functools +import inspect +import struct +import numpy as np +import caffe + + +def __default_not_set_callback__(kwargs, name): + return name not in kwargs or kwargs[name] is None + + +def wrap_param_default(param_names=None, + default_factory=None, + not_set_callback=__default_not_set_callback__): + assert param_names is not None + assert isinstance(param_names, list) or isinstance(param_names, tuple) + for each_param_name in param_names: + assert isinstance(each_param_name, basestring) + + def __impl__(func): + @functools.wraps(func) + def __wrapper__(*args, **kwargs): + if len(args) != 0: + argspec = inspect.getargspec(func) + num_positional = len(argspec.args) + if argspec.defaults: + num_positional -= len(argspec.defaults) + assert argspec.varargs or len( + args + ) <= num_positional, "Must use keyword arguments for non-positional args" + for name in param_names: + if not_set_callback(kwargs, name): # Not set + kwargs[name] = default_factory(func) + return func(*args, **kwargs) + + if hasattr(func, "argspec"): + __wrapper__.argspec = func.argspec + else: + __wrapper__.argspec = inspect.getargspec(func) + return __wrapper__ + + return __impl__ + + +class DefaultNameFactory(object): + def __init__(self, name_prefix): + self.__counter__ = 0 + self.__name_prefix__ = name_prefix + + def __call__(self, func): + if self.__name_prefix__ is None: + self.__name_prefix__ = func.__name__ + tmp = "__%s_%d__" % (self.__name_prefix__, self.__counter__) + self.__check_name__(tmp) + self.__counter__ += 1 + return tmp + + def __check_name__(self, nm): + pass + + def reset(self): + self.__counter__ = 0 + + +def wrap_name_default(name_prefix=None, name_param="name"): + """ + Decorator to set "name" arguments default to "{name_prefix}_{invoke_count}". + + .. code:: python + + @wrap_name_default("some_name") + def func(name=None): + print name # name will never be None. If name is not set, + # name will be "some_name_%d" + + :param name_prefix: name prefix. wrapped function"s __name__ if None. + :type name_prefix: basestring + :return: a decorator to set default name + :rtype: callable + """ + factory = DefaultNameFactory(name_prefix) + return wrap_param_default([name_param], factory) + + +class ModelConverter(object): + def __init__(self, caffe_model_file, caffe_pretrained_file, + paddle_output_path): + self.net = caffe.Net(caffe_model_file, caffe_pretrained_file, + caffe.TEST) + self.output_path = paddle_output_path + self.pre_layer_name = "" + self.pre_layer_type = "" + + def convert(self): + layer_dict = self.net.layer_dict + for layer_name in layer_dict.keys(): + layer = layer_dict[layer_name] + layer_params = layer.blobs + layer_type = layer.type + print layer_name, layer_type, len(layer_params) + if layer_type == "BatchNorm": + print layer_params[0].data.shape, layer_params[ + 1].data.shape, layer_params[2].data, type( + layer_params[0].data) + #print dir(layer) + #continue + if len(layer_params) > 0: + self.pre_layer_name = getattr( + self, "convert_" + layer_type + "_layer")(layer_params) + self.pre_layer_type = layer_type + return + + @staticmethod + def write_parameter(outfile, feats): + version = 0 + value_size = 4 + fo = open(outfile, "wb") + header = "" + header += struct.pack("i", version) + header += struct.pack("I", value_size) + header += struct.pack("Q", feats.size) + fo.write(header + feats.tostring()) + + @wrap_name_default("conv") + def convert_Convolution_layer(self, params, name=None): + for i in range(len(params)): + data = np.array(params[i].data) + if len(params) == 2: + suffix = "0" if i == 0 else "bias" + file = os.path.join(self.output_path, + "_%s.w%s" % (name, suffix)) + else: + file = os.path.join(self.output_path, + "_%s.w%s" % (name, str(i))) + ModelConverter.write_parameter(file, data.flatten()) + return name + + @wrap_name_default("fc_layer") + def convert_InnerProduct_layer(self, params, name=None): + for i in range(len(params)): + data = np.array(params[i].data) + if len(params) == 2: + suffix = "0" if i == 0 else "bias" + file = os.path.join(self.output_path, + "_%s.w%s" % (name, suffix)) + else: + file = os.path.join(self.output_path, + "_%s.w%s" % (name, str(i))) + data = np.transpose(data) + ModelConverter.write_parameter(file, data.flatten()) + return name + + @wrap_name_default("batch_norm") + def convert_BatchNorm_layer(self, params, name=None): + scale = np.array(params[-1].data) + for i in range(2): + data = np.array(params[i].data) * scale + file = os.path.join(self.output_path, + "_%s.w%s" % (name, str(i + 1))) + ModelConverter.write_parameter(file, data.flatten()) + return name + + def convert_Scale_layer(self, params, name=None): + assert self.pre_layer_type == "BatchNorm" + name = self.pre_layer_name + for i in range(len(params)): + data = np.array(params[i].data) + suffix = "0" if i == 0 else "bias" + file = os.path.join(self.output_path, "_%s.w%s" % (name, suffix)) + ModelConverter.write_parameter(file, data.flatten()) + return name + + +if __name__ == "__main__": + converter = ModelConverter("./ResNet-101-deploy.prototxt", + "./ResNet-101-model.caffemodel", + "./caffe2paddle_resnet/") + converter.convert() -- GitLab