diff --git a/.gitignore b/.gitignore index 8f92118b08bb30531869c28d32d335cc47116350..8c4450181d82116620d880c93789dee9dcda9d73 100644 --- a/.gitignore +++ b/.gitignore @@ -92,3 +92,4 @@ metal/images/ metal/paddle-mobile/paddle-mobile/CPU/libpaddle-mobile.a *.xcuserdatad/ */xcuserdata/ +/venv/ diff --git a/python/tools/imagetools/imagetools.py b/tools/python/imagetools/imagetools.py similarity index 100% rename from python/tools/imagetools/imagetools.py rename to tools/python/imagetools/imagetools.py diff --git a/python/tools/imagetools/img2nchw.py b/tools/python/imagetools/img2nchw.py similarity index 100% rename from python/tools/imagetools/img2nchw.py rename to tools/python/imagetools/img2nchw.py diff --git a/python/tools/imagetools/img2nhwc.py b/tools/python/imagetools/img2nhwc.py similarity index 100% rename from python/tools/imagetools/img2nhwc.py rename to tools/python/imagetools/img2nhwc.py diff --git a/python/tools/imagetools/numpy2binary.py b/tools/python/imagetools/numpy2binary.py similarity index 100% rename from python/tools/imagetools/numpy2binary.py rename to tools/python/imagetools/numpy2binary.py diff --git a/tools/python/modeltools/.gitignore b/tools/python/modeltools/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..4108f5244bc039cb95b06e391d51250bb9d0ce42 --- /dev/null +++ b/tools/python/modeltools/.gitignore @@ -0,0 +1,109 @@ +# Created by .ignore support plugin (hsz.mobi) +### Python template +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +/yolo/datas/ +/mobilenet/datas/ diff --git a/tools/python/modeltools/core/__init__.py b/tools/python/modeltools/core/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/python/tools/mdl2fluid/framework.proto b/tools/python/modeltools/core/framework.proto similarity index 100% rename from python/tools/mdl2fluid/framework.proto rename to tools/python/modeltools/core/framework.proto diff --git a/python/tools/mdl2fluid/framework_pb2.py b/tools/python/modeltools/core/framework_pb2.py similarity index 100% rename from python/tools/mdl2fluid/framework_pb2.py rename to tools/python/modeltools/core/framework_pb2.py diff --git a/python/tools/mdl2fluid/op_types.py b/tools/python/modeltools/core/op_types.py similarity index 59% rename from python/tools/mdl2fluid/op_types.py rename to tools/python/modeltools/core/op_types.py index ff7d78d20835c605dc581ef14ad2d7d5171fea1d..89f6ee47dd5e46ae952c574f246c09f859b8a5db 100644 --- a/python/tools/mdl2fluid/op_types.py +++ b/tools/python/modeltools/core/op_types.py @@ -5,22 +5,28 @@ layer_mdl_conv = 'ConvolutionLayer' layer_mdl_deepwise_conv = 'DepthwiseConvolutionLayer' layer_mdl_relu = 'ReluLayer' layer_mdl_pointwise_add = 'PointwiseConvolutionLayer' +layer_mdl_pooling = 'PoolingLayer' +layer_mdl_softmax = 'SoftmaxLayer' # fluid ops op_fluid_fusion_conv_add = 'fusion_conv_add' op_fluid_relu = 'relu' +op_fluid_pooling = 'pool2d' +op_fluid_softmax = 'softmax' # dict mdk layer --- fluid op mdl2fluid_op_layer_dict = { layer_mdl_conv: op_fluid_fusion_conv_add, layer_mdl_deepwise_conv: op_fluid_fusion_conv_add, layer_mdl_relu: op_fluid_relu, - layer_mdl_pointwise_add: op_fluid_fusion_conv_add + layer_mdl_pointwise_add: op_fluid_fusion_conv_add, + layer_mdl_pooling: op_fluid_pooling, + layer_mdl_softmax: op_fluid_softmax } mdl_outputs_key = "outputs" mdl_inputs_key = "inputs" -mdl_weight_key = "weights" +mdl_weight_key = "weight" mdl_attrs_key = "params" # dict of mdl-input _out param to fluid input out attrs @@ -39,13 +45,30 @@ fusion_conv_add_dict = { relu_dict = { mdl_inputs_key: 'X', mdl_outputs_key: 'Out', - mdl_weight_key: () + # mdl_weight_key: () } + +pool2d_dict = { + mdl_inputs_key: 'X', + mdl_outputs_key: 'Out', + # mdl_weight_key: (), + mdl_attrs_key: ('pooling_type', 'global_pooling') + +} + +softmax_dict = { + mdl_inputs_key: 'X', + mdl_outputs_key: 'Out', + mdl_weight_key: (), + mdl_attrs_key: () +} # mdl layers --- fluid ops op_io_dict = { 'fusion_conv_add': fusion_conv_add_dict, - 'relu': relu_dict + 'relu': relu_dict, + 'pool2d': pool2d_dict, + 'softmax': softmax_dict } # fluid attr key --- mdl params key @@ -60,64 +83,3 @@ fluid_attrs_type_dict = { 'strides': 6, 'groups': 6 } - -# '': "bias_term", 是不是要add 目前 yolo的模型都是 bias_term = 1 - - -# attrs { -# name: "axis" -# type: INT -# i: 1 -# } - - -# attrs_name = { -# 'name': "workspace_size_MB", -# 'type': 'INT', -# 'i': '4096' -# } -# attrs -# { -# name: "data_format" -# type: STRING -# s: "AnyLayout" -# } -# attrs -# { -# name: "use_mkldnn" -# type: BOOLEAN -# b: false -# } -# attrs -# { -# name: "use_cudnn" -# type: BOOLEAN -# b: true -# } -# attrs -# { -# name: "dilations" -# type: INTS -# ints: 1 -# ints: 1 -# } -# attrs -# { -# name: "groups" -# type: INT -# i: 1 -# } -# attrs -# { -# name: "paddings" -# type: INTS -# ints: 0 -# ints: 0 -# } -# attrs -# { -# name: "strides" -# type: INTS -# ints: 1 -# ints: 1 -# } diff --git a/tools/python/modeltools/mobilenet/__init__.py b/tools/python/modeltools/mobilenet/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tools/python/modeltools/mobilenet/converter_mobilenet.py b/tools/python/modeltools/mobilenet/converter_mobilenet.py new file mode 100644 index 0000000000000000000000000000000000000000..da95c24212f48e3ed03dcf0481c14f3ffa881986 --- /dev/null +++ b/tools/python/modeltools/mobilenet/converter_mobilenet.py @@ -0,0 +1,347 @@ +import json +import os + +from core import framework_pb2 as framework_pb2, op_types as types +from mobilenet.swicher import Swichter +import shutil + + +def load_mdl(mdl_json_path): + # print('mdl json path : ' + mdl_json_path) + with open(mdl_json_path, 'r') as f: + return json.load(f) + + +class Converter: + 'convert mdlmodel to fluidmodel' + + def __init__(self, base_dir, mdl_json_path): + self.mdl_json_path = base_dir + mdl_json_path + self.base_dir = base_dir + print mdl_json_path + self.mdl_json = load_mdl(self.mdl_json_path) + self.program_desc = framework_pb2.ProgramDesc() + self.weight_list_ = [] + self.deepwise_weight_list_ = [] + # print(json_dick) + # layers = (json_dick['layer']) + # for layer in layers: + # print(layer) + + def convert(self): + print 'convert begin.....' + # add block_desc + block_desc = self.program_desc.blocks.add() + block_desc.idx = 0 + block_desc.parent_idx = -1 + self.package_ops(block_desc) + self.package_vars(block_desc) + print 'blocks: ' + print self.program_desc.blocks + print 'convert end.....' + desc_serialize_to_string = self.program_desc.SerializeToString() + + outputmodel_ = self.base_dir + 'datas/target/outputmodel/' + if os.path.exists(outputmodel_): + shutil.rmtree(outputmodel_) + os.makedirs(outputmodel_, 0777) + # todo copy weight files + # if os.path.exists(outputmodel_): + # shutil.rmtree(outputmodel_) + # shutil.copytree('yolo/datas/multiobjects/float32s_nchw_with_head/', 'mobilenet/datas/target/outputmodel/') + + f = open(outputmodel_ + "__model__", "wb") + f.write(desc_serialize_to_string) + f.close() + + def package_ops(self, block_desc): + + self.add_op_feed(block_desc) + + # add ops with layer + if 'layer' in self.mdl_json: + + layers_ = self.mdl_json['layer'] + for layer in layers_: + desc_ops_add = block_desc.ops.add() + + # print layer + # for i in layer: + # print i + if 'name' in layer: + l_name = layer['name'] + if 'type' in layer: + self.package_ops_type(desc_ops_add, layer) + + if 'weight' in layer: + self.package_ops_weight2inputs(desc_ops_add, layer) + + if 'output' in layer: + self.package_ops_outputs(desc_ops_add, layer) + + if 'input' in layer: + self.package_ops_inputs(desc_ops_add, layer) + + self.package_ops_attrs(desc_ops_add, layer) + + self.add_op_fetch(block_desc) + + def add_op_feed(self, block_desc): + desc_ops_add = block_desc.ops.add() + inputs_add = desc_ops_add.inputs.add() + inputs_add.parameter = 'X' + inputs_add.arguments.append('feed') + desc_ops_add.type = 'feed' + outputs_add = desc_ops_add.outputs.add() + outputs_add.parameter = 'Out' + outputs_add.arguments.append('data') + attrs_add = desc_ops_add.attrs.add() + attrs_add.name = 'col' + # boolean + attrs_add.type = 0 + attrs_add.i = 0 + + def add_op_fetch(self, block_desc): + desc_ops_add = block_desc.ops.add() + inputs_add = desc_ops_add.inputs.add() + inputs_add.parameter = 'X' + inputs_add.arguments.append('conv_pred_87') + desc_ops_add.type = 'fetch' + outputs_add = desc_ops_add.outputs.add() + outputs_add.parameter = 'Out' + outputs_add.arguments.append('fetch') + attrs_add = desc_ops_add.attrs.add() + attrs_add.name = 'col' + # boolean + attrs_add.type = 0 + attrs_add.i = 0 + + @staticmethod + def package_ops_attrs(desc_ops_add, layer): + # print l_params + # print desc_ops_add.type + if desc_ops_add.type == types.op_fluid_fusion_conv_add: + Converter.pack_fusion_conv_add_attr(desc_ops_add, layer) + elif desc_ops_add.type == types.op_fluid_relu: + # fusion_conv_add : attrs + attrs_add = desc_ops_add.attrs.add() + attrs_add.name = 'use_mkldnn' + # boolean + attrs_add.type = 6 + attrs_add.b = 0 + + @staticmethod + def pack_fusion_conv_add_attr(desc_ops_add, layer): + + # fusion_conv_add : attrs + attrs_add = desc_ops_add.attrs.add() + attrs_add.name = 'workspace_size_MB' + # 0-->INT + attrs_add.type = 0 + attrs_add.i = 4096 + + attrs_add = desc_ops_add.attrs.add() + attrs_add.name = 'data_format' + # 2-->STRING + attrs_add.type = 2 + attrs_add.s = 'AnyLayout' + + attrs_add = desc_ops_add.attrs.add() + attrs_add.name = 'use_mkldnn' + # boolean + attrs_add.type = 6 + attrs_add.b = 0 + + attrs_add = desc_ops_add.attrs.add() + attrs_add.name = 'use_cudnn' + # boolean + attrs_add.type = 6 + attrs_add.b = 1 + + attrs_add = desc_ops_add.attrs.add() + attrs_add.name = 'dilations' + # ints + attrs_add.type = 3 + attrs_add.ints.append(1) + attrs_add.ints.append(1) + + attrs_add = desc_ops_add.attrs.add() + attrs_add.name = 'axis' + # int + attrs_add.type = 0 + attrs_add.i = 1 + + if 'param' in layer: + l_params = layer['param'] + + attrs_add = desc_ops_add.attrs.add() + attrs_add.name = 'paddings' + # ints + attrs_add.type = 3 + attrs_add.ints.append(l_params[types.fusion_conv_add_attrs_dict.get('paddings')]) + attrs_add.ints.append(l_params[types.fusion_conv_add_attrs_dict.get('paddings')]) + + attrs_add = desc_ops_add.attrs.add() + attrs_add.name = 'strides' + # ints + attrs_add.type = 3 + attrs_add.ints.append(l_params[types.fusion_conv_add_attrs_dict.get('strides')]) + attrs_add.ints.append(l_params[types.fusion_conv_add_attrs_dict.get('strides')]) + + attrs_add = desc_ops_add.attrs.add() + attrs_add.name = 'groups' + # int + attrs_add.type = 0 + attrs_add.i = l_params[types.fusion_conv_add_attrs_dict.get('groups')] + # attrs_add.i = 1 + + # + # op_attrs_tupl = types.op_io_dict.get(desc_ops_add.type) \ + # .get(types.mdl_attrs_key) + # + # + # + # + # # group stride padding + # print '----------------------' + # for i, val in enumerate(op_attrs_tupl): + # attrs_add = desc_ops_add.attrs.add() + # attr_name = op_attrs_tupl[i] + # print attr_name + # attrs_add.name = attr_name + # attrs_add.type = types.fluid_attrs_type_dict.get(attr_name) + # attrs_add. + # print l_params[types.fusion_conv_add_attrs_dict.get(attr_name)] + + # for p in l_params: + # attrs_add = desc_ops_add.attrs.add() + + @staticmethod + def package_ops_inputs(desc_ops_add, layer): + l_inputs = layer['input'] + for i in l_inputs: + inputs_add = desc_ops_add.inputs.add() + # print i + inputs_add.parameter = types.op_io_dict.get(desc_ops_add.type).get(types.mdl_inputs_key) + inputs_add.arguments.append(i) + + @staticmethod + def package_ops_outputs(desc_ops_add, layer): + l_outputs = layer['output'] + for o in l_outputs: + # print o + outputs_add = desc_ops_add.outputs.add() + dict = types.op_io_dict.get(desc_ops_add.type) + print 'desc_ops_add.type: ' + desc_ops_add.type + print dict + outputs_add.parameter = dict.get(types.mdl_outputs_key) + outputs_add.arguments.append(o) + + def package_ops_weight2inputs(self, desc_ops_add, layer): + l_weights = layer['weight'] + for w in l_weights: + self.weight_list_.append(w) + + if layer['type'] == types.layer_mdl_deepwise_conv: + # print l_weights[0] + self.deepwise_weight_list_.append(l_weights[0]) + + op_weight_tup = types.op_io_dict.get(desc_ops_add.type).get(types.mdl_weight_key) + if op_weight_tup is not None: + # print len(op_weight_tup) + for i, val in enumerate(op_weight_tup): + # print i + # print val + inputs_add = desc_ops_add.inputs.add() + inputs_add.parameter = op_weight_tup[i] + inputs_add.arguments.append(l_weights[i]) + + # for w in l_weights: + # inputs_add = desc_ops_add.inputs.add() + # # print w + # inputs_add.parameter = op_weight_tup[0] + # inputs_add.arguments.append(w) + + @staticmethod + def package_ops_type(desc_ops_add, layer): + l_type = layer['type'] + # print l_type + # print mdl2fluid_op_layer_dict.get(l_type) + desc_ops_add.type = types.mdl2fluid_op_layer_dict.get(l_type) + + def package_vars(self, block_desc): + vars_add = block_desc.vars.add() + vars_add.name = 'feed' + vars_add.type.type = 9 # 9 is FEED_MINIBATCH + vars_add.persistable = 1 + # fetch + vars_add = block_desc.vars.add() + vars_add.name = 'fetch' + vars_add.type.type = 10 # 10 is fetch list + vars_add.persistable = 1 + + json_matrix_ = self.mdl_json['matrix'] + # print json_matrix_ + for j in json_matrix_: + vars_add = block_desc.vars.add() + vars_add.name = j + vars_add.type.type = 7 # 7 is lodtensor + # print j + tensor = vars_add.type.lod_tensor.tensor + tensor.data_type = 5 # 5 is FP32 + + # print json_matrix_ + + dims_of_matrix = json_matrix_.get(j) + # dims_size = len(dims_of_matrix) + # print dims_size + + # if dims_size == 4: + # tensor.dims.append(dims_of_matrix[0]) # N + # tensor.dims.append(dims_of_matrix[3]) # C + # tensor.dims.append(dims_of_matrix[1]) # H + # tensor.dims.append(dims_of_matrix[2]) # W + # else: + + # issues in mdl model filter swich n and c + if j in self.deepwise_weight_list_ and len(dims_of_matrix) == 4: + print j + tensor.dims.append(dims_of_matrix[1]) + tensor.dims.append(dims_of_matrix[0]) + tensor.dims.append(dims_of_matrix[2]) + tensor.dims.append(dims_of_matrix[3]) + print tensor.dims + else: + for dims in dims_of_matrix: + # print dims + tensor.dims.append(dims) + + if j in self.weight_list_: + vars_add.persistable = 1 + dims_size = len(dims_of_matrix) + # print dims_size + # if dims_size == 4: + # # convert weight from nhwc to nchw + # Swichter().nhwc2nchw_one_slice_add_head( + # 'yolo/datas/multiobjects/float32s_nhwc/' + j + '.bin', + # 'yolo/datas/multiobjects/float32s_nchw_with_head/' + j, + # 'yolo/datas/multiobjects/float32s_nchw/' + j + '.tmp', + # dims_of_matrix[0], + # dims_of_matrix[1], + # dims_of_matrix[2], + # dims_of_matrix[3] + # ) + # else: + # Swichter().copy_add_head( + # 'yolo/datas/multiobjects/float32s_nhwc/' + j + '.bin', + # 'yolo/datas/multiobjects/float32s_nchw_with_head/' + j, + # 'yolo/datas/multiobjects/float32s_nchw/' + j + '.tmp' + # ) + else: + vars_add.persistable = 0 + + +mdl_path = "datas/sourcemodels/cls231_0802/mobileNetModel.json" +base_dir = "/Users/xiebaiyuan/PaddleProject/paddle-mobile/tools/python/modeltools/mobilenet/" +converter = Converter(base_dir, mdl_path) +converter.convert() diff --git a/python/tools/mdl2fluid/swicher.py b/tools/python/modeltools/mobilenet/swicher.py similarity index 86% rename from python/tools/mdl2fluid/swicher.py rename to tools/python/modeltools/mobilenet/swicher.py index bfe0360fd5b32f5e6fa61f6f05a0a384fb3a1e9b..04c10806029c562f429da583dcff7212b94cb162 100644 --- a/python/tools/mdl2fluid/swicher.py +++ b/tools/python/modeltools/mobilenet/swicher.py @@ -58,7 +58,7 @@ class Swichter: to_file = open(to_file_name, "wb") tmp = tmp_file.read() - head = self.read_head('/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/yolo/conv1_biases') + head = self.read_head('yolo/datas/yolo/conv1_biases') to_file.write(head) to_file.write(tmp) tmp_file.close() @@ -77,7 +77,7 @@ class Swichter: to_file = open(to_file_name, "wb") # tmp_file = open(tmp_file_name, "wb") - head = self.read_head('/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/yolo/conv1_biases') + head = self.read_head('yolo/datas/yolo/conv1_biases') to_file.write(head) to_file.write(from_file.read()) from_file.close() @@ -96,7 +96,7 @@ class Swichter: to_file = open(to_file_name, "wb") # tmp_file = open(tmp_file_name, "wb") - head = self.read_head('/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/yolo/conv1_biases') + head = self.read_head('yolo/datas/yolo/conv1_biases') to_file.write(head) to_file.write(read) from_file.close() @@ -104,12 +104,12 @@ class Swichter: pass # Swichter().nhwc2nchw_one_slice_add_head( -# '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nhwc/conv1_0.bin', -# '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nchw_with_head/conv1_0', -# '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nchw/.tmp', +# '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/modeltools/multiobjects/float32s_nhwc/conv1_0.bin', +# '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/modeltools/multiobjects/float32s_nchw_with_head/conv1_0', +# '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/modeltools/multiobjects/float32s_nchw/.tmp', # 32, # 3, 3, 3) -# Swichter().read_head('/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/yolo/conv1_biases') +# Swichter().read_head('/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/modeltools/yolo/conv1_biases') # Swichter().copy_add_head('datas/model.0.0.weight', 'datas/conv1_0', '') diff --git a/tools/python/modeltools/tools/__init__.py b/tools/python/modeltools/tools/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/python/tools/mdl2fluid/float2halffloat.py b/tools/python/modeltools/tools/float2halffloat.py similarity index 100% rename from python/tools/mdl2fluid/float2halffloat.py rename to tools/python/modeltools/tools/float2halffloat.py diff --git a/python/tools/mdl2fluid/loader.py b/tools/python/modeltools/tools/loader.py similarity index 73% rename from python/tools/mdl2fluid/loader.py rename to tools/python/modeltools/tools/loader.py index ef2258e365a84003b7b90ac480abbd9798f48f59..cb996c8bedd78004e667f1433bfdb20785e7792f 100644 --- a/python/tools/mdl2fluid/loader.py +++ b/tools/python/modeltools/tools/loader.py @@ -1,9 +1,4 @@ -import datetime import json -import os - -import google.protobuf as pbg -import framework_pb2 as framework_pb2 def loadmdl(json_path): diff --git a/python/tools/mdl2fluid/model_combine.py b/tools/python/modeltools/tools/model_combine.py similarity index 100% rename from python/tools/mdl2fluid/model_combine.py rename to tools/python/modeltools/tools/model_combine.py diff --git a/python/tools/mdl2fluid/model_reader.py b/tools/python/modeltools/tools/model_reader.py similarity index 71% rename from python/tools/mdl2fluid/model_reader.py rename to tools/python/modeltools/tools/model_reader.py index 8d53350db20739526b77663f791942299d4bc149..5f6e5f0cb9da8fb349e35211ed56f77bb9cf95da 100644 --- a/python/tools/mdl2fluid/model_reader.py +++ b/tools/python/modeltools/tools/model_reader.py @@ -1,6 +1,6 @@ import os -import framework_pb2 as framework_pb2 +from core import framework_pb2 as framework_pb2 def read_model(model_path): @@ -16,7 +16,7 @@ def read_model(model_path): # print desc.blocks except IOError: - print ": File not found. Creating a new file." + print ": File not found." def get_file_size(file_path): @@ -26,5 +26,5 @@ def get_file_size(file_path): return round(fsize, 2) -path = "newyolo/__model__" +path = '/Users/xiebaiyuan/PaddleProject/paddle-mobile/tools/python/modeltools/mobilenet/datas/sourcemodels/mobilenet_example/mobilenet/__model__' read_model(path) diff --git a/tools/python/modeltools/yolo/__init__.py b/tools/python/modeltools/yolo/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/python/tools/mdl2fluid/mdl2fluid.py b/tools/python/modeltools/yolo/mdl2fluid.py similarity index 89% rename from python/tools/mdl2fluid/mdl2fluid.py rename to tools/python/modeltools/yolo/mdl2fluid.py index a57a01d09eaf236fd9f890dcb9e8eead19aa7868..2c2d0f3e9498254f26da6ff1b88b8a33e1b31d27 100644 --- a/python/tools/mdl2fluid/mdl2fluid.py +++ b/tools/python/modeltools/yolo/mdl2fluid.py @@ -1,9 +1,7 @@ import json -import os -import framework_pb2 as framework_pb2 -import op_types as types -from swicher import Swichter +from core import framework_pb2 as framework_pb2, op_types as types +from yolo.swicher import Swichter import shutil @@ -40,10 +38,10 @@ class Converter: print self.program_desc.blocks print 'convert end.....' desc_serialize_to_string = self.program_desc.SerializeToString() - shutil.rmtree('newyolo/') - shutil.copytree('multiobjects/float32s_nchw_with_head', 'newyolo/') + shutil.rmtree('yolo/datas/newyolo/') + shutil.copytree('yolo/datas/multiobjects/float32s_nchw_with_head/', 'yolo/datas/newyolo/') - f = open("newyolo/__model__", "wb") + f = open("yolo/datas/newyolo/__model__", "wb") f.write(desc_serialize_to_string) f.close() @@ -312,9 +310,9 @@ class Converter: if dims_size == 4: # convert weight from nhwc to nchw Swichter().nhwc2nchw_one_slice_add_head( - '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nhwc/' + j + '.bin', - '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nchw_with_head/' + j, - '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nchw/' + j + '.tmp', + 'yolo/datas/multiobjects/float32s_nhwc/' + j + '.bin', + 'yolo/datas/multiobjects/float32s_nchw_with_head/' + j, + 'yolo/datas/multiobjects/float32s_nchw/' + j + '.tmp', dims_of_matrix[0], dims_of_matrix[1], dims_of_matrix[2], @@ -322,14 +320,14 @@ class Converter: ) else: Swichter().copy_add_head( - '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nhwc/' + j + '.bin', - '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nchw_with_head/' + j, - '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nchw/' + j + '.tmp' + 'yolo/datas/multiobjects/float32s_nhwc/' + j + '.bin', + 'yolo/datas/multiobjects/float32s_nchw_with_head/' + j, + 'yolo/datas/multiobjects/float32s_nchw/' + j + '.tmp' ) else: vars_add.persistable = 0 -mdl_path = "/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/YOLO_Universal.json" +mdl_path = "yolo/datas/multiobjects/YOLO_Universal.json" converter = Converter(mdl_path) converter.convert() diff --git a/tools/python/modeltools/yolo/swicher.py b/tools/python/modeltools/yolo/swicher.py new file mode 100644 index 0000000000000000000000000000000000000000..04c10806029c562f429da583dcff7212b94cb162 --- /dev/null +++ b/tools/python/modeltools/yolo/swicher.py @@ -0,0 +1,115 @@ +from array import array + + +class Swichter: + def __init__(self): + pass + + def nhwc2nchw_one_slice(self, from_file_name, to_file_name, batch, channel, height, width): + from_file = open(from_file_name, "rb") + to_file = open(to_file_name, "wb") + + float_array = array("f") + float_array.fromfile(from_file, width * height * batch * channel) + float_write_array = array("f") + + for b in range(batch): + for c in range(channel): + for h in range(height): + for w in range(width): + float_value = float_array[b * channel * width * height + + channel * (h * width + w) + c] + + float_write_array.append(float_value) + + float_write_array.tofile(to_file) + from_file.close() + to_file.close() + + def copy(self, from_file_name, to_file_name): + from_file = open(from_file_name, "rb") + to_file = open(to_file_name, "wb") + + to_file.write(from_file.read()) + from_file.close() + to_file.close() + + def nhwc2nchw_one_slice_add_head(self, from_file_name, to_file_name, tmp_file_name, batch, channel, height, width): + from_file = open(from_file_name, "rb") + tmp_file = open(tmp_file_name, "wb+") + float_array = array("f") + float_array.fromfile(from_file, width * height * batch * channel) + float_write_array = array("f") + + for b in range(batch): + for c in range(channel): + for h in range(height): + for w in range(width): + float_value = float_array[b * channel * width * height + + channel * (h * width + w) + c] + + float_write_array.append(float_value) + + float_write_array.tofile(tmp_file) + tmp_file.close() + from_file.close() + + tmp_file = open(tmp_file_name, "rb") + to_file = open(to_file_name, "wb") + + tmp = tmp_file.read() + head = self.read_head('yolo/datas/yolo/conv1_biases') + to_file.write(head) + to_file.write(tmp) + tmp_file.close() + to_file.close() + + def read_head(self, head_file): + from_file = open(head_file, "rb") + read = from_file.read(24) + # print read + from_file.close() + # print read + return read + + def copy_add_head(self, from_file_name, to_file_name, tmp_file_name): + from_file = open(from_file_name, "rb") + to_file = open(to_file_name, "wb") + # tmp_file = open(tmp_file_name, "wb") + + head = self.read_head('yolo/datas/yolo/conv1_biases') + to_file.write(head) + to_file.write(from_file.read()) + from_file.close() + to_file.close() + pass + + def copy_padding_add_head(self, from_file_name, to_file_name, tmp_file_name, padding): + print'padding = %d' % padding + from_file = open(from_file_name, "rb") + # print len(from_file.read()) + from_file.seek(padding, 0) + + read = from_file.read() + print len(read) + + to_file = open(to_file_name, "wb") + # tmp_file = open(tmp_file_name, "wb") + + head = self.read_head('yolo/datas/yolo/conv1_biases') + to_file.write(head) + to_file.write(read) + from_file.close() + to_file.close() + pass + +# Swichter().nhwc2nchw_one_slice_add_head( +# '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/modeltools/multiobjects/float32s_nhwc/conv1_0.bin', +# '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/modeltools/multiobjects/float32s_nchw_with_head/conv1_0', +# '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/modeltools/multiobjects/float32s_nchw/.tmp', +# 32, +# 3, 3, 3) + +# Swichter().read_head('/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/modeltools/yolo/conv1_biases') + +# Swichter().copy_add_head('datas/model.0.0.weight', 'datas/conv1_0', '')