From 64de944f70595ff732684908331bb8e3e3ab978e Mon Sep 17 00:00:00 2001 From: xiebaiyuan Date: Wed, 19 Sep 2018 16:38:18 +0800 Subject: [PATCH] convert --- python/tools/mdl2fluid/float16_float32.py | 34 +++++++++ python/tools/mdl2fluid/float2halffloat.py | 70 +++++++++++++++++ python/tools/mdl2fluid/mdl2fluid.py | 91 +++++++++++----------- python/tools/mdl2fluid/swicher.py | 92 +++++++++++++++++++++++ 4 files changed, 240 insertions(+), 47 deletions(-) create mode 100644 python/tools/mdl2fluid/float16_float32.py create mode 100644 python/tools/mdl2fluid/float2halffloat.py create mode 100644 python/tools/mdl2fluid/swicher.py diff --git a/python/tools/mdl2fluid/float16_float32.py b/python/tools/mdl2fluid/float16_float32.py new file mode 100644 index 0000000000..e536238955 --- /dev/null +++ b/python/tools/mdl2fluid/float16_float32.py @@ -0,0 +1,34 @@ +import binascii +import os +import numpy as np + + +def read_param(path): + try: + with open(path, "r") as f: + value = f.read(2) + a_hex = binascii.b2a_hex(value) + print a_hex + + + # value = f.read(2) + # a_hex = binascii.b2a_hex(value) + # print a_hex + # value = f.read(2) + # a_hex = binascii.b2a_hex(value) + # print a_hex + + except IOError: + print ": File not found." + + +def get_file_size(file_path): + file_path = unicode(file_path, 'utf8') + f_size = os.path.getsize(file_path) + f_size = f_size / float(1024 * 1024) + return round(f_size, 2) + + +read_param( + "/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/YOLOParameters_Universal" + ".bundle/conv1_0.bin") diff --git a/python/tools/mdl2fluid/float2halffloat.py b/python/tools/mdl2fluid/float2halffloat.py new file mode 100644 index 0000000000..3df8d43f95 --- /dev/null +++ b/python/tools/mdl2fluid/float2halffloat.py @@ -0,0 +1,70 @@ +# encoding:utf-8 +import math +import re + + +def Real2HalfFloat(data): + MINNUM = -65536 + MAXNUM = 65535 + FloatVal = 0 + if data: + if data < MINNUM: + data = MINNUM + if data > MAXNUM: + data = MAXNUM + + sign = 0 + if data < 0: + sign = 1 + data = -data + + exp = math.floor((math.log2(data))) + expout = exp + 16 + + Mantial = round(data / pow(2, exp - 10)) - 1024 + + if expout <= 0: + FloatVal = 0 + else: + FloatVal = sign * 32768 + expout * 1024 + Mantial + return FloatVal + + +def ReadCfloatData(sourcefile): + input = [] + with open(sourcfile, 'r') as f: + for line in f.readlines(): + line = line.strip() + line = re.sub('\s+', ' ', line) # 两个数字间多个空格 + input.append(line.split(' ')) + destfile = sourcefile.replace('.dat', '') + destfile = destfile.replace('.txt', '') + destfile += 'Out.dat' + with open(destfile, 'w') as fw: + for i in range(len(input)): + if len(input[i]) == 2: + real = Real2HalfFloat(float(input[i][0])) + imag = Real2HalfFloat(float(input[i][1])) + result = real * 65536 + imag + if imag and not real: + fw.write('0x0000' + "%X" % result + '\n') + elif not imag and not real: + fw.write('0x00000000' + '\n') + else: + fw.write('0x' + "%X" % result + '\n') + elif len(input[i]) == 1: + result = Real2HalfFloat(float(input[i][0])) + if result: + fw.write('0x' + "%X" % result + '\n') + else: + fw.write('0x0000' + '\n') + + +if __name__ == '__main__': + print('Tips: Input number 0 if you want to exit!\n') + while True: + sourcfile = input("input source file:\n") + if sourcfile is '0': + break + ReadCfloatData(sourcfile) + print('Transfer Success!') diff --git a/python/tools/mdl2fluid/mdl2fluid.py b/python/tools/mdl2fluid/mdl2fluid.py index 6a56de13f8..1b8b4cc896 100644 --- a/python/tools/mdl2fluid/mdl2fluid.py +++ b/python/tools/mdl2fluid/mdl2fluid.py @@ -1,6 +1,10 @@ import json +import os + import framework_pb2 as framework_pb2 import op_types as types +from swicher import Swichter +import shutil def load_mdl(mdl_json_path): @@ -35,11 +39,14 @@ class Converter: print self.program_desc.blocks print 'convert end.....' desc_serialize_to_string = self.program_desc.SerializeToString() + shutil.rmtree('newyolo/') + shutil.copytree('multiobjects/float32s_nchw_with_head', 'newyolo/') f = open("newyolo/__model__", "wb") f.write(desc_serialize_to_string) f.close() + def package_ops(self, block_desc): self.add_op_feed(block_desc) @@ -241,15 +248,6 @@ class Converter: desc_ops_add.type = types.mdl2fluid_op_layer_dict.get(l_type) def package_vars(self, block_desc): - # feed - # vars - # { - # name: "feed" - # type { - # type: FEED_MINIBATCH - # } - # persistable: true - # } vars_add = block_desc.vars.add() vars_add.name = 'feed' vars_add.type.type = 9 # 9 is FEED_MINIBATCH @@ -266,53 +264,52 @@ class Converter: vars_add = block_desc.vars.add() vars_add.name = j vars_add.type.type = 7 # 7 is lodtensor - # print j + print j tensor = vars_add.type.lod_tensor.tensor tensor.data_type = 5 # 5 is FP32 - for dims in json_matrix_.get(j): + + # print json_matrix_ + + dims_of_matrix = json_matrix_.get(j) + # dims_size = len(dims_of_matrix) + # print dims_size + + # if dims_size == 4: + # tensor.dims.append(dims_of_matrix[0]) # N + # tensor.dims.append(dims_of_matrix[3]) # C + # tensor.dims.append(dims_of_matrix[1]) # H + # tensor.dims.append(dims_of_matrix[2]) # W + # else: + for dims in dims_of_matrix: + # print dims tensor.dims.append(dims) + if j in self.weight_list_: vars_add.persistable = 1 + # todo parweight channel + dims_size = len(dims_of_matrix) + # print dims_size + if dims_size == 4: + # convert weight from nhwc to nchw + Swichter().nhwc2nchw_one_slice_add_head( + '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nhwc/' + j + '.bin', + '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nchw_with_head/' + j, + '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nchw/' + j + '.tmp', + dims_of_matrix[0], + dims_of_matrix[1], + dims_of_matrix[2], + dims_of_matrix[3] + ) + else: + Swichter().copy_add_head( + '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nhwc/' + j + '.bin', + '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nchw_with_head/' + j, + '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nchw/' + j + '.tmp' + ) else: vars_add.persistable = 0 -# print mdl_path -# # model -# mdl_model = load_mdl(mdl_path) -# for key in mdl_model: -# print key -# -# # layer -# layers = mdl_model['layer'] -# print layers -# -# for layer in layers: -# print layer -# for i in layer: -# print i -# if 'name' in layer: -# l_name = layer['name'] -# -# if 'weight' in layer: -# l_weights = layer['weight'] -# -# if 'param' in layer: -# l_params = layer['param'] -# -# if 'output' in layer: -# l_outputs = layer['output'] -# -# if 'input' in layer: -# l_inputs = layer['input'] -# -# if 'type' in layer: -# l_type = layer['type'] -# -# print mdl_model['matrix'] -# -# package() - mdl_path = "/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/YOLO_Universal.json" converter = Converter(mdl_path) converter.convert() diff --git a/python/tools/mdl2fluid/swicher.py b/python/tools/mdl2fluid/swicher.py new file mode 100644 index 0000000000..b3634d106c --- /dev/null +++ b/python/tools/mdl2fluid/swicher.py @@ -0,0 +1,92 @@ +from array import array + + +class Swichter: + def __init__(self): + pass + + def nhwc2nchw_one_slice(self, from_file_name, to_file_name, batch, channel, height, width): + from_file = open(from_file_name, "rb") + to_file = open(to_file_name, "wb") + + float_array = array("f") + float_array.fromfile(from_file, width * height * batch * channel) + float_write_array = array("f") + + for b in range(batch): + for c in range(channel): + for h in range(height): + for w in range(width): + float_value = float_array[b * channel * width * height + + channel * (h * width + w) + c] + + float_write_array.append(float_value) + + float_write_array.tofile(to_file) + from_file.close() + to_file.close() + + def copy(self, from_file_name, to_file_name): + from_file = open(from_file_name, "rb") + to_file = open(to_file_name, "wb") + + to_file.write(from_file.read()) + from_file.close() + to_file.close() + + def nhwc2nchw_one_slice_add_head(self, from_file_name, to_file_name, tmp_file_name, batch, channel, height, width): + from_file = open(from_file_name, "rb") + tmp_file = open(tmp_file_name, "wb+") + float_array = array("f") + float_array.fromfile(from_file, width * height * batch * channel) + float_write_array = array("f") + + for b in range(batch): + for c in range(channel): + for h in range(height): + for w in range(width): + float_value = float_array[b * channel * width * height + + channel * (h * width + w) + c] + + float_write_array.append(float_value) + + float_write_array.tofile(tmp_file) + tmp_file.close() + from_file.close() + + tmp_file = open(tmp_file_name, "rb") + to_file = open(to_file_name, "wb") + + tmp = tmp_file.read() + head = self.read_head('/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/yolo/conv1_biases') + to_file.write(head) + to_file.write(tmp) + tmp_file.close() + to_file.close() + + def read_head(self, head_file): + from_file = open(head_file, "rb") + read = from_file.read(20) + print read + from_file.close() + print read + return read + + def copy_add_head(self, from_file_name, to_file_name, tmp_file_name): + from_file = open(from_file_name, "rb") + to_file = open(to_file_name, "wb") + # tmp_file = open(tmp_file_name, "wb") + + head = self.read_head('/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/yolo/conv1_biases') + to_file.write(head) + to_file.write(from_file.read()) + from_file.close() + to_file.close() + pass + + +# Swichter().nhwc2nchw_one_slice( +# '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nhwc/conv5_6_dw_0.bin', +# '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nchw/conv5_6_dw_0', 1, +# 512, 3, 3) +Swichter().read_head('/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/yolo/conv1_biases') -- GitLab