convert

python/tools/mdl2fluid/float16_float32.py

+import binascii
+import os
+import numpy as np
+
+
+def read_param(path):
+    try:
+        with open(path, "r") as f:
+            value = f.read(2)
+            a_hex = binascii.b2a_hex(value)
+            print a_hex
+
+
+            # value = f.read(2)
+            # a_hex = binascii.b2a_hex(value)
+            # print a_hex
+            # value = f.read(2)
+            # a_hex = binascii.b2a_hex(value)
+            # print a_hex
+
+    except IOError:
+        print ": File not found."
+
+
+def get_file_size(file_path):
+    file_path = unicode(file_path, 'utf8')
+    f_size = os.path.getsize(file_path)
+    f_size = f_size / float(1024 * 1024)
+    return round(f_size, 2)
+
+
+read_param(
+    "/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/YOLOParameters_Universal"
+    ".bundle/conv1_0.bin")

python/tools/mdl2fluid/float2halffloat.py

+# encoding:utf-8
+import math
+import re
+
+
+def Real2HalfFloat(data):
+    MINNUM = -65536
+    MAXNUM = 65535
+    FloatVal = 0
+    if data:
+        if data < MINNUM:
+            data = MINNUM
+        if data > MAXNUM:
+            data = MAXNUM
+
+        sign = 0
+        if data < 0:
+            sign = 1
+            data = -data
+
+        exp = math.floor((math.log2(data)))
+        expout = exp + 16
+
+        Mantial = round(data / pow(2, exp - 10)) - 1024
+
+        if expout <= 0:
+            FloatVal = 0
+        else:
+            FloatVal = sign * 32768 + expout * 1024 + Mantial
+    return FloatVal
+
+
+def ReadCfloatData(sourcefile):
+    input = []
+    with open(sourcfile, 'r') as f:
+        for line in f.readlines():
+            line = line.strip()
+            line = re.sub('\s+', ' ', line)  # 两个数字间多个空格
+            input.append(line.split(' '))
+    destfile = sourcefile.replace('.dat', '')
+    destfile = destfile.replace('.txt', '')
+    destfile += 'Out.dat'
+    with open(destfile, 'w') as fw:
+        for i in range(len(input)):
+            if len(input[i]) == 2:
+                real = Real2HalfFloat(float(input[i][0]))
+                imag = Real2HalfFloat(float(input[i][1]))
+                result = real * 65536 + imag
+                if imag and not real:
+                    fw.write('0x0000' + "%X" % result + '\n')
+                elif not imag and not real:
+                    fw.write('0x00000000' + '\n')
+                else:
+                    fw.write('0x' + "%X" % result + '\n')
+            elif len(input[i]) == 1:
+                result = Real2HalfFloat(float(input[i][0]))
+                if result:
+                    fw.write('0x' + "%X" % result + '\n')
+                else:
+                    fw.write('0x0000' + '\n')
+
+
+if __name__ == '__main__':
+    print('Tips: Input number 0 if you want to exit!\n')
+    while True:
+        sourcfile = input("input source file:\n")
+        if sourcfile is '0':
+            break
+        ReadCfloatData(sourcfile)
+        print('Transfer Success!')

python/tools/mdl2fluid/mdl2fluid.py

 import json
+import os
+
 import framework_pb2 as framework_pb2
 import op_types as types
+from swicher import Swichter
+import shutil
 
 
 def load_mdl(mdl_json_path):
@@ -35,11 +39,14 @@ class Converter:
         print self.program_desc.blocks
         print 'convert end.....'
         desc_serialize_to_string = self.program_desc.SerializeToString()
+        shutil.rmtree('newyolo/')
+        shutil.copytree('multiobjects/float32s_nchw_with_head', 'newyolo/')
 
         f = open("newyolo/__model__", "wb")
         f.write(desc_serialize_to_string)
         f.close()
 
+
     def package_ops(self, block_desc):
 
         self.add_op_feed(block_desc)
@@ -241,15 +248,6 @@ class Converter:
         desc_ops_add.type = types.mdl2fluid_op_layer_dict.get(l_type)
 
     def package_vars(self, block_desc):
-        # feed
-        # vars
-        # {
-        #     name: "feed"
-        #     type {
-        #         type: FEED_MINIBATCH
-        #     }
-        #     persistable: true
-        # }
         vars_add = block_desc.vars.add()
         vars_add.name = 'feed'
         vars_add.type.type = 9  # 9 is FEED_MINIBATCH
@@ -266,53 +264,52 @@ class Converter:
             vars_add = block_desc.vars.add()
             vars_add.name = j
             vars_add.type.type = 7  # 7 is lodtensor
-            # print j
+            print j
             tensor = vars_add.type.lod_tensor.tensor
             tensor.data_type = 5  # 5 is FP32
-            for dims in json_matrix_.get(j):
+
+            # print json_matrix_
+
+            dims_of_matrix = json_matrix_.get(j)
+            # dims_size = len(dims_of_matrix)
+            # print dims_size
+
+            # if dims_size == 4:
+            #     tensor.dims.append(dims_of_matrix[0])  # N
+            #     tensor.dims.append(dims_of_matrix[3])  # C
+            #     tensor.dims.append(dims_of_matrix[1])  # H
+            #     tensor.dims.append(dims_of_matrix[2])  # W
+            # else:
+            for dims in dims_of_matrix:
+                # print dims
                 tensor.dims.append(dims)
+
             if j in self.weight_list_:
                 vars_add.persistable = 1
+                # todo parweight channel
+                dims_size = len(dims_of_matrix)
+                # print dims_size
+                if dims_size == 4:
+                    # convert weight from nhwc to nchw
+                    Swichter().nhwc2nchw_one_slice_add_head(
+                        '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nhwc/' + j + '.bin',
+                        '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nchw_with_head/' + j,
+                        '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nchw/' + j + '.tmp',
+                        dims_of_matrix[0],
+                        dims_of_matrix[1],
+                        dims_of_matrix[2],
+                        dims_of_matrix[3]
+                    )
+                else:
+                    Swichter().copy_add_head(
+                        '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nhwc/' + j + '.bin',
+                        '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nchw_with_head/' + j,
+                        '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nchw/' + j + '.tmp'
+                    )
             else:
                 vars_add.persistable = 0
 
 
-# print mdl_path
-# # model
-# mdl_model = load_mdl(mdl_path)
-# for key in mdl_model:
-#     print key
-#
-# # layer
-# layers = mdl_model['layer']
-# print layers
-#
-# for layer in layers:
-#     print layer
-#     for i in layer:
-#         print i
-#     if 'name' in layer:
-#         l_name = layer['name']
-#
-#     if 'weight' in layer:
-#         l_weights = layer['weight']
-#
-#     if 'param' in layer:
-#         l_params = layer['param']
-#
-#     if 'output' in layer:
-#         l_outputs = layer['output']
-#
-#     if 'input' in layer:
-#         l_inputs = layer['input']
-#
-#     if 'type' in layer:
-#         l_type = layer['type']
-#
-# print mdl_model['matrix']
-#
-# package()
-
 mdl_path = "/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/YOLO_Universal.json"
 converter = Converter(mdl_path)
 converter.convert()

python/tools/mdl2fluid/swicher.py

+from array import array
+
+
+class Swichter:
+    def __init__(self):
+        pass
+
+    def nhwc2nchw_one_slice(self, from_file_name, to_file_name, batch, channel, height, width):
+        from_file = open(from_file_name, "rb")
+        to_file = open(to_file_name, "wb")
+
+        float_array = array("f")
+        float_array.fromfile(from_file, width * height * batch * channel)
+        float_write_array = array("f")
+
+        for b in range(batch):
+            for c in range(channel):
+                for h in range(height):
+                    for w in range(width):
+                        float_value = float_array[b * channel * width * height
+                                                  + channel * (h * width + w) + c]
+
+                        float_write_array.append(float_value)
+
+        float_write_array.tofile(to_file)
+        from_file.close()
+        to_file.close()
+
+    def copy(self, from_file_name, to_file_name):
+        from_file = open(from_file_name, "rb")
+        to_file = open(to_file_name, "wb")
+
+        to_file.write(from_file.read())
+        from_file.close()
+        to_file.close()
+
+    def nhwc2nchw_one_slice_add_head(self, from_file_name, to_file_name, tmp_file_name, batch, channel, height, width):
+        from_file = open(from_file_name, "rb")
+        tmp_file = open(tmp_file_name, "wb+")
+        float_array = array("f")
+        float_array.fromfile(from_file, width * height * batch * channel)
+        float_write_array = array("f")
+
+        for b in range(batch):
+            for c in range(channel):
+                for h in range(height):
+                    for w in range(width):
+                        float_value = float_array[b * channel * width * height
+                                                  + channel * (h * width + w) + c]
+
+                        float_write_array.append(float_value)
+
+        float_write_array.tofile(tmp_file)
+        tmp_file.close()
+        from_file.close()
+
+        tmp_file = open(tmp_file_name, "rb")
+        to_file = open(to_file_name, "wb")
+
+        tmp = tmp_file.read()
+        head = self.read_head('/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/yolo/conv1_biases')
+        to_file.write(head)
+        to_file.write(tmp)
+        tmp_file.close()
+        to_file.close()
+
+    def read_head(self, head_file):
+        from_file = open(head_file, "rb")
+        read = from_file.read(20)
+        print read
+        from_file.close()
+        print read
+        return read
+
+    def copy_add_head(self, from_file_name, to_file_name, tmp_file_name):
+        from_file = open(from_file_name, "rb")
+        to_file = open(to_file_name, "wb")
+        # tmp_file = open(tmp_file_name, "wb")
+
+        head = self.read_head('/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/yolo/conv1_biases')
+        to_file.write(head)
+        to_file.write(from_file.read())
+        from_file.close()
+        to_file.close()
+        pass
+
+
+# Swichter().nhwc2nchw_one_slice(
+#     '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nhwc/conv5_6_dw_0.bin',
+#     '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nchw/conv5_6_dw_0', 1,
+#     512, 3, 3)
+Swichter().read_head('/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/yolo/conv1_biases')