current convert

tools/python/imagetools/img2nchw.py

@@ -45,13 +45,13 @@ def combine_bgrs_nchw(bgrs, means_b_g_r, scale, channel_type=ChannelType.BGR):
 
     print '------------------'
     print bgrs_float_array[0]
-    print bgrs_float_array[416 * 416 * 2 + 416 * 2 + 2]
+    print bgrs_float_array[224 * 224 * 2 + 224 * 2 + 2]
 
     # for i in range(0, 9):
     #     print'bs %d' % i
     #     print bs[i] / 255.
 
-    print bs[416 * 2 + 2] / 255.
+    print bs[224 * 2 + 2] / 255.
     print '--------------combine_bgrs_nchw-----------------end'
 
     return bgrs_float_array
@@ -64,6 +64,6 @@ def combine_bgrs_nchw(bgrs, means_b_g_r, scale, channel_type=ChannelType.BGR):
 # cv2.waitKey(0)
 
 
-bgrs = tools.resize_take_rgbs('datas/newyolo.jpg', (416, 416, 3))
+bgrs = tools.resize_take_rgbs('datas/jpgs/0000_0.9834-148196_82452-0ad4b83ec6bc0f9c5f28101539267054.jpg_p0_0.126571263346.jpg', (224, 224, 3))
 array = combine_bgrs_nchw(bgrs, (0, 0, 0), 1. / 255, ChannelType.RGB)
-tools.save_to_file('datas/desktop_1_3_416_416_nchw_float', array)
+tools.save_to_file('datas/desktop_1_3_224_224_nchw_float', array)

tools/python/imagetools/numpy2binary.py

@@ -15,11 +15,11 @@ from array import array
 # image.resize(shape_h_w)
 
 
-data = np.fromfile('datas/img.res')
+data = np.fromfile('/Users/xiebaiyuan/PaddleProject/paddle-mobile/tools/python/imagetools/datas/jpgs2/0000_0.9834-148196_82452-0ad4b83ec6bc0f9c5f28101539267054.jpg_p0_0.126571263346.jpg.input.npfile','f')
 print data.size
-print data[0]
+print data
 
-data.reshape(1, 3, 416, 416)
+data.reshape(1, 3, 224, 224)
 out_array = array('f')
 print'--------------------'
 print data.size
@@ -27,12 +27,12 @@ print data[0]
 
 print '如果是nhwc --------'
 # rgb rgb rgb rgb rgb
-print data[416 * 3 * 2 + 3 * 2 + 2]
+print data[224 * 3 * 2 + 3 * 2 + 2]
 # print data[2]
 
 print '如果是nchw --------'
 # rgb rgb rgb rgb rgb
-print data[416 * 416 * 2 + 416 * 2 + 2]
+print data[224 * 224 * 2 + 224 * 2 + 2]
 # print data[2]
 
 # 明明是nchw
@@ -42,6 +42,8 @@ for i in range(0, data.size):
 
 print len(out_array)
 
-print out_array[416 * 416 * 2 + 416 * 2 + 2]
+print out_array[224 * 224 * 2 + 224 * 2 + 2]
+
+# print out_array
 
-tools.save_to_file('datas/in_put_1_3_416_416_2', out_array)
+tools.save_to_file('datas/in_put_1_3_224_224_nchw', out_array)

tools/python/modeltools/core/op_types.py

@@ -77,6 +77,14 @@ fusion_conv_add_attrs_dict = {
     'strides': 'stride',
     'groups': 'group'
 }
+
+# fluid attr key  ---  mdl params key
+pool2d_attrs_dict = {
+    'global_pooling': 'global_pooling',
+    'pooling_type': 'type'
+}
+
+
 # fluid attr key  ---  mdl params key
 fluid_attrs_type_dict = {
     'paddings': 0,

tools/python/modeltools/mobilenet/converter_mobilenet.py

+# coding=utf-8
 import json
 import os
 
@@ -12,13 +13,25 @@ def load_mdl(mdl_json_path):
         return json.load(f)
 
 
+def create_if_not_exit(target_dir):
+    if os.path.exists(target_dir):
+        shutil.rmtree(target_dir)
+    os.makedirs(target_dir, 0777)
+
+
 class Converter:
     'convert mdlmodel to fluidmodel'
 
     def __init__(self, base_dir, mdl_json_path):
+        print 'base_dir:  ' + base_dir
         self.mdl_json_path = base_dir + mdl_json_path
         self.base_dir = base_dir
         print mdl_json_path
+        self.source_weights_dir = self.base_dir + 'datas/sourcemodels/source_weights/'
+        self.target_weight_dir = self.base_dir + 'datas/target/target_weights/'
+
+        create_if_not_exit(self.target_weight_dir)
+
         self.mdl_json = load_mdl(self.mdl_json_path)
         self.program_desc = framework_pb2.ProgramDesc()
         self.weight_list_ = []
@@ -41,16 +54,18 @@ class Converter:
         print 'convert end.....'
         desc_serialize_to_string = self.program_desc.SerializeToString()
 
-        outputmodel_ = self.base_dir + 'datas/target/outputmodel/'
-        if os.path.exists(outputmodel_):
-            shutil.rmtree(outputmodel_)
-        os.makedirs(outputmodel_, 0777)
-        # todo copy weight files
-        # if os.path.exists(outputmodel_):
-        #     shutil.rmtree(outputmodel_)
-        # shutil.copytree('yolo/datas/multiobjects/float32s_nchw_with_head/', 'mobilenet/datas/target/outputmodel/')
+        outputmodel_dir = self.base_dir + 'datas/target/mobilenet_classfication/'
+        if os.path.exists(outputmodel_dir):
+            shutil.rmtree(outputmodel_dir)
+        os.makedirs(outputmodel_dir, 0777)
 
-        f = open(outputmodel_ + "__model__", "wb")
+        if os.path.exists(outputmodel_dir):
+            shutil.rmtree(outputmodel_dir)
+        # create_if_not_exit(outputmodel_dir)
+
+        shutil.copytree(self.target_weight_dir, outputmodel_dir)
+
+        f = open(outputmodel_dir + "__model__", "wb")
         f.write(desc_serialize_to_string)
         f.close()
 
@@ -105,7 +120,8 @@ class Converter:
         desc_ops_add = block_desc.ops.add()
         inputs_add = desc_ops_add.inputs.add()
         inputs_add.parameter = 'X'
-        inputs_add.arguments.append('conv_pred_87')
+        # todo pick last layer --> op output
+        inputs_add.arguments.append('Softmax')
         desc_ops_add.type = 'fetch'
         outputs_add = desc_ops_add.outputs.add()
         outputs_add.parameter = 'Out'
@@ -129,6 +145,128 @@ class Converter:
             # boolean
             attrs_add.type = 6
             attrs_add.b = 0
+        elif desc_ops_add.type == types.op_fluid_pooling:
+            Converter.pack_pooling_attr(desc_ops_add, layer)
+            pass
+        elif desc_ops_add.type == types.op_fluid_softmax:
+            pass
+
+    @staticmethod
+    def pack_pooling_attr(desc_ops_add, layer):
+        print layer
+        l_params = layer['param']
+
+        attrs_add = desc_ops_add.attrs.add()
+        attrs_add.name = 'use_mkldnn'
+        # boolean
+        attrs_add.type = 6
+        attrs_add.b = 0
+
+        attrs_add = desc_ops_add.attrs.add()
+        attrs_add.name = 'use_cudnn'
+        # boolean
+        attrs_add.type = 6
+        attrs_add.b = 1
+
+        attrs_add = desc_ops_add.attrs.add()
+        attrs_add.name = 'paddings'
+        # ints
+        attrs_add.type = 3
+        attrs_add.ints.append(0)
+        attrs_add.ints.append(0)
+
+        attrs_add = desc_ops_add.attrs.add()
+        attrs_add.name = 'strides'
+        # ints
+        attrs_add.type = 3
+        attrs_add.ints.append(1)
+        attrs_add.ints.append(1)
+
+        attrs_add = desc_ops_add.attrs.add()
+        attrs_add.name = 'global_pooling'
+        # boolean
+        attrs_add.type = 6
+        attrs_add.b = (l_params[types.pool2d_attrs_dict.get('global_pooling')])
+
+        attrs_add = desc_ops_add.attrs.add()
+        attrs_add.name = 'pooling_type'
+        # 2-->STRING
+        attrs_add.type = 2
+        # 注意这里 avg but mdl is ave
+        attrs_add.s = l_params[types.pool2d_attrs_dict.get('pooling_type')]
+
+        attrs_add = desc_ops_add.attrs.add()
+        attrs_add.name = 'ceil_mode'
+        # boolean
+        attrs_add.type = 6
+        attrs_add.b = 1
+
+        attrs_add = desc_ops_add.attrs.add()
+        attrs_add.name = 'ksize'
+        # ints
+        attrs_add.type = 3
+        attrs_add.ints.append(7)
+        attrs_add.ints.append(7)
+
+    # type: "pool2d"
+    # attrs
+    # {
+    #     name: "use_mkldnn"
+    #     type: BOOLEAN
+    #     b: false
+    # }
+    # attrs
+    # {
+    #     name: "ceil_mode"
+    #     type: BOOLEAN
+    #     b: true
+    # }
+    # attrs
+    # {
+    #     name: "use_cudnn"
+    #     type: BOOLEAN
+    #     b: true
+    # }
+    # attrs
+    # {
+    #     name: "paddings"
+    #     type: INTS
+    #     ints: 0
+    #     ints: 0
+    # }
+    # attrs
+    # {
+    #     name: "strides"
+    #     type: INTS
+    #     ints: 1
+    #     ints: 1
+    # }
+    # attrs
+    # {
+    #     name: "global_pooling"
+    #     type: BOOLEAN
+    #     b: false
+    # }
+    # attrs
+    # {
+    #     name: "data_format"
+    #     type: STRING
+    #     s: "AnyLayout"
+    # }
+    # attrs
+    # {
+    #     name: "ksize"
+    #     type: INTS
+    #     ints: 7
+    #     ints: 7
+    # }
+    # attrs
+    # {
+    #     name: "pooling_type"
+    #     type: STRING
+    #     s: "avg"
+    # }
+    # is_target: false
 
     @staticmethod
     def pack_fusion_conv_add_attr(desc_ops_add, layer):
@@ -181,6 +319,13 @@ class Converter:
             attrs_add.ints.append(l_params[types.fusion_conv_add_attrs_dict.get('paddings')])
             attrs_add.ints.append(l_params[types.fusion_conv_add_attrs_dict.get('paddings')])
 
+            # attrs_add = desc_ops_add.attrs.add()
+            # attrs_add.name = 'paddings'
+            # # ints
+            # attrs_add.type = 3
+            # attrs_add.ints.append(0)
+            # attrs_add.ints.append(0)
+
             attrs_add = desc_ops_add.attrs.add()
             attrs_add.name = 'strides'
             # ints
@@ -188,6 +333,13 @@ class Converter:
             attrs_add.ints.append(l_params[types.fusion_conv_add_attrs_dict.get('strides')])
             attrs_add.ints.append(l_params[types.fusion_conv_add_attrs_dict.get('strides')])
 
+            # attrs_add = desc_ops_add.attrs.add()
+            # attrs_add.name = 'strides'
+            # # ints
+            # attrs_add.type = 3
+            # attrs_add.ints.append(6)
+            # attrs_add.ints.append(6)
+
             attrs_add = desc_ops_add.attrs.add()
             attrs_add.name = 'groups'
             # int
@@ -232,8 +384,8 @@ class Converter:
             # print o
             outputs_add = desc_ops_add.outputs.add()
             dict = types.op_io_dict.get(desc_ops_add.type)
-            print 'desc_ops_add.type:  ' + desc_ops_add.type
-            print dict
+            # print 'desc_ops_add.type:  ' + desc_ops_add.type
+            # print dict
             outputs_add.parameter = dict.get(types.mdl_outputs_key)
             outputs_add.arguments.append(o)
 
@@ -305,7 +457,7 @@ class Converter:
 
             # issues in mdl model filter swich n and c
             if j in self.deepwise_weight_list_ and len(dims_of_matrix) == 4:
-                print j
+                print "deep wise issue fit:  " + j
                 tensor.dims.append(dims_of_matrix[1])
                 tensor.dims.append(dims_of_matrix[0])
                 tensor.dims.append(dims_of_matrix[2])
@@ -320,6 +472,12 @@ class Converter:
                 vars_add.persistable = 1
                 dims_size = len(dims_of_matrix)
                 # print dims_size
+                # print 'weight name : ' + j
+                Swichter().copy_add_head(
+                    self.source_weights_dir + j + '.bin',
+                    self.target_weight_dir + j
+                )
+
                 # if dims_size == 4:
                 #     # convert weight from nhwc to nchw
                 #     Swichter().nhwc2nchw_one_slice_add_head(
@@ -341,7 +499,7 @@ class Converter:
                 vars_add.persistable = 0
 
 
-mdl_path = "datas/sourcemodels/cls231_0802/mobileNetModel.json"
+mdl_path = "datas/sourcemodels/source_profile/mobileNetModel.json"
 base_dir = "/Users/xiebaiyuan/PaddleProject/paddle-mobile/tools/python/modeltools/mobilenet/"
 converter = Converter(base_dir, mdl_path)
 converter.convert()

tools/python/modeltools/mobilenet/swicher.py

+import os
+import shutil
 from array import array
 
 
@@ -58,7 +60,7 @@ class Swichter:
         to_file = open(to_file_name, "wb")
 
         tmp = tmp_file.read()
-        head = self.read_head('yolo/datas/yolo/conv1_biases')
+        head = self.read_head('yolo/datas/yolo/head')
         to_file.write(head)
         to_file.write(tmp)
         tmp_file.close()
@@ -72,12 +74,14 @@ class Swichter:
         # print read
         return read
 
-    def copy_add_head(self, from_file_name, to_file_name, tmp_file_name):
+    def copy_add_head(self, from_file_name, to_file_name):
+
         from_file = open(from_file_name, "rb")
         to_file = open(to_file_name, "wb")
         # tmp_file = open(tmp_file_name, "wb")
 
-        head = self.read_head('yolo/datas/yolo/conv1_biases')
+        head = self.read_head(
+            '/Users/xiebaiyuan/PaddleProject/paddle-mobile/tools/python/modeltools/mobilenet/datas/sourcemodels/head/head')
         to_file.write(head)
         to_file.write(from_file.read())
         from_file.close()
@@ -96,7 +100,7 @@ class Swichter:
         to_file = open(to_file_name, "wb")
         # tmp_file = open(tmp_file_name, "wb")
 
-        head = self.read_head('yolo/datas/yolo/conv1_biases')
+        head = self.read_head('yolo/datas/yolo/head')
         to_file.write(head)
         to_file.write(read)
         from_file.close()
@@ -110,6 +114,6 @@ class Swichter:
 #     32,
 #     3, 3, 3)
 
-# Swichter().read_head('/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/modeltools/yolo/conv1_biases')
+# Swichter().read_head('/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/modeltools/yolo/head')
 
 # Swichter().copy_add_head('datas/model.0.0.weight', 'datas/conv1_0', '')

tools/python/modeltools/yolo/swicher.py

@@ -58,7 +58,7 @@ class Swichter:
         to_file = open(to_file_name, "wb")
 
         tmp = tmp_file.read()
-        head = self.read_head('yolo/datas/yolo/conv1_biases')
+        head = self.read_head('yolo/datas/yolo/head')
         to_file.write(head)
         to_file.write(tmp)
         tmp_file.close()
@@ -77,7 +77,7 @@ class Swichter:
         to_file = open(to_file_name, "wb")
         # tmp_file = open(tmp_file_name, "wb")
 
-        head = self.read_head('yolo/datas/yolo/conv1_biases')
+        head = self.read_head('yolo/datas/yolo/head')
         to_file.write(head)
         to_file.write(from_file.read())
         from_file.close()
@@ -96,7 +96,7 @@ class Swichter:
         to_file = open(to_file_name, "wb")
         # tmp_file = open(tmp_file_name, "wb")
 
-        head = self.read_head('yolo/datas/yolo/conv1_biases')
+        head = self.read_head('yolo/datas/yolo/head')
         to_file.write(head)
         to_file.write(read)
         from_file.close()
@@ -110,6 +110,6 @@ class Swichter:
 #     32,
 #     3, 3, 3)
 
-# Swichter().read_head('/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/modeltools/yolo/conv1_biases')
+# Swichter().read_head('/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/modeltools/yolo/head')
 
 # Swichter().copy_add_head('datas/model.0.0.weight', 'datas/conv1_0', '')