From d62369e905024808d67292c841750f5f7ec834d5 Mon Sep 17 00:00:00 2001
From: xiebaiyuan <xiebaiyuan@139.com>
Date: Wed, 26 Sep 2018 16:43:28 +0800
Subject: [PATCH] update python tools for models update yolo mdl2fluid tools

---
 python/tools/imagetools/imagetools.py     | 61 ++++++++++++++++++++
 python/tools/imagetools/img2nchw.py       | 69 +++++++++++++++++++++++
 python/tools/imagetools/img2nhwc.py       | 34 +++++++++++
 python/tools/imagetools/numpy2binary.py   | 47 +++++++++++++++
 python/tools/mdl2fluid/float16_float32.py | 34 -----------
 python/tools/mdl2fluid/model_combine.py   | 19 +++++++
 python/tools/mdl2fluid/swicher.py         | 35 ++++++++++--
 7 files changed, 259 insertions(+), 40 deletions(-)
 create mode 100644 python/tools/imagetools/imagetools.py
 create mode 100644 python/tools/imagetools/img2nchw.py
 create mode 100644 python/tools/imagetools/img2nhwc.py
 create mode 100644 python/tools/imagetools/numpy2binary.py
 delete mode 100644 python/tools/mdl2fluid/float16_float32.py
 create mode 100644 python/tools/mdl2fluid/model_combine.py

diff --git a/python/tools/imagetools/imagetools.py b/python/tools/imagetools/imagetools.py
new file mode 100644
index 0000000000..2a44328580
--- /dev/null
+++ b/python/tools/imagetools/imagetools.py
@@ -0,0 +1,61 @@
+# coding=utf-8
+import cv2
+from array import array
+
+
+def resize_take_rgbs(path, shape_h_w):
+    print '--------------resize_take_rgbs-----------------begin'
+    image = cv2.imread(path)
+    # print image.shape
+    cv2.imshow("before", image)
+
+    print_rgb(image[0, 0])
+    # image len may be for .just check it
+    # image.resize(shape_h_w)
+
+    image = cv2.resize(image, (shape_h_w[0], shape_h_w[1]))
+
+    cv2.imshow("after", image)
+    print image.shape
+    height = shape_h_w[0]
+    width = shape_h_w[1]
+
+    rs_ = []
+    gs_ = []
+    bs_ = []
+    for h in range(0, height):
+        for w in range(0, width):
+            bs_.append(image[h, w, 0])
+            gs_.append(image[h, w, 1])
+            rs_.append(image[h, w, 2])
+
+    # print image[2, 2, 0]/255.
+    print len(bs_)
+    print len(gs_)
+    print len(rs_)
+    print '--------------resize_take_rgbs-----------------end'
+    return bs_, gs_, rs_
+
+
+def print_rgb((b, g, r)):
+    print "像素 - R:%d,G:%d,B:%d" % (r, g, b)  # 显示像素值
+    #
+    # image[0, 0] = (100, 150, 200)  # 更改位置(0,0)处的像素
+    #
+    # (b, g, r) = image[0, 0]  # 再次读取(0,0)像素
+    # print "位置(0,0)处的像素 - 红:%d,绿:%d,蓝:%d" % (r, g, b)  # 显示更改后的像素值
+    #
+    # corner = image[0:100, 0:100]  # 读取像素块
+    # cv2.imshow("Corner", corner)  # 显示读取的像素块
+    #
+    # image[0:100, 0:100] = (0, 255, 0);  # 更改读取的像素块
+    #
+    # cv2.imshow("Updated", image)  # 显示图像
+    #
+    # cv2.waitKey(0)  # 程序暂停
+
+
+def save_to_file(to_file_name, array):
+    to_file = open(to_file_name, "wb")
+    array.tofile(to_file)
+    to_file.close()
diff --git a/python/tools/imagetools/img2nchw.py b/python/tools/imagetools/img2nchw.py
new file mode 100644
index 0000000000..70ca456a1b
--- /dev/null
+++ b/python/tools/imagetools/img2nchw.py
@@ -0,0 +1,69 @@
+# coding=utf-8
+import cv2
+from array import array
+import imagetools as tools
+from enum import Enum
+
+
+class ChannelType(Enum):
+    RGB = 0,
+    BGR = 1
+
+
+def combine_bgrs_nchw(bgrs, means_b_g_r, scale, channel_type=ChannelType.BGR):
+    print '--------------combine_bgrs_nchw-----------------begin'
+    print "scale: %f" % scale
+    print means_b_g_r
+    # print len(bgrs)
+    bs = bgrs[0]
+    gs = bgrs[1]
+    rs = bgrs[2]
+
+    assert len(bs) == len(gs) == len(rs)
+    print len(bs)
+    bgrs_float_array = array('f')
+
+    if channel_type == ChannelType.BGR:
+        print 'bgr'
+        for i in range(0, len(bs)):
+            bgrs_float_array.append((bs[i] - means_b_g_r[0]) * scale)  # b
+        for i in range(0, len(gs)):
+            bgrs_float_array.append((gs[i] - means_b_g_r[1]) * scale)  # g
+        for i in range(0, len(rs)):
+            bgrs_float_array.append((rs[i] - means_b_g_r[2]) * scale)  # r
+    elif channel_type == ChannelType.RGB:
+        print 'rgb'
+
+        for i in range(0, len(rs)):
+            bgrs_float_array.append((rs[i] - means_b_g_r[2]) * scale)  # r
+        for i in range(0, len(gs)):
+            bgrs_float_array.append((gs[i] - means_b_g_r[1]) * scale)  # g
+        for i in range(0, len(bs)):
+            bgrs_float_array.append((bs[i] - means_b_g_r[0]) * scale)  # b
+
+    print len(bgrs_float_array)
+
+    print '------------------'
+    print bgrs_float_array[0]
+    print bgrs_float_array[416 * 416 * 2 + 416 * 2 + 2]
+
+    # for i in range(0, 9):
+    #     print'bs %d' % i
+    #     print bs[i] / 255.
+
+    print bs[416 * 2 + 2] / 255.
+    print '--------------combine_bgrs_nchw-----------------end'
+
+    return bgrs_float_array
+
+
+# bgrs = tools.resize_take_rgbs('banana.jpeg', (224, 224, 3))
+# array = combine_bgrs_nchw(bgrs, (103.94, 116.78, 123.68), 0.017, array,ChannelType.BGR)
+# tools.save_to_file('banana_1_3_224_224_nchw_float')
+
+# cv2.waitKey(0)
+
+
+bgrs = tools.resize_take_rgbs('datas/newyolo.jpg', (416, 416, 3))
+array = combine_bgrs_nchw(bgrs, (0, 0, 0), 1. / 255, ChannelType.RGB)
+tools.save_to_file('datas/desktop_1_3_416_416_nchw_float', array)
diff --git a/python/tools/imagetools/img2nhwc.py b/python/tools/imagetools/img2nhwc.py
new file mode 100644
index 0000000000..c982fe303e
--- /dev/null
+++ b/python/tools/imagetools/img2nhwc.py
@@ -0,0 +1,34 @@
+# coding=utf-8
+import cv2
+from array import array
+import imagetools as tools
+
+
+def combine_bgrs_nhwc(bgrs, means_b_g_r, scale):
+    print "scale: %f" % scale
+    print means_b_g_r
+    # print len(bgrs)
+    bs = bgrs[0]
+    gs = bgrs[1]
+    rs = bgrs[2]
+    assert len(bs) == len(gs) == len(rs)
+    # print len(bs)
+    bgrs_float_array = array('f')
+    for i in range(0, len(bs)):
+        bgrs_float_array.append((rs[i] - means_b_g_r[2]) * scale)  # r
+        bgrs_float_array.append((gs[i] - means_b_g_r[1]) * scale)  # g
+        bgrs_float_array.append((bs[i] - means_b_g_r[0]) * scale)  # b
+
+    print len(bgrs_float_array)
+
+    print '------------------'
+    print bgrs_float_array[0]
+    print bgrs_float_array[999]
+    return bgrs_float_array
+
+
+bgrs = tools.resize_take_rgbs('newyolo_1.jpg', (416, 416, 3))
+array = combine_bgrs_nhwc(bgrs, (0, 0, 0), 1.0 / 255)
+tools.save_to_file('desktop_1_3_416_416_nhwc_float', array)
+
+cv2.waitKey(0)
diff --git a/python/tools/imagetools/numpy2binary.py b/python/tools/imagetools/numpy2binary.py
new file mode 100644
index 0000000000..dd4bc6e100
--- /dev/null
+++ b/python/tools/imagetools/numpy2binary.py
@@ -0,0 +1,47 @@
+# coding=utf-8
+
+# 这个脚本是可以将numpy合并到二进制
+import cv2
+import numpy as np
+import imagetools as tools
+from array import array
+
+#
+# image = cv2.imread(path)
+# print image.shape
+#
+# print_rgb(image[0, 0])
+# # image len may be for .just check it
+# image.resize(shape_h_w)
+
+
+data = np.fromfile('datas/img.res')
+print data.size
+print data[0]
+
+data.reshape(1, 3, 416, 416)
+out_array = array('f')
+print'--------------------'
+print data.size
+print data[0]
+
+print '如果是nhwc --------'
+# rgb rgb rgb rgb rgb
+print data[416 * 3 * 2 + 3 * 2 + 2]
+# print data[2]
+
+print '如果是nchw --------'
+# rgb rgb rgb rgb rgb
+print data[416 * 416 * 2 + 416 * 2 + 2]
+# print data[2]
+
+# 明明是nchw
+
+for i in range(0, data.size):
+    out_array.append(data[i])
+
+print len(out_array)
+
+print out_array[416 * 416 * 2 + 416 * 2 + 2]
+
+tools.save_to_file('datas/in_put_1_3_416_416_2', out_array)
diff --git a/python/tools/mdl2fluid/float16_float32.py b/python/tools/mdl2fluid/float16_float32.py
deleted file mode 100644
index e536238955..0000000000
--- a/python/tools/mdl2fluid/float16_float32.py
+++ /dev/null
@@ -1,34 +0,0 @@
-import binascii
-import os
-import numpy as np
-
-
-def read_param(path):
-    try:
-        with open(path, "r") as f:
-            value = f.read(2)
-            a_hex = binascii.b2a_hex(value)
-            print a_hex
-
-
-            # value = f.read(2)
-            # a_hex = binascii.b2a_hex(value)
-            # print a_hex
-            # value = f.read(2)
-            # a_hex = binascii.b2a_hex(value)
-            # print a_hex
-
-    except IOError:
-        print ": File not found."
-
-
-def get_file_size(file_path):
-    file_path = unicode(file_path, 'utf8')
-    f_size = os.path.getsize(file_path)
-    f_size = f_size / float(1024 * 1024)
-    return round(f_size, 2)
-
-
-read_param(
-    "/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/YOLOParameters_Universal"
-    ".bundle/conv1_0.bin")
diff --git a/python/tools/mdl2fluid/model_combine.py b/python/tools/mdl2fluid/model_combine.py
new file mode 100644
index 0000000000..ae3ca8a786
--- /dev/null
+++ b/python/tools/mdl2fluid/model_combine.py
@@ -0,0 +1,19 @@
+# coding=utf-8
+import os
+
+path = "yolo_v2_tofile_source/"  # 文件夹目录
+to_file_path = "yolo_v2_tofile_combined/params"
+files = os.listdir(path)  # 得到文件夹下的所有文件名称
+files.sort(cmp=None, key=str.lower)
+to_file = open(to_file_path, "wb")
+
+for file in files:  # 遍历文件夹
+    if not os.path.isdir(file):  # 判断是否是文件夹，不是文件夹才打开
+        f = open(path + "/" + file)  # 打开文件
+        name = f.name
+        print 'name:  ' + name
+        from_file = open(name, "rb")
+        to_file.write(from_file.read())
+        from_file.close()
+
+to_file.close()
diff --git a/python/tools/mdl2fluid/swicher.py b/python/tools/mdl2fluid/swicher.py
index 0cf39959ff..bfe0360fd5 100644
--- a/python/tools/mdl2fluid/swicher.py
+++ b/python/tools/mdl2fluid/swicher.py
@@ -66,7 +66,7 @@ class Swichter:
 
     def read_head(self, head_file):
         from_file = open(head_file, "rb")
-        read = from_file.read(20)
+        read = from_file.read(24)
         # print read
         from_file.close()
         # print read
@@ -84,9 +84,32 @@ class Swichter:
         to_file.close()
         pass
 
+    def copy_padding_add_head(self, from_file_name, to_file_name, tmp_file_name, padding):
+        print'padding  = %d' % padding
+        from_file = open(from_file_name, "rb")
+        # print len(from_file.read())
+        from_file.seek(padding, 0)
+
+        read = from_file.read()
+        print len(read)
+
+        to_file = open(to_file_name, "wb")
+        # tmp_file = open(tmp_file_name, "wb")
+
+        head = self.read_head('/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/yolo/conv1_biases')
+        to_file.write(head)
+        to_file.write(read)
+        from_file.close()
+        to_file.close()
+        pass
+
+# Swichter().nhwc2nchw_one_slice_add_head(
+#     '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nhwc/conv1_0.bin',
+#     '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nchw_with_head/conv1_0',
+#     '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nchw/.tmp',
+#     32,
+#     3, 3, 3)
+
+# Swichter().read_head('/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/yolo/conv1_biases')
 
-# Swichter().nhwc2nchw_one_slice(
-#     '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nhwc/conv5_6_dw_0.bin',
-#     '/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/multiobjects/float32s_nchw/conv5_6_dw_0', 1,
-#     512, 3, 3)
-Swichter().read_head('/Users/xiebaiyuan/PaddleProject/paddle-mobile/python/tools/mdl2fluid/yolo/conv1_biases')
+# Swichter().copy_add_head('datas/model.0.0.weight', 'datas/conv1_0', '')
-- 
GitLab