run.py 15.7 KB
Newer Older
Y
Yanzhan Yang 已提交
1
# -*- coding: utf-8 -*
Y
Yanzhan Yang 已提交
2 3 4 5 6 7 8 9 10 11 12
import os
import sys
import math
import subprocess
import numpy as np
import paddle.fluid as fluid

model_path = "model"
checked_model_path = "checked_model"
feed_path = "feeds"
output_path = "outputs"
Y
Yanzhan Yang 已提交
13
diff_threshold = 0.01
Y
Yanzhan Yang 已提交
14 15
is_lod = False
mobile_model_path = ""
Y
Yanzhan Yang 已提交
16
fast_check = False
17 18 19
is_sample_step = False
sample_step = 1
sample_num = 20
Y
Yanzhan Yang 已提交
20 21

np.set_printoptions(linewidth=150)
Y
Yanzhan Yang 已提交
22 23 24 25 26 27 28

mobile_exec_root = "/data/local/tmp/bin"
mobile_src_root = os.path.abspath("../../../")
if mobile_src_root.endswith("/"):
    mobile_src_root = mobile_src_root[:-1]

dot = "•"
Y
Yanzhan Yang 已提交
29 30 31 32
black = lambda x: "\033[30m" + str(x) + "\033[0m"
red = lambda x: "\033[31m" + str(x) + "\033[0m"
green = lambda x: "\033[32m" + str(x) + "\033[0m"
yellow = lambda x: "\033[33m" + str(x) + "\033[0m"
Y
Yanzhan Yang 已提交
33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67
reset = lambda x: "\033[0m" + str(x)

def pp_tab(x, level=0):
    header = ""
    for i in range(0, level):
        header += "\t"
    print(header + str(x))
def pp_black(x, level=0):
    pp_tab(black(x) + reset(""), level)
def pp_red(x, level=0):
    pp_tab(red(x) + reset(""), level)
def pp_green(x, level=0):
    pp_tab(green(x) + reset(""), level)
def pp_yellow(x, level=0):
    pp_tab(yellow(x) + reset(""), level)

def sh(command):
    pipe = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    return pipe.stdout.read().decode("utf-8")
def push(src, dest=""):
    sh("adb push {} {}".format(src, mobile_exec_root + "/" + dest))

pp_yellow(dot + " start inspecting fluid model")

exe = fluid.Executor(fluid.CPUPlace())
exe.run(fluid.default_startup_program())

# 加载模型
def load_model(model_path):
    prog, feeds, fetches = fluid.io.load_inference_model(dirname=model_path, executor=exe, model_filename="model", params_filename="params")
    return (prog, feeds, fetches)

prog, feeds, fetches = load_model(model_path)

# 强制要求所有张量的形状,在model和params中一致,并重新保存模型
68
def resave_model(feed_kv):
Y
Yanzhan Yang 已提交
69 70 71 72
    if len(mobile_model_path) > 0:
        pp_green("has set mobile_model_path, stop checking model & params", 1)
        sh("cp {}/* {}".format(mobile_model_path, checked_model_path))
        return
Y
Yanzhan Yang 已提交
73 74 75 76 77
    ops = prog.current_block().ops
    vars = prog.current_block().vars
    # 强制所有var为可持久化
    p_names = []
    for name in vars:
Y
Yanzhan Yang 已提交
78
        name = str(name)
Y
Yanzhan Yang 已提交
79 80 81 82
        v = fluid.framework._get_var(name, prog)
        if not v.persistable:
            v.persistable = True
            p_names.append(name)
83
    outputs = run_model(feed_kv=feed_kv)
Y
Yanzhan Yang 已提交
84 85 86
    has_found_wrong_shape = False
    # 修正每个var的形状
    for name in vars:
Y
Yanzhan Yang 已提交
87
        name = str(name)
Y
Yanzhan Yang 已提交
88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113
        v = vars[name]
        if v.persistable:
            v1 = fluid.global_scope().find_var(name)
            try:
                t1 = v1.get_tensor()
                shape = t1.shape()
            except:
                continue
            if v.desc.shape() != shape:
                has_found_wrong_shape = True
            v.desc.set_shape(shape)
    # 恢复var的可持久化属性
    for name in p_names:
        v = fluid.framework._get_var(name, prog)
        v.persistable = False
    fluid.io.save_inference_model(dirname=checked_model_path, feeded_var_names=feeds, target_vars=fetches, executor=exe, main_program=prog, model_filename="model", params_filename="params")
    if has_found_wrong_shape:
        pp_red("has found wrong shape", 1)
    else:
        pp_green("has not found wrong shape", 1)
    pp_green("new model is saved into directory 【{}】".format(checked_model_path), 1)

# 生成feed的key-value对
def gen_feed_kv():
    feed_kv = {}
    for feed_name in feeds:
114
        feed_shape = get_feed_var_shape(feed_name)
Y
Yanzhan Yang 已提交
115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133
        data = np.random.random(feed_shape).astype("float32")
        feed_kv[feed_name] = data
    return feed_kv

# 保存feed的key-value对
def save_feed_kv(feed_kv):
    for feed_name in feed_kv:
        feed_data = feed_kv[feed_name]
        feed_list = feed_data.flatten().tolist()
        if not os.path.exists(feed_path):
            os.mkdir(feed_path)
        file_name = feed_name.replace("/", "_")
        out_file = open(feed_path + "/" + file_name, "w")
        for feed_item in feed_list:
            out_file.write("{}\n".format(feed_item))
        out_file.close()

last_feed_var_name = None
last_feed_file_name = None
134
last_feed_var_lod = None
Y
Yanzhan Yang 已提交
135 136
# 加载feed的key-value对
def load_feed_kv():
Y
Yanzhan Yang 已提交
137 138
    if not os.path.exists(feed_path):
        return None
Y
Yanzhan Yang 已提交
139 140
    global last_feed_var_name
    global last_feed_file_name
141
    global last_feed_var_lod
Y
Yanzhan Yang 已提交
142 143 144 145
    feed_kv = {}
    pp_yellow(dot + dot + " checking feed info")
    pp_green("feed data is saved into directory 【{}】".format(feed_path), 1)
    for feed_name in feeds:
146
        feed_shape = get_feed_var_shape(feed_name)
Y
Yanzhan Yang 已提交
147 148 149 150
        pp_tab("feed var name : {}; feed var shape : {}".format(feed_name, feed_shape), 1)
        file_name = feed_name.replace("/", "_")
        last_feed_var_name = feed_name
        last_feed_file_name = file_name
Y
Yanzhan Yang 已提交
151 152 153 154 155 156 157 158 159 160
        feed_file_path = feed_path + "/" + file_name
        if not os.path.exists(feed_file_path):
            return None
        data = np.loadtxt(feed_file_path)
        expected_len = 1
        for dim in feed_shape:
            expected_len *= dim
        if len(data) != expected_len:
            return None
        data = data.reshape(feed_shape).astype("float32")
161 162 163 164 165 166 167 168
        
        if is_lod:
            data = data.reshape((1, *feed_shape)).astype("float32")
            tensor = fluid.LoDTensor()
            seq_lens = [len(seq) for seq in data]
            cur_len = 0
            lod = [cur_len]
            for l in seq_lens:
Y
Yanzhan Yang 已提交
169
                cur_len += l
170 171 172 173 174 175 176 177
                lod.append(cur_len)
            data = data.reshape(feed_shape)
            tensor.set(data, fluid.CPUPlace())
            tensor.set_lod([lod])
            last_feed_var_lod = lod
            feed_kv[feed_name] = tensor
        else:
            feed_kv[feed_name] = data
Y
Yanzhan Yang 已提交
178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199
    return feed_kv

# 运行模型
def run_model(feed_kv=None):
    if feed_kv is None:
        feed_kv = gen_feed_kv()
    outputs = exe.run(prog, feed=feed_kv, fetch_list=fetches, return_numpy=False)
    results = []
    for output in outputs:
        results.append(np.array(output))
    return results

# 获取变量形状
def get_var_shape(var_name):
    vars = prog.current_block().vars
    shape = vars[var_name].desc.shape()
    for i in range(len(shape)):
        dim = shape[i]
        if dim == -1:
            shape[i] = 1
    return shape

200 201 202 203 204 205
# 获取输入变量形状
def get_feed_var_shape(var_name):
    # 如果想写死输入形状,放开以下语句
    # return [1, 3, 224, 224]
    return get_var_shape(var_name)

Y
Yanzhan Yang 已提交
206 207 208 209 210 211 212
# 获取var的数据
def get_var_data(var_name, feed_kv=None):
    # 强制var为可持久化
    v = fluid.framework._get_var(var_name, prog)
    persistable = v.persistable
    if not persistable:
        v.persistable = True
213
    # outputs = run_model(feed_kv=feed_kv)
Y
Yanzhan Yang 已提交
214 215 216 217 218 219 220
    output = np.array(fluid.global_scope().find_var(var_name).get_tensor())
    # 恢复var的可持久化属性
    v.persistable = persistable
    return output

output_var_cache = {}
def tensor_sample(tensor):
221 222 223 224 225
    if is_sample_step:
        step = sample_step
    else:
        step = math.floor(len(tensor) / sample_num)
    step = max(step, 1)
Y
Yanzhan Yang 已提交
226
    sample = []
227
    for i in range(0, len(tensor), step):
Y
Yanzhan Yang 已提交
228 229 230
        sample.append(tensor[i])
    return sample

231
op_cache = {}
Y
Yanzhan Yang 已提交
232 233 234 235 236
# 获取每层输出的数据
def save_all_op_output(feed_kv=None):
    if not os.path.exists(output_path):
        os.mkdir(output_path)
    ops = prog.current_block().ops
Y
Yanzhan Yang 已提交
237 238 239
    fetch_names = []
    for fetch in fetches:
        fetch_names.append(fetch.name)
Y
Yanzhan Yang 已提交
240
    feed_names = feeds
Y
Yanzhan Yang 已提交
241 242 243
    for i in range(len(ops)):
        op = ops[i]
        var_name = None
244 245 246 247
        var_name_index = -1
        for index in range(len(op.output_names)):
            if op.output_names[index] in ["Y", "Out", "Output"]:
                var_name_index = index
Y
Yanzhan Yang 已提交
248
                break
249 250 251 252 253 254 255 256 257 258 259 260 261 262 263
        if var_name_index != -1:
            var_name = op.output_arg_names[var_name_index]
        else:
            for name in op.output_arg_names:
                var_name = name
                if "tmp" in name:
                    break
        # real_var_name = None
        # if op.type == "fetch":
        #     for name in op.input_arg_names:
        #         real_var_name = name
        #         if "tmp" in name:
        #             break
        # else:
        #     real_var_name = var_name
Y
Yanzhan Yang 已提交
264
        if fast_check:
Y
Yanzhan Yang 已提交
265
            if var_name not in fetch_names and var_name not in feed_names:
Y
Yanzhan Yang 已提交
266
                continue
Y
Yanzhan Yang 已提交
267 268 269 270 271 272 273
        try:
            data = get_var_data(var_name, feed_kv=feed_kv).flatten().tolist()
            sample = tensor_sample(data)
            output_var_cache[var_name] = (sample)
            op_cache[i] = (var_name, op)
            file_name = var_name.replace("/", "_")
            out_file = open(output_path + "/" + file_name, "w")
274 275 276 277 278 279
            if var_name in feed_names:
                for item in data:
                    out_file.write("{}\n".format(item))
            else:
                for item in sample:
                    out_file.write("{}\n".format(item))
Y
Yanzhan Yang 已提交
280 281 282 283 284 285 286 287 288 289 290 291 292 293
            out_file.close()
        except:
            pass
    pp_green("all the op outputs are saved into directory 【{}】".format(output_path), 1)

ops = prog.current_block().ops
vars = prog.current_block().vars

pp_yellow(dot + dot + " checking op list")
op_types = set()
for op in ops:
    op_types.add(op.type)
pp_tab("op types : {}".format(op_types), 1)

Y
Yanzhan Yang 已提交
294 295 296 297
def check_mobile_results(args, fuse, mem_opt):
    args = "{} {} {}".format("1" if fuse else "0", "1" if mem_opt else "0", args)
    res = sh("adb shell \"cd {} && export LD_LIBRARY_PATH=. && ./test-net {}\"".format(mobile_exec_root, args))
    lines = res.split("\n")
298
    # print(lines)
Y
Yanzhan Yang 已提交
299 300 301 302
    for line in lines:
        if line.startswith("auto-test-debug"):
            print(line)
    pp_yellow(dot + dot + " checking paddle mobile results for {} -- {} ".format(green("【fusion】" if fuse else "【non fusion】"), green("【memory-optimization】" if mem_opt else "【non-memory-optimization】")))
Y
Yanzhan Yang 已提交
303 304 305
    mobile_var_cache = {}
    for line in lines:
        parts = line.split(" ")
Y
Yanzhan Yang 已提交
306 307 308
        if len(parts) < 2:
            continue
        if "auto-test" != parts[0]:
Y
Yanzhan Yang 已提交
309 310 311 312 313
            continue
        if parts[1] == "load-time-cost":
            pp_green("load time cost : {}".format(parts[2]), 1) 
        elif parts[1] == "predict-time-cost":
            pp_green("predict time cost : {}".format(parts[2]), 1) 
314 315
        elif parts[1] == "preprocess-time-cost":
            pp_green("preprocess time cost : {}".format(parts[2]), 1)
Y
Yanzhan Yang 已提交
316 317 318 319 320 321 322
        elif parts[1] == "var":
            var_name = parts[2]
            values = list(map(lambda x: float(x), parts[3:]))
            mobile_var_cache[var_name] = values
    error_index = None
    error_values1 = None
    error_values2 = None
Y
Yanzhan Yang 已提交
323 324 325 326
    checked_names = []
    fetch_names = []
    for fetch in fetches:
        fetch_names.append(fetch.name)
Y
Yanzhan Yang 已提交
327 328
    for index in op_cache:
        op_output_var_name, op = op_cache[index]
Y
Yanzhan Yang 已提交
329 330 331 332 333 334 335 336
        if mem_opt:
            found_in_fetch = False
            for fetch in fetches:
                if op_output_var_name == fetch.name:
                    found_in_fetch = True
                    break
            if not found_in_fetch:
                continue
Y
Yanzhan Yang 已提交
337 338 339 340 341 342 343 344 345 346 347 348
        if not op_output_var_name in output_var_cache:
            continue
        if not op_output_var_name in mobile_var_cache:
            continue
        values1 = output_var_cache[op_output_var_name]
        values2 = mobile_var_cache[op_output_var_name]
        if len(values1) != len(values2):
            error_index = index
        if error_index == None:
            for i in range(len(values1)):
                v1 = values1[i]
                v2 = values2[i]
Y
Yanzhan Yang 已提交
349
                if abs(v1 - v2) > diff_threshold:
Y
Yanzhan Yang 已提交
350 351
                    error_index = index
                    break
Y
Yanzhan Yang 已提交
352
        checked_names.append(op_output_var_name)
Y
Yanzhan Yang 已提交
353 354 355 356
        if error_index != None:
            error_values1 = values1
            error_values2 = values2
            break
Y
Yanzhan Yang 已提交
357 358 359 360 361
    if error_index == None:
        for name in fetch_names:
            if name not in checked_names:
                error_index = -1
                break
Y
Yanzhan Yang 已提交
362 363
    if error_index == None:
        pp_green("outputs are all correct", 1)
Y
Yanzhan Yang 已提交
364 365
    elif error_index == -1:
        pp_red("outputs are missing")
Y
Yanzhan Yang 已提交
366
    else:
Y
Yanzhan Yang 已提交
367 368
        error_values1 = np.array(error_values1)
        error_values2 = np.array(error_values2)
Y
Yanzhan Yang 已提交
369 370
        # pp_red("mobile op is not correct, error occurs at {}th op, op's type is {}")
        pp_red("corresponding fluid op is {}th op, op's type is {}".format(error_index, op_cache[error_index][1].type), 1)
Y
Yanzhan Yang 已提交
371 372 373 374
        pp_red("fluid results are : ", 1)
        pp_red(str(error_values1).replace("\n", "\n" + "\t" * 1), 1)
        pp_red("paddle mobile results are : ", 1)
        pp_red(str(error_values2).replace("\n", "\n" + "\t" * 1), 1)
Y
Yanzhan Yang 已提交
375 376 377 378 379 380
    # print(output_var_cache)
    # print(mobile_var_cache)

def main():
    # 加载kv
    feed_kv = load_feed_kv()
Y
Yanzhan Yang 已提交
381 382 383 384
    if feed_kv == None:
        feed_kv = gen_feed_kv()
        save_feed_kv(feed_kv)
        feed_kv = load_feed_kv()
Y
Yanzhan Yang 已提交
385 386 387 388 389 390
    # 预测
    pp_yellow(dot + dot + " checking inference")
    outputs = run_model(feed_kv=feed_kv)
    pp_tab("fluid output : {}".format(outputs), 1)
    # 重新保存模型
    pp_yellow(dot + dot + " checking model correctness")
391
    resave_model(feed_kv=feed_kv)
Y
Yanzhan Yang 已提交
392 393 394
    # 输出所有中间结果
    pp_yellow(dot + dot + " checking output result of every op")
    save_all_op_output(feed_kv=feed_kv)
395 396 397 398 399
    pp_yellow(dot + dot + " checking fetch info")
    for fetch in fetches:
        fetch_name = fetch.name
        fetch_shape = get_var_shape(fetch_name)
        pp_tab("fetch var name : {}; fetch var shape : {}".format(fetch_name, fetch_shape), 1)
400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421
    # 输出所有op、var信息
    info_file = open("info.txt", "w")
    for i in range(len(ops)):
        op = ops[i]
        info_file.write("{}th op: type - {}\n".format(i, op.type))
        info_file.write("inputs:\n")
        for var_name in op.input_arg_names:
            try:
                shape = get_var_shape(var_name)
                shape_str = ", ".join(list(map(lambda x: str(x), shape)))
                info_file.write("var {} : {}\n".format(var_name, shape_str))
            except:
                pass
        info_file.write("outputs:\n")
        for var_name in op.output_arg_names:
            try:
                shape = get_var_shape(var_name)
                shape_str = ", ".join(list(map(lambda x: str(x), shape)))
                info_file.write("var {} : {}\n".format(var_name, shape_str))
            except:
                pass
    info_file.close()
Y
Yanzhan Yang 已提交
422 423 424 425 426 427 428 429 430
    # 开始检查mobile的正确性
    print("")
    print("==================================================")
    print("")
    pp_yellow(dot + " start inspecting paddle mobile correctness & performance")
    push(checked_model_path)
    push(feed_path + "/" + last_feed_file_name, "input.txt")
    push(mobile_src_root + "/build/release/arm-v7a/build/libpaddle-mobile.so")
    push(mobile_src_root + "/test/build/test-net")
431
    last_feed_var_shape = get_feed_var_shape(last_feed_var_name)
Y
Yanzhan Yang 已提交
432 433 434
    args = str(len(last_feed_var_shape))
    for dim in last_feed_var_shape:
        args += " " + str(dim)
435 436 437 438 439 440 441
    if is_lod:
        args += " 1"
        args += " " + str(len(last_feed_var_lod))
        for dim in last_feed_var_lod:
            args += " " + str(dim)
    else:
        args += " 0"
Y
Yanzhan Yang 已提交
442
    args += " " + str(len(output_var_cache))
443 444 445 446 447
    args += " " + str(1 if is_sample_step else 0)
    if is_sample_step:
        args += " " + str(sample_step)
    else:
        args += " " + str(sample_num)
Y
Yanzhan Yang 已提交
448 449
    for var_name in output_var_cache.keys():
        args += " " + var_name
Y
Yanzhan Yang 已提交
450 451 452
    if not fast_check:
        check_mobile_results(args, False, False)
        check_mobile_results(args, False, True)
Y
Yanzhan Yang 已提交
453 454
    check_mobile_results(args, True, False)
    check_mobile_results(args, True, True)
Y
Yanzhan Yang 已提交
455 456 457

if __name__ == "__main__":
    main()