Created by: wzgwzg

paddle 1.5.0, cuda 8.0, cudnn v7, k40 单卡训练

报错：Cannot find fetch variable in scope, fetch_var_name is tmp_12 at xxx fetch_op.cc:37

部分代码： def net_config(image, label, model, args): model_list = [m for m in dir(models) if "__" not in m] assert args.model in model_list, "{} is not lists: {}".format(args.model, model_list) model_name = args.model if "Ft_Net" in model_name: x3_g_pool_fc, x4_g_pool_fc, x4_p_pool_fc, x3_g_avg_fc, x4_g_avg_fc, x4_p_avg_fc, x3_g_max_fc, x4_g_max_fc, x4_p_max_fc = model.net(input=image) cost_1, pred_1 = calc_loss(x3_g_pool_fc, label) avg_cost_1 = fluid.layers.mean(x=cost_1) 。。。。。。 total_cost = (cost_1 + cost_2 + cost_3 + cost_4 + cost_5 + cost_6 + cost_7 + cost_8 + cost_9) / 9.0 acc_1 = fluid.layers.accuracy(input=pred_1, label=label, k=1) acc_2 = fluid.layers.accuracy(input=pred_2, label=label, k=1) acc_3 = fluid.layers.accuracy(input=pred_3, label=label, k=1) acc_4 = fluid.layers.accuracy(input=pred_4, label=label, k=1) acc_5 = fluid.layers.accuracy(input=pred_5, label=label, k=1) acc_6 = fluid.layers.accuracy(input=pred_6, label=label, k=1) acc_7 = fluid.layers.accuracy(input=pred_7, label=label, k=1) acc_8 = fluid.layers.accuracy(input=pred_8, label=label, k=1) acc_9 = fluid.layers.accuracy(input=pred_9, label=label, k=1)

    return total_cost, acc_1, acc_2, acc_3, acc_4, acc_5, acc_6, acc_7, acc_8, acc_9

def train(args): model_name = args.model checkpoint = args.checkpoint pretrained_model = args.pretrained_model with_memory_optimization = args.with_mem_opt model_save_dir = args.model_save_dir num_instances = args.num_instances

startup_prog = fluid.Program()
train_prog = fluid.Program()

train_py_reader, total_cost, acc_1, acc_2, acc_3, acc_4, acc_5, acc_6, acc_7, acc_8, acc_9, global_lr = \
    build_program(is_train=True, main_prog=train_prog, startup_prog=startup_prog, args=args)

train_fetch_vars = [total_cost, acc_1, acc_2, acc_3, acc_4, acc_5, acc_6, acc_7, acc_8, acc_9, global_lr]
train_fetch_list = []
for var in train_fetch_vars:
    var.persistable=True
    train_fetch_list.append(var.name)     

if with_memory_optimization:
    fluid.memory_optimize(train_prog)

place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(startup_prog)
。。。。。。
build_strategy = fluid.BuildStrategy()
build_strategy.enable_inplace = args.with_inplace
exec_strategy = fluid.ExecutionStrategy()
exec_strategy.num_threads = device_num
if num_trainers > 1 and args.use_gpu:
    dist_utils.prepare_for_multi_process(exe, build_strategy, train_prog)
    exec_strategy.num_threads = 1

train_exe = fluid.ParallelExecutor(
    main_program=train_prog,
    use_cuda=bool(args.use_gpu),
    loss_name=total_cost.name,
    build_strategy=build_strategy,
    exec_strategy=exec_strategy)

for pass_id in range(args.num_epochs):
    train_py_reader.start()
    train_info = [[], [], [], [], [], [], [], [], [], []]
    train_time = []
    batch_id = 0
    try:
        while True:
            t1 = time.time()
            total_loss, tmp_acc_1, tmp_acc_2, tmp_acc_3, tmp_acc_4, tmp_acc_5, tmp_acc_6, tmp_acc_7, tmp_acc_8, tmp_acc_9, lr = exe.run(
                fetch_list=train_fetch_list)