Created by: BigFishMaster

when running imagenet-1000 classification based on se_resnext50, if I use use_nccl = True, fluid.memory_optimize() must be used. Otherwise, the scripts DO NOT output any logs and the cuda use-rate are ZERO showed by nvidia-smi

    if parallel:
        places = fluid.layers.get_places()
        pd = fluid.layers.ParallelDo(places, use_nccl=use_nccl)
        with pd.do():
            img_ = pd.read_input(image)
            label_ = pd.read_input(label)
            prediction, avg_cost, accuracy, accuracy5 = net_conf(img_, label_, class_dim)

            for o in [avg_cost, accuracy, accuracy5]:
                pd.write_output(o)

        avg_cost, accuracy, accuracy5 = pd()
        # get mean loss and acc through every devices.
        avg_cost = fluid.layers.mean(x=avg_cost)
        accuracy = fluid.layers.mean(x=accuracy)
        accuracy5 = fluid.layers.mean(x=accuracy5)
    else:
        prediction, avg_cost, accuracy, accuracy5 = net_conf(image, label, class_dim)

    #print("network:", fluid.default_main_program())
    #print("network:", fluid.default_startup_program())

    inference_program = fluid.default_main_program().clone()
    epoch = [30, 60, 90]
    total_images = 1281167
    pass_each_epoch = int(total_images/batch_size + 1)
    bd = [e*pass_each_epoch for e in epoch]
    lr = [0.1,0.01,0.001,0.0001]

    print("Training with learning rates:", bd, lr)

    optimizer = fluid.optimizer.Momentum(
        learning_rate=fluid.layers.piecewise_decay(
            boundaries = bd,
            values = lr
        ),
        momentum=0.9,
        regularization=fluid.regularizer.L2Decay(1e-4))
    opts = optimizer.minimize(avg_cost)
    fluid.memory_optimize(fluid.default_main_program())