Created by: gongweibao

learning_rate=lr_decay * TrainTaskConfig.learning_rate,

           if args.local:
                lr_decay = fluid.layers.learning_rate_scheduler.noam_decay(
                    ModelHyperParams.d_model, TrainTaskConfig.warmup_steps)
                optimizer = fluid.optimizer.Adam(
                    learning_rate=lr_decay * TrainTaskConfig.learning_rate,
                    beta1=TrainTaskConfig.beta1,
                    beta2=TrainTaskConfig.beta2,
                    epsilon=TrainTaskConfig.eps)
            elif args.sync == False:
                optimizer = fluid.optimizer.SGD(0.003)
            optimizer.minimize(avg_cost)