Created by: wangxicoding
PR types
New features
PR changes
APIs
Describe
Old PR https://github.com/PaddlePaddle/Paddle/pull/22434 , rewrite in this PR.
Add fp16 grad compress to fleet meta optimizer. The strategy name is fp16_allreduce
Test on ResNet50 model,run with fleetrun --gpus 0,1,2,3 fp16_compress_demo.py
import paddle
paddle.enable_static()
import fleetx as X
import paddle.fluid as fluid
import paddle.distributed.fleet as fleet
fleet.init(is_collective=True)
configs = X.parse_train_configs()
model = X.applications.Resnet50("NCHW")
imagenet_downloader = X.utils.ImageNetDownloader()
local_path = imagenet_downloader.download_from_hdfs('imagenet.yaml',local_path='./ImageNet')
loader = model.load_imagenet_from_file("{}/train.txt".format(local_path), batch_size=32)
dist_strategy = fleet.DistributedStrategy()
dist_strategy.fp16_allreduce = True
optimizer = fluid.optimizer.Momentum(learning_rate=configs.lr, momentum=configs.momentum)
optimizer = fleet.distributed_optimizer(optimizer, strategy=dist_strategy)
optimizer.minimize(model.loss)
trainer = X.MultiGPUTrainer()
trainer.fit(model, loader, epoch=10, use_dali=True)