From e9820280b4194c395fa9b412ca50d5e77752b37e Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Thu, 10 Dec 2020 21:08:11 +0800 Subject: [PATCH] add custom op gelu for bert amp training (#5008) * add custom op gelu * refine run_pretrain --- PaddleNLP/benchmark/bert/run_pretrain.py | 2 +- PaddleNLP/benchmark/bert/run_pretrain_single.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/PaddleNLP/benchmark/bert/run_pretrain.py b/PaddleNLP/benchmark/bert/run_pretrain.py index ebbe4e2f..17e386d2 100644 --- a/PaddleNLP/benchmark/bert/run_pretrain.py +++ b/PaddleNLP/benchmark/bert/run_pretrain.py @@ -257,7 +257,7 @@ def do_train(args): ]) if args.use_amp: amp_list = paddle.fluid.contrib.mixed_precision.AutoMixedPrecisionLists( - custom_white_list=['softmax']) + custom_white_list=['softmax', 'layer_norm', 'gelu']) optimizer = paddle.fluid.contrib.mixed_precision.decorate( optimizer, amp_list, diff --git a/PaddleNLP/benchmark/bert/run_pretrain_single.py b/PaddleNLP/benchmark/bert/run_pretrain_single.py index 74a57a91..bc1e0ed9 100644 --- a/PaddleNLP/benchmark/bert/run_pretrain_single.py +++ b/PaddleNLP/benchmark/bert/run_pretrain_single.py @@ -228,7 +228,7 @@ def do_train(args): ]) if args.use_amp: amp_list = paddle.fluid.contrib.mixed_precision.AutoMixedPrecisionLists( - custom_white_list=['layer_norm', 'softmax']) + custom_white_list=['layer_norm', 'softmax', 'gelu']) optimizer = paddle.fluid.contrib.mixed_precision.decorate( optimizer, amp_list, -- GitLab