From 90017d3a31beee0ef5421ac08edcd0fa441eea11 Mon Sep 17 00:00:00 2001 From: Samyam Rajbhandari Date: Sat, 11 Apr 2020 23:51:15 -0700 Subject: [PATCH] AllReduce bucket fix. (#186) --- deepspeed/pt/deepspeed_light.py | 1 + 1 file changed, 1 insertion(+) diff --git a/deepspeed/pt/deepspeed_light.py b/deepspeed/pt/deepspeed_light.py index 850ea9d1..765e9023 100755 --- a/deepspeed/pt/deepspeed_light.py +++ b/deepspeed/pt/deepspeed_light.py @@ -840,6 +840,7 @@ class DeepSpeedLight(Module): if numel > numel_per_bucket: self.allreduce_and_copy(small_bucket) small_bucket = [] + numel = 0 if len(small_bucket) > 0: self.allreduce_and_copy(small_bucket) -- GitLab