fix nccl dist train bug

d76ebd78 · yi.wu · 88fa9c2e · d76ebd78
显示空白变更内容
内联并排

Showing with 4 addition and 0 deletion

paddle/fluid/operators/gen_nccl_id_op.cc paddle/fluid/operators/gen_nccl_id_op.cc +4 -0

未找到文件。
--- a/paddle/fluid/operators/gen_nccl_id_op.cc
+++ b/paddle/fluid/operators/gen_nccl_id_op.cc
@@ -67,6 +67,10 @@ class GenNCCLIdOp : public framework::OperatorBase {
      client->AsyncSendVar(ep, dev_ctx, *scope, NCCL_ID_VARNAME);
    }
    client->Wait();
+    for (auto& ep : endpoint_list) {
+      client->AsyncSendBatchBarrier(ep);
+    }
+    client->Wait();
    VLOG(3) << "sending completed...";
  }