add device synchronize for p2p (#55461)

python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import distutils.util
+import os
 
 import numpy as np
 
@@ -408,6 +410,11 @@ def _p2p_helper(
     if len(ops) > 0:
         batch_send_recv_on_calc_stream(ops)
 
+        if distutils.util.strtobool(
+            os.getenv('FLAGS_p2p_device_synchronize', '0')
+        ):
+            paddle.device.cuda.synchronize()
+
     tensors_for_all_gather = []
     if tensor_recv_prev is not None:
         if isinstance(tensor_recv_prev, tuple):