diff --git a/python/paddle/fluid/device_worker.py b/python/paddle/fluid/device_worker.py
index 87f0a3d9ee6b9aaa7088f22019dd36c5770bfadc..9b8590523255f90e4fa4f92a33b9f4de84eaf3af 100644
--- a/python/paddle/fluid/device_worker.py
+++ b/python/paddle/fluid/device_worker.py
@@ -84,9 +84,9 @@ class DownpourSGD(DeviceWorker):
                     pc.push_sparse_table_id.extend([i])
                 for i in program_configs[program_id]["push_dense"]:
                     pc.push_dense_table_id.extend([i])
+                    dense_table_set.add(i)
                 for i in program_configs[program_id]["pull_sparse"]:
                     pc.pull_sparse_table_id.extend([i])
-                    dense_table_set.add(i)
                 for i in program_configs[program_id]["pull_dense"]:
                     pc.pull_dense_table_id.extend([i])
                     dense_table_set.add(i)