diff --git a/paddle/fluid/framework/fleet/fleet_wrapper.cc b/paddle/fluid/framework/fleet/fleet_wrapper.cc
index 1497628e64cf9ca42cdad8dbba4079ddfde95be3..be359ad332c52840c8b30402a445b7a3b6dc28cf 100644
--- a/paddle/fluid/framework/fleet/fleet_wrapper.cc
+++ b/paddle/fluid/framework/fleet/fleet_wrapper.cc
@@ -170,7 +170,8 @@ void FleetWrapper::PullDenseVarsAsync(
     const std::vector<std::string>& var_names,
     std::vector<::std::future<int32_t>>* pull_dense_status) {
 #ifdef PADDLE_WITH_PSLIB
-  std::vector<paddle::ps::Region> regions;
+  auto& regions = _regions[tid];
+  regions.clear();
   regions.resize(var_names.size());
   for (auto i = 0u; i < var_names.size(); ++i) {
     Variable* var = scope.FindVar(var_names[i]);
@@ -189,7 +190,8 @@ void FleetWrapper::PullDenseVarsSync(
     const Scope& scope, const uint64_t tid,
     const std::vector<std::string>& var_names) {
 #ifdef PADDLE_WITH_PSLIB
-  std::vector<paddle::ps::Region> regions;
+  auto& regions = _regions[tid];
+  regions.clear();
   regions.reserve(var_names.size());
   for (auto& t : var_names) {
     Variable* var = scope.FindVar(t);
diff --git a/paddle/fluid/framework/fleet/fleet_wrapper.h b/paddle/fluid/framework/fleet/fleet_wrapper.h
index ed3217b376b01d52fad28d147fc5f511ffd7f2b4..9e08ef64741fa11a8f0cf2aa7d0b40dcd4facdb8 100644
--- a/paddle/fluid/framework/fleet/fleet_wrapper.h
+++ b/paddle/fluid/framework/fleet/fleet_wrapper.h
@@ -146,6 +146,7 @@ class FleetWrapper {
 
  private:
   static std::shared_ptr<FleetWrapper> s_instance_;
+  std::map<uint64_t, std::vector<paddle::ps::Region>> _regions;
 
  protected:
   static bool is_initialized_;
diff --git a/python/paddle/fluid/incubate/fleet/base/role_maker.py b/python/paddle/fluid/incubate/fleet/base/role_maker.py
index baaeb1abef7c71bde50900dc371b3a23bb3fff12..d7088d2b0115fda0eb973229f144394d7e7dcc50 100644
--- a/python/paddle/fluid/incubate/fleet/base/role_maker.py
+++ b/python/paddle/fluid/incubate/fleet/base/role_maker.py
@@ -74,6 +74,7 @@ class MPIRoleMaker(RoleMakerBase):
     """
 
     def __init__(self):
+        super(MPIRoleMaker, self).__init__()
         from mpi4py import MPI
         self.comm_ = MPI.COMM_WORLD
         self.MPI = MPI
diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/optimizer_factory.py b/python/paddle/fluid/incubate/fleet/parameter_server/optimizer_factory.py
index c292881140464d38ab83a7ad6f3523f649b1e096..461aac8e1e59d19a620f80a3d2c8a1c02110456d 100644
--- a/python/paddle/fluid/incubate/fleet/parameter_server/optimizer_factory.py
+++ b/python/paddle/fluid/incubate/fleet/parameter_server/optimizer_factory.py
@@ -141,9 +141,9 @@ class DistributedAdam(DistributedOptimizerImplBase):
                                        data_norm_params, data_norm_grads)
                 #program_config.pull_dense_table_id.extend([dense_table_index])
                 #program_config.push_dense_table_id.extend([dense_table_index])
-                program_config[program_id]["pull_dense"].extend(
+                program_configs[program_id]["pull_dense"].extend(
                     [dense_table_index])
-                program_config[program_id]["push_dense"].extend(
+                program_configs[program_id]["push_dense"].extend(
                     [dense_table_index])
             dense_table_index += 1
             #program_configs.append(program_config)