[FIX] pg partition map handle pg remove corrupted after globalize

src/storage/ob_partition_service.cpp

@@ -4249,6 +4249,8 @@ int ObPartitionService::inner_del_partition_impl(const ObPartitionKey& pkey, con
       STORAGE_LOG(WARN, "remove election from election mgr error", K(ret), K(pkey));
     } else if (OB_FAIL(pg_mgr_.del_pg(pkey, file_id))) {
       STORAGE_LOG(WARN, "pg mgr remove partition group error", K(ret), K(pkey));
+    } else if (OB_FAIL(pg->get_pg_storage().post_del_pg())) {
+      STORAGE_LOG(WARN, "failed to call post del_pg", K(ret), K(pkey));
     } else if (OB_FAIL(pg->get_pg_storage().remove_all_pg_index())) {
       STORAGE_LOG(WARN, "failed to remove all pg index", K(ret), K(pkey));
     } else {

src/storage/ob_pg_partition.h

@@ -181,21 +181,31 @@ static const int64_t SHRINK_THRESHOLD = 128;
 typedef common::ObLinkHashMap<common::ObPartitionKey, ObPGPartition, PGPartitionInfoAlloc> ObPGPartitionMap;
 
 // thread safe
-class ObPartitionKeyList {
+class ObPGPartitionList {
 private:
   class Node : public ObDLinkBase<Node> {
   public:
-    Node(const common::ObPartitionKey& key) : key_(key)
+    Node(const common::ObPartitionKey &key) : key_(key), part_(NULL)
     {}
     const common::ObPartitionKey key_;
+    // no-NULL when PartitionGroup removed
+    ObPGPartition *part_;
+    TO_STRING_KV(K(key_), KP(part_));
   };
   typedef common::ObDList<Node> List;
 
 public:
-  ObPartitionKeyList() : lock_(), list_()
+  ObPGPartitionList() : lock_(), list_(), contains_part_info_(false)
   {}
+  void reset()
+  {
+    TCWLockGuard guard(lock_);
+    STORAGE_LOG(INFO, "reset partition list", K(this), K_(list), K_(contains_part_info), K(lbt()));
+    list_.reset();
+    contains_part_info_ = false;
+  }
 
-  inline int size()
+  inline int size() const
   {
     return list_.get_size();
   }
@@ -224,8 +234,7 @@ public:
   // NOTICE : _assumption_ : no concurrent remove of same `pkey`
   bool remove_latest(const common::ObPartitionKey& pkey)
   {
-    Node* n = NULL;
-
+    Node *n = NULL;
     {
       TCRLockGuard guard(lock_);
       DLIST_FOREACH_BACKWARD_X(curr, list_, NULL == n)
@@ -235,7 +244,6 @@ public:
         }
       }
     }
-
     if (NULL != n) {
       {
         TCWLockGuard guard(lock_);
@@ -243,7 +251,6 @@ public:
       }
       ob_free(n);
     }
-
     return NULL != n;
   }
 
@@ -266,14 +273,54 @@ public:
     DLIST_REMOVE_ALL_NORET(cur, list_)
     {
       cur->unlink();
-      fn(cur->key_);
+      fn(cur->key_, cur->part_);
       ob_free(cur);
     }
   }
+  template <typename Fn>
+  int set_partition_info(Fn &fn)
+  {
+    int ret = OB_SUCCESS;
+    TCWLockGuard guard(lock_);
+    DLIST_FOREACH(cur, list_)
+    {
+      ret = fn(cur->key_, cur->part_);
+    }
+    if (OB_SUCC(ret)) {
+      contains_part_info_ = true;
+    }
+    STORAGE_LOG(INFO, "set_partition_info OK", K(this), K_(list), K_(contains_part_info));
+    return ret;
+  }
+  int get_part_info(const ObPartitionKey &pkey, ObPGPartition *&part) const
+  {
+    int ret = OB_ENTRY_NOT_EXIST;
+    bool found_pkey = false;
+    TCRLockGuard guard(lock_);
+    if (contains_part_info_) {
+      DLIST_FOREACH_NORET(cur, list_)
+      {
+        if (pkey == cur->key_) {
+          found_pkey = true;
+          part = cur->part_;
+          if (OB_NOT_NULL(part)) {
+            ret = OB_SUCCESS;
+          }
+          break;
+        }
+      }
+    }
+    if (OB_SUCCESS != ret) {
+      STORAGE_LOG(
+          WARN, "get_part_info fail", K(ret), K(pkey), K(found_pkey), K(this), K_(list), K_(contains_part_info));
+    }
+    return ret;
+  }
 
 private:
   TCRWLock lock_;
   List list_;
+  bool contains_part_info_;
 };
 
 class ObPGPartitionGuard {
@@ -285,10 +332,30 @@ public:
     (void)set_pg_partition(pkey, map);
   }
 
-  int set_pg_partition(const common::ObPartitionKey& pkey, const ObPGPartitionMap& map)
+  ObPGPartitionGuard(const common::ObPartitionKey &pkey, const ObPGPartitionList &part_list)
+      : pg_partition_(NULL), map_(NULL)
+  {
+    (void)set_pg_partition(pkey, part_list);
+  }
+
+  int set_pg_partition(const common::ObPartitionKey &pkey, const ObPGPartitionList &part_list)
   {
     int ret = OB_SUCCESS;
+    if (!pkey.is_valid()) {
+      ret = OB_INVALID_ARGUMENT;
+      STORAGE_LOG(WARN, "invalid argument", K(ret), K(pkey));
+    } else if (OB_FAIL(part_list.get_part_info(pkey, pg_partition_))) {
+      STORAGE_LOG(WARN, "get pg part from partition list fail", K(ret), K(pkey), KP(&part_list));
+    } else {
+      pkey_ = pkey;
+      map_ = NULL;
+    }
+    return ret;
+  }
 
+  int set_pg_partition(const common::ObPartitionKey &pkey, const ObPGPartitionMap &map)
+  {
+    int ret = OB_SUCCESS;
     if (!pkey.is_valid()) {
       ret = OB_INVALID_ARGUMENT;
       STORAGE_LOG(WARN, "invalid argument", K(ret), K(pkey));
@@ -298,7 +365,6 @@ public:
       pkey_ = pkey;
       map_ = &const_cast<ObPGPartitionMap&>(map);
     }
-
     return ret;
   }
 

src/storage/ob_pg_storage.h

@@ -168,9 +168,8 @@ public:
   int set_emergency_release();
   int get_active_memtable(ObTableHandle& handle);
 
-  // PG, pg partition
-  int get_pg_partition(const common::ObPartitionKey& pkey, ObPGPartitionGuard& guard);
-  const common::ObPartitionKey& get_partition_key() const
+  int get_pg_partition(const common::ObPartitionKey &pkey, ObPGPartitionGuard &guard) const;
+  const common::ObPartitionKey &get_partition_key() const
   {
     return pkey_;
   }
@@ -308,8 +307,9 @@ public:
   {
     return pg_memtable_mgr_.get_readable_info();
   }
-  int get_pkey_for_table(const int64_t table_id, ObPartitionKey& pkey);
-  int update_split_state_after_merge(int64_t& split_state);
+  int get_pkey_for_table(const int64_t table_id, ObPartitionKey &pkey);
+  int update_split_state_after_merge(int64_t &split_state);
+  int post_del_pg();
   int remove_all_pg_index();
   int update_split_table_store(
       const common::ObPartitionKey& pkey, int64_t table_id, bool is_major_split, ObTablesHandle& handle);
@@ -448,6 +448,29 @@ private:
   private:
     ObPartitionArray& pkeys_;
   };
+  class LocalizePGPartitionFunctor {
+  public:
+    explicit LocalizePGPartitionFunctor(ObPGPartitionMap &map) : map_(map)
+    {}
+    ~LocalizePGPartitionFunctor()
+    {}
+    int operator()(const common::ObPartitionKey &pkey, ObPGPartition *&part)
+    {
+      int ret = OB_SUCCESS;
+      if (!pkey.is_valid()) {
+        ret = OB_INVALID_ARGUMENT;
+        STORAGE_LOG(ERROR, "invalid pkey", K(pkey));
+      } else if (OB_FAIL(map_.get(pkey, part))) {
+        STORAGE_LOG(ERROR, "fail to get part", K(ret), K(pkey));
+      } else {
+        STORAGE_LOG(INFO, "localize part OK", K(pkey), K(*part));
+      }
+      return ret;
+    }
+
+  private:
+    ObPGPartitionMap &map_;
+  };
   class RemovePGIndexFunctor {
   public:
     explicit RemovePGIndexFunctor(ObPartitionGroupIndex& pg_index) : pg_index_(pg_index)
@@ -462,6 +485,8 @@ private:
         STORAGE_LOG(ERROR, "invalid pkey", K(pkey));
       } else if (OB_FAIL(pg_index_.remove_partition(pkey))) {
         STORAGE_LOG(ERROR, "failed to remove pg index", K(ret), K(pkey));
+      } else {
+        STORAGE_LOG(INFO, "remove part from index OK", K(pkey));
       }
       return ret;
     }
@@ -503,13 +528,20 @@ private:
   public:
     RemovePGPartitionFunctor(ObPGPartitionMap& map) : map_(map)
     {}
-    void operator()(const common::ObPartitionKey& pkey)
+    int operator()(const common::ObPartitionKey &pkey, ObPGPartition *part = NULL)
     {
+      int ret = OB_SUCCESS;
       if (!pkey.is_valid()) {
+        ret = OB_INVALID_ARGUMENT;
         STORAGE_LOG(WARN, "invalid pkey", K(pkey));
       } else {
-        map_.del(pkey);
+        if (part != NULL) {
+          map_.revert(part);
+        } else if (OB_FAIL(map_.del(pkey))) {
+          STORAGE_LOG(WARN, "del pg partition from map fail", K(pkey));
+        }
       }
+      return ret;
     }
 
   private:
@@ -634,13 +666,14 @@ private:
   bool is_inited_;
   bool is_removed_;
   common::ObPartitionKey pkey_;
-  ObIPartitionComponentFactory* cp_fty_;
-  transaction::ObTransService* txs_;
-  clog::ObIPartitionLogService* pls_;
-  ObIPartitionGroup* pg_;
-  ObPGPartition* pg_partition_;
-  share::schema::ObMultiVersionSchemaService* schema_service_;
-  ObPartitionKeyList partition_list_;
+  ObIPartitionComponentFactory *cp_fty_;
+  transaction::ObTransService *txs_;
+  clog::ObIPartitionLogService *pls_;
+  ObIPartitionGroup *pg_;
+  ObPGPartition *pg_partition_;
+  share::schema::ObMultiVersionSchemaService *schema_service_;
+  ObPGPartitionList partition_list_;
+  bool part_list_contains_part_info_;  // true when pg removed
 
   // pg meta and memstore
   common::ObBucketLock bucket_lock_;