[CP] fix table and tenant has different paxos num

src/rootserver/restore/ob_restore_scheduler.cpp

@@ -2374,18 +2374,13 @@ int ObRestoreScheduler::convert_table_options(const uint64_t tenant_id)
         if (OB_FAIL(ret)) {
         } else if (!table->get_primary_zone().empty()) {
           int tmp_ret = OB_SUCCESS;
-          ObArray<share::ObZoneReplicaAttrSet> locality;
-          const ObString &locality_str = table->get_locality_str();
           ObSimpleTableSchemaV2 new_table;
+          const ObString &locality_str = table->get_locality_str();
           if (OB_FAIL(new_table.assign(*table))) {
             LOG_WARN("fail to assign new table", K(ret), KPC(table));
           } else if (OB_SUCCESS !=
                      (tmp_ret = ddl_service_->check_create_table_replica_options(new_table, schema_guard))) {
             LOG_WARN("locality not match", K(tmp_ret), K(table_id), K(locality_str));
-          } else if (OB_SUCCESS != (tmp_ret = table->get_zone_replica_attr_array(locality))) {
-            LOG_WARN("fail to get locality array", K(tmp_ret), K(table_id), K(locality_str));
-          } else if (OB_SUCCESS != (tmp_ret = check_locality_valid(locality))) {
-            LOG_WARN("locality not supported", K(tmp_ret), K(table_id), K(locality_str));
           }
           if (OB_SUCCESS != tmp_ret) {
             LOG_INFO("backup primary_zone not match, just reset", K(tmp_ret), K(table_id), K(locality_str));
@@ -2399,6 +2394,9 @@ int ObRestoreScheduler::convert_table_options(const uint64_t tenant_id)
         if (OB_FAIL(ret)) {
         } else if (!table->get_locality_str().empty()) {
           int tmp_ret = OB_SUCCESS;
+          ObArray<share::ObZoneReplicaAttrSet> locality;
+          const ObString &locality_str = table->get_locality_str();
+
           if (OB_SUCCESS !=
               (tmp_ret = ddl_service_->check_table_locality_with_tenant(schema_guard, *tenant_schema, *table))) {
             LOG_INFO("backup locality not match, just reset",
@@ -2407,6 +2405,13 @@ int ObRestoreScheduler::convert_table_options(const uint64_t tenant_id)
                 table->get_table_id(),
                 "locality",
                 table->get_locality_str().ptr());
+          } else if (OB_SUCCESS != (tmp_ret = table->get_zone_replica_attr_array(locality))) {
+            LOG_WARN("fail to get locality array", K(tmp_ret), K(table_id), K(locality_str));
+          } else if (OB_SUCCESS != (tmp_ret = check_locality_valid(locality))) {
+            LOG_WARN("locality not supported", K(tmp_ret), K(table_id), K(locality_str));
+          }
+
+          if (OB_SUCCESS != tmp_ret) {
             arg.type_ = ObRestoreModifySchemaArg::RESET_TABLE_LOCALITY;
             if (OB_FAIL(rpc_proxy_->timeout(DEFAULT_TIMEOUT).modify_schema_in_restore(arg))) {
               LOG_WARN("fail to modify table's locality", K(ret), K(arg));

src/share/schema/ob_schema_struct.cpp

@@ -2649,14 +2649,8 @@ int ObDatabaseSchema::get_first_primary_zone_inherit(share::schema::ObSchemaGett
     const common::ObIArray<rootserver::ObReplicaAddr>& replica_addrs, common::ObZone& first_primary_zone) const
 {
   int ret = OB_SUCCESS;
-  const share::schema::ObSimpleTenantSchema* simple_tenant = nullptr;
   first_primary_zone.reset();
-  if (OB_FAIL(schema_guard.get_tenant_info(tenant_id_, simple_tenant))) {
-    LOG_WARN("fail to get tenant info", K(ret), K_(tenant_id));
-  } else if (OB_UNLIKELY(nullptr == simple_tenant)) {
-    ret = OB_TENANT_NOT_EXIST;
-    LOG_WARN("tenant schema ptr is null", K(ret), KPC(simple_tenant));
-  } else if (primary_zone_.empty() || simple_tenant->is_restore()) {
+  if (primary_zone_.empty()) {
     const share::schema::ObTenantSchema* tenant_schema = nullptr;
     if (OB_FAIL(schema_guard.get_tenant_info(tenant_id_, tenant_schema))) {
       LOG_WARN("fail to get tenant info", K(ret), K_(tenant_id));
@@ -2722,17 +2716,7 @@ int ObDatabaseSchema::get_primary_zone_inherit(ObSchemaGetterGuard& schema_guard
   int ret = OB_SUCCESS;
   bool use_tenant_primary_zone = GCTX.is_standby_cluster() && OB_SYS_TENANT_ID != tenant_id_;
   primary_zone.reset();
-  if (!use_tenant_primary_zone) {
-    const share::schema::ObSimpleTenantSchema* simple_tenant = nullptr;
-    if (OB_FAIL(schema_guard.get_tenant_info(tenant_id_, simple_tenant))) {
-      LOG_WARN("fail to get tenant info", K(ret), K_(tenant_id));
-    } else if (OB_UNLIKELY(nullptr == simple_tenant)) {
-      ret = OB_TENANT_NOT_EXIST;
-      LOG_WARN("tenant schema ptr is null", K(ret), KPC(simple_tenant));
-    } else {
-      use_tenant_primary_zone = simple_tenant->is_restore();
-    }
-  }
+
   if (OB_FAIL(ret)) {
   } else if (!get_primary_zone().empty() && !use_tenant_primary_zone) {
     if (OB_FAIL(primary_zone.set_primary_zone_array(get_primary_zone_array()))) {
@@ -5233,16 +5217,9 @@ int ObTablegroupSchema::get_first_primary_zone_inherit(share::schema::ObSchemaGe
     const common::ObIArray<rootserver::ObReplicaAddr>& replica_addrs, common::ObZone& first_primary_zone) const
 {
   int ret = OB_SUCCESS;
-  const uint64_t tenant_id = get_tenant_id();
-  const share::schema::ObSimpleTenantSchema* simple_tenant = nullptr;
   first_primary_zone.reset();
-  if (OB_FAIL(schema_guard.get_tenant_info(tenant_id_, simple_tenant))) {
-    LOG_WARN("fail to get tenant info", K(ret), K_(tenant_id));
-  } else if (OB_UNLIKELY(nullptr == simple_tenant)) {
-    ret = OB_TENANT_NOT_EXIST;
-    LOG_WARN("tenant schema ptr is null", K(ret), KPC(simple_tenant));
-  } else if (get_primary_zone().empty() || simple_tenant->is_restore()) {
-    const share::schema::ObTenantSchema* tenant_schema = nullptr;
+  if (get_primary_zone().empty()) {
+    const share::schema::ObTenantSchema *tenant_schema = nullptr;
     if (OB_FAIL(schema_guard.get_tenant_info(tenant_id_, tenant_schema))) {
       LOG_WARN("fail to get tenant info", K(ret), K_(tenant_id));
     } else if (OB_UNLIKELY(nullptr == tenant_schema)) {
@@ -5358,15 +5335,22 @@ int ObTablegroupSchema::get_zone_replica_attr_array_inherit(
   int ret = OB_SUCCESS;
   bool use_tenant_locality = GCTX.is_standby_cluster() && OB_SYS_TENANT_ID != tenant_id_;
   locality.reset();
-  if (!use_tenant_locality) {
-    const share::schema::ObSimpleTenantSchema* simple_tenant = nullptr;
+  if (!use_tenant_locality && !locality_info_.locality_str_.empty()) {
+    const share::schema::ObSimpleTenantSchema *simple_tenant = nullptr;
     if (OB_FAIL(schema_guard.get_tenant_info(tenant_id_, simple_tenant))) {
       LOG_WARN("fail to get tenant info", K(ret), K_(tenant_id));
     } else if (OB_UNLIKELY(nullptr == simple_tenant)) {
       ret = OB_TENANT_NOT_EXIST;
       LOG_WARN("tenant schema ptr is null", K(ret), KPC(simple_tenant));
-    } else {
-      use_tenant_locality = simple_tenant->is_restore();
+    } else if (simple_tenant->is_restore()) {
+      bool has_not_f_replica = false;
+      if (OB_FAIL(check_has_own_not_f_replica(has_not_f_replica))) {
+        LOG_WARN("failed to check has not f replica", KR(ret));
+      } else if (has_not_f_replica) {
+        ret = OB_SCHEMA_EAGAIN;
+        LOG_WARN(
+            "has not full replica while tenant is restore, try latter", KR(ret), K(locality_info_), KPC(simple_tenant));
+      }
     }
   }
   if (OB_FAIL(ret)) {
@@ -5394,17 +5378,7 @@ int ObTablegroupSchema::get_locality_str_inherit(
   int ret = OB_SUCCESS;
   bool use_tenant_locality = OB_SYS_TENANT_ID != tenant_id_ && GCTX.is_standby_cluster();
   locality_str = &get_locality_str();
-  if (!use_tenant_locality) {
-    const share::schema::ObSimpleTenantSchema* simple_tenant = nullptr;
-    if (OB_FAIL(guard.get_tenant_info(tenant_id_, simple_tenant))) {
-      LOG_WARN("fail to get tenant info", K(ret), K_(tenant_id));
-    } else if (OB_UNLIKELY(nullptr == simple_tenant)) {
-      ret = OB_TENANT_NOT_EXIST;
-      LOG_WARN("tenant schema ptr is null", K(ret), KPC(simple_tenant));
-    } else {
-      use_tenant_locality = simple_tenant->is_restore();
-    }
-  }
+
   if (OB_FAIL(ret)) {
   } else if (use_tenant_locality || nullptr == locality_str || locality_str->empty()) {
     const ObSimpleTenantSchema* tenant_schema = nullptr;
@@ -5414,6 +5388,18 @@ int ObTablegroupSchema::get_locality_str_inherit(
       ret = OB_ERR_UNEXPECTED;
       LOG_WARN("fail to get tenant schema", K(ret), "tenant_id", get_tenant_id());
     } else {
+      if (tenant_schema->is_restore()) {
+        bool has_not_f_replica = false;
+        if (OB_FAIL(check_has_own_not_f_replica(has_not_f_replica))) {
+          LOG_WARN("failed to check has not f replica", KR(ret));
+        } else if (has_not_f_replica) {
+          ret = OB_SCHEMA_EAGAIN;
+          LOG_WARN("has not full replica while tenant is restore, try latter",
+              KR(ret),
+              K(locality_info_),
+              KPC(tenant_schema));
+        }
+      }
       locality_str = &tenant_schema->get_locality_str();
     }
   }
@@ -5493,17 +5479,7 @@ int ObTablegroupSchema::get_primary_zone_inherit(ObSchemaGetterGuard& schema_gua
   int ret = OB_SUCCESS;
   bool use_tenant_primary_zone = GCTX.is_standby_cluster() && OB_SYS_TENANT_ID != tenant_id_;
   primary_zone.reset();
-  if (!use_tenant_primary_zone) {
-    const share::schema::ObSimpleTenantSchema* simple_tenant = nullptr;
-    if (OB_FAIL(schema_guard.get_tenant_info(tenant_id_, simple_tenant))) {
-      LOG_WARN("fail to get tenant info", K(ret), K_(tenant_id));
-    } else if (OB_UNLIKELY(nullptr == simple_tenant)) {
-      ret = OB_TENANT_NOT_EXIST;
-      LOG_WARN("tenant schema ptr is null", K(ret), KPC(simple_tenant));
-    } else {
-      use_tenant_primary_zone = simple_tenant->is_restore();
-    }
-  }
+
   if (OB_FAIL(ret)) {
   } else if (!get_primary_zone().empty() && !use_tenant_primary_zone) {
     if (OB_FAIL(primary_zone.set_primary_zone_array(get_primary_zone_array()))) {
@@ -5605,6 +5581,27 @@ int ObTablegroupSchema::get_full_replica_num(share::schema::ObSchemaGetterGuard&
   return ret;
 }
 
+int ObTablegroupSchema::check_has_own_not_f_replica(bool &has_not_f_replica) const
+{
+  int ret = OB_SUCCESS;
+  has_not_f_replica = false;
+  if (locality_info_.locality_str_.empty()) {
+    has_not_f_replica = false;
+  } else {
+    FOREACH_CNT_X(locality, locality_info_.zone_replica_attr_array_, OB_SUCC(ret) && !has_not_f_replica)
+    {
+      if (OB_ISNULL(locality)) {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_WARN("get invalid locality set", K(ret), KP(locality));
+      } else if (locality->get_specific_replica_num() != locality->get_full_replica_num() ||
+                 OB_ALL_SERVER_CNT == locality->get_readonly_replica_num()) {
+        has_not_f_replica = true;
+      }
+    }
+  }
+  return ret;
+}
+
 int ObTablegroupSchema::get_paxos_replica_num(share::schema::ObSchemaGetterGuard& schema_guard, int64_t& num) const
 {
   int ret = OB_SUCCESS;

src/share/schema/ob_schema_struct.h

@@ -2647,6 +2647,7 @@ public:
   virtual const common::ObString &get_previous_locality_str() const = 0;
   virtual int get_paxos_replica_num(share::schema::ObSchemaGetterGuard &schema_guard, int64_t &num) const = 0;
   virtual share::ObDuplicateScope get_duplicate_scope() const = 0;
+  virtual int check_has_own_not_f_replica(bool &has_not_f_replica) const = 0;
   virtual void set_duplicate_scope(const share::ObDuplicateScope duplicate_scope) = 0;
   virtual void set_duplicate_scope(const int64_t duplicate_scope) = 0;
   inline virtual int64_t get_part_func_expr_num() const
@@ -3273,6 +3274,7 @@ public:
   {
     return true;
   }
+  virtual int check_has_own_not_f_replica(bool &has_not_f_replica) const override;
 
   DECLARE_VIRTUAL_TO_STRING;
 

src/share/schema/ob_table_schema.cpp

@@ -474,14 +474,21 @@ int ObSimpleTableSchemaV2::get_first_primary_zone_inherit(share::schema::ObSchem
     const common::ObIArray<rootserver::ObReplicaAddr>& replica_addrs, common::ObZone& first_primary_zone) const
 {
   int ret = OB_SUCCESS;
-  const share::schema::ObSimpleTenantSchema* simple_tenant = nullptr;
   first_primary_zone.reset();
-  if (OB_FAIL(schema_guard.get_tenant_info(tenant_id_, simple_tenant))) {
+  //sys table use tenant
+  bool use_tenant_primary_zone = false;
+  const share::schema::ObTenantSchema *tenant_schema = nullptr;
+  if (OB_FAIL(schema_guard.get_tenant_info(tenant_id_, tenant_schema))) {
     LOG_WARN("fail to get tenant info", K(ret), K_(tenant_id));
-  } else if (OB_UNLIKELY(nullptr == simple_tenant)) {
+  } else if (OB_UNLIKELY(nullptr == tenant_schema)) {
     ret = OB_TENANT_NOT_EXIST;
-    LOG_WARN("tenant schema ptr is null", K(ret), KPC(simple_tenant));
-  } else if (primary_zone_.empty() || simple_tenant->is_restore()) {
+    LOG_WARN("tenant schema ptr is null", K(ret), KPC(tenant_schema));
+  } else if (tenant_schema->is_restore() && common::is_sys_table(table_id_)) {
+    use_tenant_primary_zone = true;
+  }
+
+  if (OB_FAIL(ret)) {
+  } else if (primary_zone_.empty() || use_tenant_primary_zone) {
     const uint64_t tablegroup_id = get_tablegroup_id();
     if (OB_INVALID_ID == tablegroup_id || !is_new_tablegroup_id(tablegroup_id)) {
       const share::schema::ObDatabaseSchema* db_schema = nullptr;
@@ -604,19 +611,33 @@ int ObSimpleTableSchemaV2::get_zone_replica_attr_array_inherit(
     ObSchemaGetterGuard& schema_guard, ZoneLocalityIArray& locality) const
 {
   int ret = OB_SUCCESS;
-  bool use_tenant_locality = OB_SYS_TENANT_ID != tenant_id_ && GCTX.is_standby_cluster();
+  bool use_tenant_locality = false;
   locality.reuse();
   if (!has_partition()) {
     // No partition, no concept of locality
-  } else if (!use_tenant_locality) {
-    const share::schema::ObSimpleTenantSchema* simple_tenant = nullptr;
+  } else {
+    const share::schema::ObSimpleTenantSchema *simple_tenant = nullptr;
     if (OB_FAIL(schema_guard.get_tenant_info(tenant_id_, simple_tenant))) {
       LOG_WARN("fail to get tenant info", K(ret), K_(tenant_id));
     } else if (OB_UNLIKELY(nullptr == simple_tenant)) {
       ret = OB_TENANT_NOT_EXIST;
       LOG_WARN("tenant schema ptr is null", K(ret), KPC(simple_tenant));
-    } else {
-      use_tenant_locality = simple_tenant->is_restore();
+    } else if (simple_tenant->is_restore() && common::is_sys_table(table_id_)) {
+      use_tenant_locality = true;
+    }
+
+    if (OB_FAIL(ret)) {
+    } else if (!use_tenant_locality && !locality_str_.empty()) {
+      if (simple_tenant->is_restore()) {
+        bool has_not_f_replica = false;
+        if (OB_FAIL(check_has_own_not_f_replica(has_not_f_replica))) {
+          LOG_WARN("failed to check has not f replica", KR(ret));
+        } else if (has_not_f_replica) {
+          ret = OB_SCHEMA_EAGAIN;
+          LOG_WARN("has not full replica while tenant is restore, try latter", KR(ret),
+              K(locality_str_), KPC(simple_tenant));
+        }
+      }
     }
   }
   if (OB_FAIL(ret)) {
@@ -654,28 +675,21 @@ int ObSimpleTableSchemaV2::get_primary_zone_inherit(
     ObSchemaGetterGuard& schema_guard, ObPrimaryZone& primary_zone) const
 {
   int ret = OB_SUCCESS;
-  bool use_tenant_primary_zone = OB_SYS_TENANT_ID != tenant_id_ && GCTX.is_standby_cluster();
   primary_zone.reset();
-  if (!use_tenant_primary_zone) {
-    const share::schema::ObSimpleTenantSchema* simple_tenant = nullptr;
-    if (OB_FAIL(schema_guard.get_tenant_info(tenant_id_, simple_tenant))) {
-      LOG_WARN("fail to get tenant info", K(ret), K_(tenant_id));
-    } else if (OB_UNLIKELY(nullptr == simple_tenant)) {
-      ret = OB_TENANT_NOT_EXIST;
-      LOG_WARN("tenant schema ptr is null", K(ret), KPC(simple_tenant));
-    } else {
-      use_tenant_primary_zone = simple_tenant->is_restore();
-    }
+  bool use_tenant_primary_zone = false;
+  const ObTenantSchema *tenant_schema = NULL;
+  if (OB_FAIL(schema_guard.get_tenant_info(tenant_id_, tenant_schema))) {
+    LOG_WARN("fail to get tenant info", K(ret), K_(tenant_id));
+  } else if (OB_UNLIKELY(nullptr == tenant_schema)) {
+    ret = OB_TENANT_NOT_EXIST;
+    LOG_WARN("tenant schema ptr is null", K(ret), KPC(tenant_schema));
+  } else if (tenant_schema->is_restore() && common::is_sys_table(table_id_)) {
+    use_tenant_primary_zone = true;
   }
+
   if (OB_FAIL(ret)) {
   } else if (use_tenant_primary_zone) {
-    const ObTenantSchema* tenant_schema = NULL;
-    if (OB_FAIL(schema_guard.get_tenant_info(get_tenant_id(), tenant_schema))) {
-      LOG_WARN("fail to get tenant schema", K(ret), K(database_id_), K(tenant_id_));
-    } else if (OB_UNLIKELY(NULL == tenant_schema)) {
-      ret = OB_ERR_UNEXPECTED;
-      LOG_WARN("tenant schema null", K(ret), K(database_id_), K(tenant_id_), KP(tenant_schema));
-    } else if (OB_FAIL(tenant_schema->get_primary_zone_inherit(schema_guard, primary_zone))) {
+    if (OB_FAIL(tenant_schema->get_primary_zone_inherit(schema_guard, primary_zone))) {
       LOG_WARN("fail to get primary zone array", K(ret), K(database_id_), K(tenant_id_));
     }
   } else if (!get_primary_zone().empty()) {
@@ -1179,6 +1193,25 @@ int ObSimpleTableSchemaV2::get_all_replica_num(share::schema::ObSchemaGetterGuar
   return ret;
 }
 
+int ObSimpleTableSchemaV2::check_has_own_not_f_replica(bool &has_not_f_replica) const
+{
+  int ret = OB_SUCCESS;
+  has_not_f_replica = false;
+  if (locality_str_.empty()) {
+    has_not_f_replica = false;
+  } else {
+    for (int64_t i = 0; i < zone_replica_attr_array_.count() && !has_not_f_replica; ++i) {
+      const SchemaZoneReplicaAttrSet &set = zone_replica_attr_array_.at(i);
+      if (set.get_specific_replica_num() != set.get_full_replica_num() ||
+          OB_ALL_SERVER_CNT == set.get_readonly_replica_num()) {
+        // R@all_server not in get_specific_replica_num
+        has_not_f_replica = true;
+      }
+    }
+  }
+  return ret;
+}
+
 int ObSimpleTableSchemaV2::check_has_all_server_readonly_replica(
     share::schema::ObSchemaGetterGuard& guard, bool& has) const
 {
@@ -1438,20 +1471,31 @@ int ObSimpleTableSchemaV2::get_locality_str_inherit(
     share::schema::ObSchemaGetterGuard& guard, const common::ObString*& locality_str) const
 {
   int ret = OB_SUCCESS;
-  bool use_tenant_locality = OB_SYS_TENANT_ID != tenant_id_ && GCTX.is_standby_cluster();
   locality_str = NULL;
-  if (OB_INVALID_ID == get_table_id()) {
+  bool use_tenant_locality = false;
+  const share::schema::ObSimpleTenantSchema *simple_tenant = nullptr;
+  if (OB_FAIL(guard.get_tenant_info(tenant_id_, simple_tenant))) {
+    LOG_WARN("fail to get tenant info", K(ret), K_(tenant_id));
+  } else if (OB_UNLIKELY(nullptr == simple_tenant)) {
+    ret = OB_TENANT_NOT_EXIST;
+    LOG_WARN("tenant schema ptr is null", K(ret), KPC(simple_tenant));
+  } else if (simple_tenant->is_restore() && common::is_sys_table(table_id_)) {
+    use_tenant_locality = true;
+  }
+
+  if (OB_FAIL(ret)) {
+  } else if (OB_INVALID_ID == get_table_id()) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("invalid table_id", K(ret), K_(table_id));
-  } else if (!use_tenant_locality) {
-    const share::schema::ObSimpleTenantSchema* simple_tenant = nullptr;
-    if (OB_FAIL(guard.get_tenant_info(tenant_id_, simple_tenant))) {
-      LOG_WARN("fail to get tenant info", K(ret), K_(tenant_id));
-    } else if (OB_UNLIKELY(nullptr == simple_tenant)) {
-      ret = OB_TENANT_NOT_EXIST;
-      LOG_WARN("tenant schema ptr is null", K(ret), KPC(simple_tenant));
-    } else {
-      use_tenant_locality = simple_tenant->is_restore();
+  } else if (!use_tenant_locality && !get_locality_str().empty()) {
+    if (simple_tenant->is_restore()) {
+      bool has_not_f_replica = false;
+      if (OB_FAIL(check_has_own_not_f_replica(has_not_f_replica))) {
+        LOG_WARN("failed to check has not f replica", KR(ret));
+      } else if (has_not_f_replica) {
+        ret = OB_SCHEMA_EAGAIN;
+        LOG_WARN("has not full replica while tenant is restore, try latter", KR(ret), KPC(simple_tenant));
+      }
     }
   }
   if (OB_FAIL(ret)) {

src/share/schema/ob_table_schema.h

@@ -419,6 +419,8 @@ public:
       const common::ObRegion& region, bool& readonly_at_all) const;
   int check_has_all_server_readonly_replica(share::schema::ObSchemaGetterGuard& guard, bool& has) const;
   int check_is_all_server_readonly_replica(share::schema::ObSchemaGetterGuard& guard, bool& is) const;
+  virtual int check_has_own_not_f_replica(bool &has_not_f_replica) const override;
+
   void reset_locality_options();
 
   void reset_primary_zone_options();