Fix root inspection error

src/rootserver/ob_root_inspection.cpp

@@ -937,9 +937,8 @@ int ObRootInspection::check_sys_stat_(const uint64_t tenant_id)
     LOG_WARN("schema_service is null", KR(ret));
   } else if (OB_FAIL(check_cancel())) {
     LOG_WARN("check_cancel failed", KR(ret));
-  } else if (!schema_service_->is_tenant_full_schema(tenant_id)) {
-    ret = OB_EAGAIN;
-    LOG_WARN("schema is not ready, try again", KR(ret), K(tenant_id));
+  } else if (OB_FAIL(check_tenant_status_(tenant_id))) {
+    LOG_WARN("fail to check tenant status", KR(ret), K(tenant_id));
   } else if (OB_FAIL(sys_stat.set_initial_values(tenant_id))) {
     LOG_WARN("set initial values failed", KR(ret), K(tenant_id));
   } else if (OB_FAIL(extra_cond.assign_fmt("tenant_id = %lu",
@@ -999,9 +998,8 @@ int ObRootInspection::check_sys_param_(const uint64_t tenant_id)
     LOG_WARN("schema_service is null", KR(ret));
   } else if (OB_FAIL(check_cancel())) {
     LOG_WARN("check_cancel failed", KR(ret));
-  } else if (!schema_service_->is_tenant_full_schema(tenant_id)) {
-    ret = OB_EAGAIN;
-    LOG_WARN("schema is not ready, try again", KR(ret), K(tenant_id));
+  } else if (OB_FAIL(check_tenant_status_(tenant_id))) {
+    LOG_WARN("fail to check tenant status", KR(ret), K(tenant_id));
   } else if (OB_FAIL(extra_cond.assign_fmt("tenant_id = %lu",
              ObSchemaUtils::get_extract_tenant_id(exec_tenant_id, tenant_id)))) {
     LOG_WARN("extra_cond assign_fmt failed", KR(ret), K(tenant_id));
@@ -1271,15 +1269,6 @@ int ObRootInspection::check_sys_table_schemas_(
              || OB_INVALID_TENANT_ID == tenant_id)) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("invalid tenant_id", KR(ret), K(tenant_id));
-  } else if (OB_ISNULL(schema_service_)) {
-    ret = OB_ERR_UNEXPECTED;
-    LOG_WARN("schema_service is null", KR(ret));
-  } else if (OB_FAIL(schema_service_->get_tenant_refreshed_schema_version(
-                     tenant_id, schema_version))) {
-    LOG_WARN("fail to get tenant refreshed schema_version", KR(ret), K(tenant_id));
-  } else if (!ObSchemaService::is_formal_version(schema_version)) {
-    ret = OB_EAGAIN;
-    LOG_WARN("schema is not ready, try again", KR(ret), K(tenant_id), K(schema_version));
   } else {
     const schema_create_func *creator_ptr_array[] = {
       share::all_core_table_schema_creator,
@@ -1302,6 +1291,8 @@ int ObRootInspection::check_sys_table_schemas_(
         table_schema.reset();
         if (OB_FAIL(check_cancel())) {
           LOG_WARN("check_cancel failed", KR(ret));
+        } else if (OB_FAIL(check_tenant_status_(tenant_id))) {
+          LOG_WARN("fail to check tenant status", KR(ret), K(tenant_id));
         } else if (OB_FAIL((*creator_ptr)(table_schema))) {
           LOG_WARN("create table schema failed", KR(ret));
         } else if (!is_sys_tenant(tenant_id)
@@ -1326,6 +1317,10 @@ int ObRootInspection::check_sys_table_schemas_(
             LOG_WARN("check sys view failed", KR(tmp_ret), K(tenant_id),
                      "table_id", table_schema.get_table_id(), "table_name", table_schema.get_table_name());
             back_ret = OB_SUCCESS == back_ret ? tmp_ret : back_ret;
+            // sql may has occur other error except OB_SCHEMA_ERROR, we should not continue is such situation.
+            if (OB_SCHEMA_ERROR != tmp_ret) {
+              ret = OB_SUCC(ret) ? back_ret : tmp_ret;
+            }
           }
         }
       } // end for
@@ -1585,7 +1580,12 @@ int ObRootInspection::check_sys_view_(
         }
       }
       if (OB_FAIL(ret)) {
+        if (OB_ERR_VIEW_INVALID == ret) {
+          ret = OB_SCHEMA_ERROR;
           LOG_ERROR("check sys view: expand failed", KR(ret), K(tenant_id), K(table_name));
+        } else {
+          LOG_WARN("check sys view: expand failed", KR(ret), K(tenant_id), K(table_name));
+        }
       } else if (OB_ISNULL(result = res.get_result())) {
         ret = OB_ERR_UNEXPECTED;
         LOG_WARN("failed to get sql result", KR(ret), K(tenant_id));
@@ -1865,6 +1865,8 @@ int ObRootInspection::check_data_version_()
     FOREACH_X(tenant_id, tenant_ids, OB_SUCC(ret)) {
       if (OB_FAIL(check_cancel())) {
         LOG_WARN("check_cancel failed", KR(ret));
+      } else if (OB_FAIL(check_tenant_status_(*tenant_id))) {
+        LOG_WARN("fail to check tenant status", KR(ret), K(*tenant_id));
       } else if (OB_TMP_FAIL(check_data_version_(*tenant_id))) {
         LOG_WARN("fail to check data version by tenant", KR(tmp_ret), K(*tenant_id));
         backup_ret = OB_SUCCESS == backup_ret ? tmp_ret : backup_ret;
@@ -1917,6 +1919,36 @@ int ObRootInspection::check_cancel()
   return ret;
 }
 
+int ObRootInspection::check_tenant_status_(const uint64_t tenant_id)
+{
+  int ret = OB_SUCCESS;
+  ObSchemaGetterGuard guard;
+  const ObSimpleTenantSchema *tenant = NULL;
+  int64_t schema_version = OB_INVALID_VERSION;
+  if (OB_ISNULL(schema_service_)) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("schema service is null", KR(ret));
+  } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, guard))) {
+    LOG_WARN("fail to get schema guard", KR(ret));
+  } else if (OB_FAIL(guard.get_tenant_info(tenant_id, tenant))) {
+    LOG_WARN("fail to get tenant schema", KR(ret), K(tenant_id));
+  } else if (OB_ISNULL(tenant)) {
+    // tenant may has been dropped;
+    ret = OB_EAGAIN;
+    LOG_WARN("tenant may be dropped, don't continue", KR(ret), K(tenant_id));
+  } else if (!tenant->is_normal()) {
+    ret = OB_EAGAIN;
+    LOG_WARN("tenant status is not noraml, should check next round", KR(ret), K(tenant_id));
+  } else if (OB_FAIL(schema_service_->get_tenant_refreshed_schema_version(tenant_id, schema_version))) {
+    LOG_WARN("fail to get tenant schema version", KR(ret), K(tenant_id));
+  } else if (!ObSchemaService::is_formal_version(schema_version)) {
+    ret = OB_EAGAIN;
+    LOG_WARN("schema version is not formal, observer may be restarting or inner table schema changed, "
+             "should check next round", KR(ret), K(tenant_id), K(schema_version));
+  }
+  return ret;
+}
+
 ObUpgradeInspection::ObUpgradeInspection()
   : inited_(false), schema_service_(NULL), root_inspection_(NULL)
 {

src/rootserver/ob_root_inspection.h

@@ -225,6 +225,7 @@ private:
   int check_sys_view_(const uint64_t tenant_id,
                       const share::schema::ObTableSchema &hard_code_table);
   int check_cancel();
+  int check_tenant_status_(const uint64_t tenant_id);
 private:
   bool inited_;
   volatile bool stopped_;

src/share/config/ob_config.cpp

@@ -996,15 +996,16 @@ ObConfigVersionItem::ObConfigVersionItem(ObConfigContainer *container,
 
 bool ObConfigVersionItem::set(const char *str)
 {
-  int64_t old_value = value_;
+  int64_t old_value = get_value();
   bool value_update = value_updated();
   bool valid = ObConfigIntegralItem::set(str);
-  if (valid && value_update && old_value > value_) {
-    OB_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "Attention!!! data version is retrogressive", K(old_value), K_(value));
+  int64_t new_value = get_value();
+  if (valid && value_update && old_value > new_value) {
+    OB_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "Attention!!! data version is retrogressive", K(old_value), K(new_value));
   }
-  if (old_value != value_) {
+  if (value_update && old_value != new_value) {
     ObTaskController::get().allow_next_syslog();
-    OB_LOG(INFO, "Config data version changed", K(old_value), K_(value), K(value_update), K(valid));
+    OB_LOG(INFO, "Config data version changed", K(old_value), K(new_value), K(value_update), K(valid));
   }
   return valid;
 }

src/share/config/ob_config.h

@@ -846,7 +846,6 @@ protected:
   virtual int64_t parse(const char *str, bool &valid) const override;
 
 private:
-  uint64_t value_;
   DISALLOW_COPY_AND_ASSIGN(ObConfigVersionItem);
 };
 

src/share/ob_share_util.cpp

@@ -17,6 +17,7 @@
 #include "lib/time/ob_time_utility.h"
 #include "lib/oblog/ob_log_module.h"
 #include "share/ob_cluster_version.h" // for GET_MIN_DATA_VERSION
+#include "lib/mysqlclient/ob_isql_client.h"
 namespace oceanbase
 {
 using namespace common;
@@ -112,5 +113,82 @@ int ObShareUtil::generate_arb_replica_num(
   }
   return ret;
 }
+
+int ObShareUtil::fetch_current_cluster_version(
+    common::ObISQLClient &client,
+    uint64_t &cluster_version)
+{
+  int ret = OB_SUCCESS;
+  ObSqlString sql;
+  sqlclient::ObMySQLResult *result = NULL;
+  SMART_VAR(ObMySQLProxy::MySQLResult, res) {
+  if (OB_FAIL(sql.assign_fmt(
+      "select value from %s where name = '%s'",
+      OB_ALL_SYS_PARAMETER_TNAME, "min_observer_version"))) {
+    LOG_WARN("fail to assign fmt", KR(ret), K(sql));
+  } else if (OB_FAIL(client.read(res, OB_SYS_TENANT_ID, sql.ptr()))) {
+    LOG_WARN("execute sql failed", KR(ret), K(sql));
+  } else if (OB_ISNULL(result = res.get_result())) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("fail to get result", KR(ret));
+  } else if (OB_FAIL(result->next())) {
+    if (OB_ITER_END == ret) {
+      ret = OB_ENTRY_NOT_EXIST;
+      LOG_WARN("min_observer_version not exist, may be in bootstrap stage", KR(ret));
+    } else {
+      LOG_WARN("fail to get next", KR(ret));
+    }
+  } else {
+    ObString value;
+    EXTRACT_VARCHAR_FIELD_MYSQL(*result, "value", value);
+    if (FAILEDx(ObClusterVersion::get_version(value, cluster_version))) {
+      LOG_WARN("fail to get version", KR(ret), K(value));
+    }
+  }
+  } // end SMART_VAR
+  return ret;
+}
+
+int ObShareUtil::fetch_current_data_version(
+    common::ObISQLClient &client,
+    const uint64_t tenant_id,
+    uint64_t &data_version)
+{
+  int ret = OB_SUCCESS;
+  ObSqlString sql;
+  sqlclient::ObMySQLResult *result = NULL;
+  const uint64_t exec_tenant_id = gen_meta_tenant_id(tenant_id);
+  SMART_VAR(ObMySQLProxy::MySQLResult, res) {
+  if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id
+                  || OB_INVALID_TENANT_ID == exec_tenant_id)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("tenant_id is invalid", KR(ret), K(tenant_id), K(exec_tenant_id));
+  } else if (OB_FAIL(sql.assign_fmt(
+      "select value from %s where name = '%s'",
+      OB_TENANT_PARAMETER_TNAME, "compatible"))) {
+    LOG_WARN("fail to assign fmt", KR(ret), K(tenant_id), K(sql));
+  } else if (OB_FAIL(client.read(res, exec_tenant_id, sql.ptr()))) {
+    LOG_WARN("execute sql failed", KR(ret), K(tenant_id), K(sql));
+  } else if (OB_ISNULL(result = res.get_result())) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("fail to get result", KR(ret), K(tenant_id));
+  } else if (OB_FAIL(result->next())) {
+    if (OB_ITER_END == ret) {
+      ret = OB_ENTRY_NOT_EXIST;
+      LOG_WARN("compatible not exist, create tenant process may be doing or failed ",
+               KR(ret), K(tenant_id));
+    } else {
+      LOG_WARN("fail to get next", KR(ret), K(tenant_id));
+    }
+  } else {
+    ObString value;
+    EXTRACT_VARCHAR_FIELD_MYSQL(*result, "value", value);
+    if (FAILEDx(ObClusterVersion::get_version(value, data_version))) {
+      LOG_WARN("fail to get version", KR(ret), K(value));
+    }
+  }
+  } // end SMART_VAR
+  return ret;
+}
 } //end namespace share
 } //end namespace oceanbase

src/share/ob_share_util.h

@@ -18,6 +18,7 @@ namespace oceanbase
 namespace common
 {
 class ObTimeoutCtx;
+class ObISQLClient;
 }
 namespace share
 {
@@ -42,6 +43,15 @@ public:
       const uint64_t tenant_id,
       const ObLSID &ls_id,
       int64_t &arb_replica_num);
+
+  static int fetch_current_cluster_version(
+             common::ObISQLClient &client,
+             uint64_t &cluster_version);
+
+  static int fetch_current_data_version(
+             common::ObISQLClient &client,
+             const uint64_t tenant_id,
+             uint64_t &data_version);
 };
 }//end namespace share
 }//end namespace oceanbase