[CP] fix bad MEMORY_LOG when ServerGTimer hang

deps/oblib/src/lib/alloc/ob_malloc_allocator.cpp

@@ -480,7 +480,8 @@ void ObMallocAllocator::print_tenant_memory_usage(uint64_t tenant_id) const
         if (OB_SUCC(ret)) {
           ObPageManagerCenter::get_instance().print_tenant_stat(tenant_id, buf, BUFLEN, ctx_pos);
         }
-
+        buf[std::min(ctx_pos, BUFLEN - 1)] = '\0';
+        allow_next_syslog();
         _LOG_INFO("[MEMORY] tenant: %lu, limit: %'lu hold: %'lu rpc_hold: %'lu cache_hold: %'lu "
                   "cache_used: %'lu cache_item_count: %'lu \n%s",
             tenant_id,

deps/oblib/src/lib/alloc/ob_tenant_ctx_allocator.cpp

@@ -22,7 +22,6 @@
 
 using namespace oceanbase::lib;
 using namespace oceanbase::common;
-
 int ObTenantCtxAllocator::iter_label(VisitFunc func) const
 {
   int ret = OB_SUCCESS;
@@ -139,7 +138,7 @@ void ObTenantCtxAllocator::print_usage() const
       return OB_SUCCESS;
     });
   }
-
+  allow_next_syslog();
   _LOG_INFO("\n[MEMORY] tenant_id=%5ld ctx_id=%25s hold=% '15ld used=% '15ld\n%s",
       tenant_id_,
       get_global_ctx_info().get_ctx_name(ctx_id_),

deps/oblib/src/lib/oblog/ob_log.cpp

@@ -39,6 +39,11 @@ using namespace oceanbase::lib;
 namespace oceanbase {
 namespace common {
 __thread char ObLogger::local_buf_[];
+
+void __attribute__((weak)) allow_next_syslog(int64_t)
+{
+  // do nothing
+}
 extern void update_easy_log_level();
 lib::ObRateLimiter* ObLogger::default_log_limiter_ = nullptr;
 RLOCAL(lib::ObRateLimiter*, ObLogger::tl_log_limiter_);

deps/oblib/src/lib/oblog/ob_log.h

@@ -56,6 +56,7 @@ class ObFIFOAllocator;
 class ObPLogItem;
 class ObString;
 class ObLogCompressor;
+extern void allow_next_syslog(int64_t count = 1);
 
 #define OB_LOGGER ::oceanbase::common::ObLogger::get_logger()
 #define OB_LOG_NEED_TO_PRINT(level) (OB_UNLIKELY(OB_LOGGER.need_to_print(OB_LOG_LEVEL_##level)))

src/observer/ob_server.cpp

@@ -402,6 +402,8 @@ void ObServer::destroy()
     ObBGThreadMonitor::get_instance().destroy();
     TG_DESTROY(lib::TGDefIDs::ServerGTimer);
     LOG_WARN("timer destroyed");
+    TG_DESTROY(lib::TGDefIDs::MemDumpTimer);
+    LOG_WARN("memory dump timer destroyed");
     TG_DESTROY(lib::TGDefIDs::FreezeTimer);
     LOG_WARN("freeze timer destroyed");
     TG_DESTROY(lib::TGDefIDs::SqlMemTimer);
@@ -646,6 +648,8 @@ int ObServer::stop()
   LOG_INFO("begin stop timer");
   TG_STOP(lib::TGDefIDs::ServerGTimer);
   LOG_WARN("timer stopped");
+  TG_STOP(lib::TGDefIDs::MemDumpTimer);
+  LOG_WARN("memory dump timer stopped");
   TG_STOP(lib::TGDefIDs::FreezeTimer);
   LOG_WARN("freeze timer stopped");
   TG_STOP(lib::TGDefIDs::SqlMemTimer);
@@ -756,6 +760,7 @@ int ObServer::wait()
 
   // timer
   TG_WAIT(lib::TGDefIDs::ServerGTimer);
+  TG_WAIT(lib::TGDefIDs::MemDumpTimer);
   TG_WAIT(lib::TGDefIDs::FreezeTimer);
   TG_WAIT(lib::TGDefIDs::SqlMemTimer);
   TG_WAIT(lib::TGDefIDs::ServerTracerTimer);
@@ -931,6 +936,8 @@ int ObServer::init_config()
       LOG_ERROR("local address isn't valid", K(self_addr_), K(ret));
     } else if (OB_FAIL(TG_START(lib::TGDefIDs::ServerGTimer))) {
       LOG_ERROR("init timer fail", K(ret));
+    } else if (OB_FAIL(TG_START(lib::TGDefIDs::MemDumpTimer))) {
+      LOG_ERROR("init memory dump timer fail", KR(ret));
     } else if (OB_FAIL(TG_START(lib::TGDefIDs::FreezeTimer))) {
       LOG_ERROR("init freeze timer fail", K(ret));
     } else if (OB_FAIL(TG_START(lib::TGDefIDs::SqlMemTimer))) {
@@ -1295,7 +1302,7 @@ int ObServer::init_multi_tenant()
       LOG_ERROR("Fail to add server tenant to tenant manager, ", K(ret));
     } else if (OB_FAIL(omti.set_tenant_mem_limit(OB_SERVER_TENANT_ID, 0, INT64_MAX))) {
       LOG_ERROR("Fail to set tenant mem limit, ", K(ret));
-    } else if (OB_FAIL(omti.register_timer_task(lib::TGDefIDs::ServerGTimer))) {
+    } else if (OB_FAIL(omti.register_timer_task(lib::TGDefIDs::MemDumpTimer))) {
       LOG_ERROR("Fail to register timer task", K(ret));
     }
   }

src/share/ob_task_define.cpp

@@ -19,6 +19,13 @@ using namespace oceanbase::lib;
 using namespace oceanbase::common;
 
 namespace oceanbase {
+namespace common {
+void allow_next_syslog(int64_t count)
+{
+  share::ObTaskController::get().allow_next_syslog(count);
+}
+} // common
+
 namespace share {
 
 class ObLogRateLimiter : public lib::ObSimpleRateLimiter {

src/share/ob_thread_define.h

@@ -144,4 +144,5 @@ TG_DEF(DDLRetryGhostIndex, DDLRetryGhostIndex, "", TG_STATIC, TIMER)
 TG_DEF(StoreFileAutoExtend, StoreFileAutoExtend, "", TG_STATIC, TIMER)
 TG_DEF(TTLScheduler, TTLScheduler, "", TG_STATIC, TIMER)
 TG_DEF(CTASCleanUpTimer, CTASCleanUpTimer, "", TG_STATIC, TIMER)
+TG_DEF(MemDumpTimer, MemDumpTimer, "", TG_STATIC, TIMER)
 #endif