From 778c43f157f95c476c7f4398c2bf5def2d96c52f Mon Sep 17 00:00:00 2001 From: obdev Date: Tue, 23 Apr 2024 07:22:53 +0000 Subject: [PATCH] [CP] fix bug of mem chunk size --- .../ob_table_load_assigned_memory_manager.cpp | 19 ++++++++++--------- .../ob_table_load_assigned_memory_manager.h | 2 +- .../direct_load/ob_direct_load_mem_loader.cpp | 2 -- ...direct_load_multiple_heap_table_sorter.cpp | 2 -- 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/src/observer/table_load/ob_table_load_assigned_memory_manager.cpp b/src/observer/table_load/ob_table_load_assigned_memory_manager.cpp index 2cda7fa87..83285e246 100644 --- a/src/observer/table_load/ob_table_load_assigned_memory_manager.cpp +++ b/src/observer/table_load/ob_table_load_assigned_memory_manager.cpp @@ -12,10 +12,11 @@ #define USING_LOG_PREFIX SERVER +#include "share/rc/ob_tenant_base.h" #include "observer/table_load/ob_table_load_assigned_memory_manager.h" #include "observer/omt/ob_multi_tenant.h" -#include "share/rc/ob_tenant_base.h" #include "observer/omt/ob_tenant.h" +#include "storage/direct_load/ob_direct_load_mem_define.h" namespace oceanbase { @@ -36,7 +37,7 @@ using namespace omt; ObTableLoadAssignedMemoryManager::ObTableLoadAssignedMemoryManager() : avail_sort_memory_(0), avail_memory_(0), - sort_task_count_(0), + chunk_count_(0), is_inited_(false) { } @@ -66,9 +67,9 @@ int ObTableLoadAssignedMemoryManager::assign_memory(bool is_sort, int64_t assign LOG_WARN("ObTableLoadAssignedMemoryManager not init", KR(ret), KP(this)); } else { ObMutexGuard guard(mutex_); - sort_task_count_ += (is_sort ? 1 : 0); + chunk_count_ += (is_sort ? assign_memory / ObDirectLoadExternalMultiPartitionRowChunk::MIN_MEMORY_LIMIT : 0); avail_sort_memory_ -= (is_sort ? 0 : assign_memory); - LOG_INFO("ObTableLoadAssignedMemoryManager::assign_memory", K(MTL_ID()), K(is_sort), K(assign_memory), K(sort_task_count_), K(avail_sort_memory_)); + LOG_INFO("ObTableLoadAssignedMemoryManager::assign_memory", K(MTL_ID()), K(is_sort), K(chunk_count_), K(assign_memory), K(avail_sort_memory_)); } return ret; @@ -82,9 +83,9 @@ int ObTableLoadAssignedMemoryManager::recycle_memory(bool is_sort, int64_t assig LOG_WARN("ObTableLoadAssignedMemoryManager not init", KR(ret), KP(this)); } else { ObMutexGuard guard(mutex_); - sort_task_count_ -= (is_sort ? 1 : 0); + chunk_count_ -= (is_sort ? assign_memory / ObDirectLoadExternalMultiPartitionRowChunk::MIN_MEMORY_LIMIT : 0); avail_sort_memory_ += (is_sort ? 0 : assign_memory); - LOG_INFO("ObTableLoadAssignedMemoryManager::recycle_memory", K(MTL_ID()), K(is_sort), K(assign_memory), K(sort_task_count_), K(avail_sort_memory_)); + LOG_INFO("ObTableLoadAssignedMemoryManager::recycle_memory", K(MTL_ID()), K(is_sort), K(chunk_count_), K(assign_memory), K(avail_sort_memory_)); } return ret; @@ -123,11 +124,11 @@ int ObTableLoadAssignedMemoryManager::get_sort_memory(int64_t &sort_memory) LOG_WARN("ObTableLoadAssignedMemoryManager not init", KR(ret), KP(this)); } else { ObMutexGuard guard(mutex_); - if (OB_UNLIKELY(sort_task_count_ == 0)) { + if (OB_UNLIKELY(chunk_count_ == 0)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected sort_task_count_ equal to zero", KR(ret)); + LOG_WARN("unexpected chunk_count_ equal to zero", KR(ret)); } else { - sort_memory = avail_sort_memory_ / sort_task_count_; + sort_memory = avail_sort_memory_ / chunk_count_; } } diff --git a/src/observer/table_load/ob_table_load_assigned_memory_manager.h b/src/observer/table_load/ob_table_load_assigned_memory_manager.h index 0d76f6eb2..a38991f11 100644 --- a/src/observer/table_load/ob_table_load_assigned_memory_manager.h +++ b/src/observer/table_load/ob_table_load_assigned_memory_manager.h @@ -39,7 +39,7 @@ public: private: int64_t avail_sort_memory_; int64_t avail_memory_; - int64_t sort_task_count_; + int64_t chunk_count_; mutable lib::ObMutex mutex_; bool is_inited_; }; diff --git a/src/storage/direct_load/ob_direct_load_mem_loader.cpp b/src/storage/direct_load/ob_direct_load_mem_loader.cpp index 4da2e7548..b88c4e7de 100644 --- a/src/storage/direct_load/ob_direct_load_mem_loader.cpp +++ b/src/storage/direct_load/ob_direct_load_mem_loader.cpp @@ -109,8 +109,6 @@ int ObDirectLoadMemLoader::work() sort_memory = mem_ctx_->table_data_desc_.mem_chunk_size_; } else if (OB_FAIL(ObTableLoadService::get_sort_memory(sort_memory))) { LOG_WARN("fail to get sort memory", KR(ret)); - } else { - sort_memory /= mem_ctx_->table_data_desc_.max_mem_chunk_count_; } if (OB_SUCC(ret)) { if (OB_FAIL(chunk->init(MTL_ID(), sort_memory))) { diff --git a/src/storage/direct_load/ob_direct_load_multiple_heap_table_sorter.cpp b/src/storage/direct_load/ob_direct_load_multiple_heap_table_sorter.cpp index 3ad5a16a3..888b83e17 100644 --- a/src/storage/direct_load/ob_direct_load_multiple_heap_table_sorter.cpp +++ b/src/storage/direct_load/ob_direct_load_multiple_heap_table_sorter.cpp @@ -207,8 +207,6 @@ int ObDirectLoadMultipleHeapTableSorter::work() sort_memory = mem_ctx_->table_data_desc_.heap_table_mem_chunk_size_; } else if (OB_FAIL(ObTableLoadService::get_sort_memory(sort_memory))) { LOG_WARN("fail to get sort memory", KR(ret)); - } else { - sort_memory /= mem_ctx_->table_data_desc_.session_count_; } if (OB_SUCC(ret)) { chunk = OB_NEW(ChunkType, ObMemAttr(MTL_ID(), "TLD_MemChunkVal"), sort_memory); -- GitLab