diff --git a/.gitignore b/.gitignore index a4b35dc9d484bc8f944b9fd82e9337882f4dccd2..b6c4247481e87ffcca4fd501a73f4d8eecd6e07a 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ cov_html/ # temp shards/all_in_one_with_mysql/metadata/ shards/mishards/.env +*.swp diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b9a7a16b719135f18712388f8982e21d2a1e2f8..b9ca1bc7fc51af57ca21ab8de6f7b48044b12f15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ Please mark all change in change log and use the issue from GitHub +# Milvus 0.9.0 (TBD) + +## Bug + +## Feature + +## Improvement +- \#221 Refactor LOG macro + +## Task + # Milvus 0.8.0 (TBD) ## Bug diff --git a/core/conf/log_config.template b/core/conf/log_config.template index a3fb15af6df84c6266993c246a42a2b4d0ac78f4..32864fa2f4437604496d7088838c152f2ba9ea9b 100644 --- a/core/conf/log_config.template +++ b/core/conf/log_config.template @@ -1,5 +1,5 @@ * GLOBAL: - FORMAT = "%datetime | %level | %logger | %msg" + FORMAT = "[%datetime][%level]%msg" FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%y-%M-%d-%H:%m}-global.log" ENABLED = true TO_FILE = true @@ -24,4 +24,4 @@ FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%y-%M-%d-%H:%m}-error.log" * FATAL: ENABLED = true - FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%y-%M-%d-%H:%m}-fatal.log" \ No newline at end of file + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%y-%M-%d-%H:%m}-fatal.log" diff --git a/core/src/cache/Cache.inl b/core/src/cache/Cache.inl index 607976e4117e16ee4df65a65ec163df78d0ed17f..561cdb3a42c6d49929b7295cda3d5d36e271aa4b 100644 --- a/core/src/cache/Cache.inl +++ b/core/src/cache/Cache.inl @@ -76,7 +76,7 @@ bool Cache::reserve(const int64_t item_size) { std::lock_guard lock(mutex_); if (item_size > capacity_) { - SERVER_LOG_ERROR << header_ << " item size " << (item_size >> 20) << "MB too big to insert into cache capacity" + LOG_SERVER_ERROR_ << header_ << " item size " << (item_size >> 20) << "MB too big to insert into cache capacity" << (capacity_ >> 20) << "MB"; return false; } @@ -92,7 +92,7 @@ Cache::clear() { std::lock_guard lock(mutex_); lru_.clear(); usage_ = 0; - SERVER_LOG_DEBUG << header_ << " Clear cache !"; + LOG_SERVER_DEBUG_ << header_ << " Clear cache !"; } @@ -102,9 +102,9 @@ Cache::print() { std::lock_guard lock(mutex_); size_t cache_count = lru_.size(); // for (auto it = lru_.begin(); it != lru_.end(); ++it) { - // SERVER_LOG_DEBUG << it->first; + // LOG_SERVER_DEBUG_ << it->first; // } - SERVER_LOG_DEBUG << header_ << " [item count]: " << cache_count << ", [usage] " << (usage_ >> 20) + LOG_SERVER_DEBUG_ << header_ << " [item count]: " << cache_count << ", [usage] " << (usage_ >> 20) << "MB, [capacity] " << (capacity_ >> 20) << "MB"; } @@ -128,15 +128,15 @@ Cache::insert_internal(const std::string& key, const ItemObj& item) { // if usage exceed capacity, free some items if (usage_ > capacity_) { - SERVER_LOG_DEBUG << header_ << " Current usage " << (usage_ >> 20) << "MB is too high for capacity " + LOG_SERVER_DEBUG_ << header_ << " Current usage " << (usage_ >> 20) << "MB is too high for capacity " << (capacity_ >> 20) << "MB, start free memory"; free_memory_internal(capacity_); } // insert new item lru_.put(key, item); - SERVER_LOG_DEBUG << header_ << " Insert " << key << " size: " << (item_size >> 20) << "MB into cache"; - SERVER_LOG_DEBUG << header_ << " Count: " << lru_.size() << ", Usage: " << (usage_ >> 20) << "MB, Capacity: " + LOG_SERVER_DEBUG_ << header_ << " Insert " << key << " size: " << (item_size >> 20) << "MB into cache"; + LOG_SERVER_DEBUG_ << header_ << " Count: " << lru_.size() << ", Usage: " << (usage_ >> 20) << "MB, Capacity: " << (capacity_ >> 20) << "MB"; } @@ -153,8 +153,8 @@ Cache::erase_internal(const std::string& key) { lru_.erase(key); usage_ -= item_size; - SERVER_LOG_DEBUG << header_ << " Erase " << key << " size: " << (item_size >> 20) << "MB from cache"; - SERVER_LOG_DEBUG << header_ << " Count: " << lru_.size() << ", Usage: " << (usage_ >> 20) << "MB, Capacity: " + LOG_SERVER_DEBUG_ << header_ << " Erase " << key << " size: " << (item_size >> 20) << "MB from cache"; + LOG_SERVER_DEBUG_ << header_ << " Count: " << lru_.size() << ", Usage: " << (usage_ >> 20) << "MB, Capacity: " << (capacity_ >> 20) << "MB"; } @@ -180,7 +180,7 @@ Cache::free_memory_internal(const int64_t target_size) { ++it; } - SERVER_LOG_DEBUG << header_ << " To be released memory size: " << (released_size >> 20) << "MB"; + LOG_SERVER_DEBUG_ << header_ << " To be released memory size: " << (released_size >> 20) << "MB"; for (auto& key : key_array) { erase_internal(key); diff --git a/core/src/cache/CacheMgr.inl b/core/src/cache/CacheMgr.inl index b45b403595e41b23f37fa43fe86f97b4a5abf3b2..887e2a47f70922bfcdd51fb058e0d8e2eecd1d07 100644 --- a/core/src/cache/CacheMgr.inl +++ b/core/src/cache/CacheMgr.inl @@ -24,7 +24,7 @@ template uint64_t CacheMgr::ItemCount() const { if (cache_ == nullptr) { - SERVER_LOG_ERROR << "Cache doesn't exist"; + LOG_SERVER_ERROR_ << "Cache doesn't exist"; return 0; } return (uint64_t)(cache_->size()); @@ -34,7 +34,7 @@ template bool CacheMgr::ItemExists(const std::string& key) { if (cache_ == nullptr) { - SERVER_LOG_ERROR << "Cache doesn't exist"; + LOG_SERVER_ERROR_ << "Cache doesn't exist"; return false; } return cache_->exists(key); @@ -44,7 +44,7 @@ template ItemObj CacheMgr::GetItem(const std::string& key) { if (cache_ == nullptr) { - SERVER_LOG_ERROR << "Cache doesn't exist"; + LOG_SERVER_ERROR_ << "Cache doesn't exist"; return nullptr; } server::Metrics::GetInstance().CacheAccessTotalIncrement(); @@ -55,7 +55,7 @@ template void CacheMgr::InsertItem(const std::string& key, const ItemObj& data) { if (cache_ == nullptr) { - SERVER_LOG_ERROR << "Cache doesn't exist"; + LOG_SERVER_ERROR_ << "Cache doesn't exist"; return; } cache_->insert(key, data); @@ -66,7 +66,7 @@ template void CacheMgr::EraseItem(const std::string& key) { if (cache_ == nullptr) { - SERVER_LOG_ERROR << "Cache doesn't exist"; + LOG_SERVER_ERROR_ << "Cache doesn't exist"; return; } cache_->erase(key); @@ -77,7 +77,7 @@ template bool CacheMgr::Reserve(const int64_t size) { if (cache_ == nullptr) { - SERVER_LOG_ERROR << "Cache doesn't exist"; + LOG_SERVER_ERROR_ << "Cache doesn't exist"; return false; } return cache_->reserve(size); @@ -87,7 +87,7 @@ template void CacheMgr::PrintInfo() { if (cache_ == nullptr) { - SERVER_LOG_ERROR << "Cache doesn't exist"; + LOG_SERVER_ERROR_ << "Cache doesn't exist"; return; } cache_->print(); @@ -97,7 +97,7 @@ template void CacheMgr::ClearCache() { if (cache_ == nullptr) { - SERVER_LOG_ERROR << "Cache doesn't exist"; + LOG_SERVER_ERROR_ << "Cache doesn't exist"; return; } cache_->clear(); @@ -107,7 +107,7 @@ template int64_t CacheMgr::CacheUsage() const { if (cache_ == nullptr) { - SERVER_LOG_ERROR << "Cache doesn't exist"; + LOG_SERVER_ERROR_ << "Cache doesn't exist"; return 0; } return cache_->usage(); @@ -117,7 +117,7 @@ template int64_t CacheMgr::CacheCapacity() const { if (cache_ == nullptr) { - SERVER_LOG_ERROR << "Cache doesn't exist"; + LOG_SERVER_ERROR_ << "Cache doesn't exist"; return 0; } return cache_->capacity(); @@ -127,7 +127,7 @@ template void CacheMgr::SetCapacity(int64_t capacity) { if (cache_ == nullptr) { - SERVER_LOG_ERROR << "Cache doesn't exist"; + LOG_SERVER_ERROR_ << "Cache doesn't exist"; return; } cache_->set_capacity(capacity); diff --git a/core/src/codecs/default/DefaultDeletedDocsFormat.cpp b/core/src/codecs/default/DefaultDeletedDocsFormat.cpp index c499e40fe0426a84db2dc95e299f1eb4eddf6e27..8651a05be79b52eb7062ae751526ee2c919b5ae4 100644 --- a/core/src/codecs/default/DefaultDeletedDocsFormat.cpp +++ b/core/src/codecs/default/DefaultDeletedDocsFormat.cpp @@ -44,14 +44,14 @@ DefaultDeletedDocsFormat::read(const storage::FSHandlerPtr& fs_ptr, segment::Del int del_fd = open(del_file_path.c_str(), O_RDONLY, 00664); if (del_fd == -1) { std::string err_msg = "Failed to open file: " + del_file_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_CANNOT_CREATE_FILE, err_msg); } size_t num_bytes; if (::read(del_fd, &num_bytes, sizeof(size_t)) == -1) { std::string err_msg = "Failed to read from file: " + del_file_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_WRITE_ERROR, err_msg); } @@ -61,7 +61,7 @@ DefaultDeletedDocsFormat::read(const storage::FSHandlerPtr& fs_ptr, segment::Del if (::read(del_fd, deleted_docs_list.data(), num_bytes) == -1) { std::string err_msg = "Failed to read from file: " + del_file_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_WRITE_ERROR, err_msg); } @@ -69,7 +69,7 @@ DefaultDeletedDocsFormat::read(const storage::FSHandlerPtr& fs_ptr, segment::Del if (::close(del_fd) == -1) { std::string err_msg = "Failed to close file: " + del_file_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_WRITE_ERROR, err_msg); } } @@ -92,7 +92,7 @@ DefaultDeletedDocsFormat::write(const storage::FSHandlerPtr& fs_ptr, const segme int del_fd = open(temp_path.c_str(), O_RDWR | O_CREAT, 00664); if (del_fd == -1) { std::string err_msg = "Failed to open file: " + temp_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_CANNOT_CREATE_FILE, err_msg); } @@ -100,7 +100,7 @@ DefaultDeletedDocsFormat::write(const storage::FSHandlerPtr& fs_ptr, const segme if (exists) { if (::read(del_fd, &old_num_bytes, sizeof(size_t)) == -1) { std::string err_msg = "Failed to read from file: " + temp_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_WRITE_ERROR, err_msg); } } else { @@ -114,12 +114,12 @@ DefaultDeletedDocsFormat::write(const storage::FSHandlerPtr& fs_ptr, const segme int off = lseek(del_fd, 0, SEEK_SET); if (off == -1) { std::string err_msg = "Failed to seek file: " + temp_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_WRITE_ERROR, err_msg); } if (::write(del_fd, &new_num_bytes, sizeof(size_t)) == -1) { std::string err_msg = "Failed to write to file" + temp_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_WRITE_ERROR, err_msg); } @@ -127,18 +127,18 @@ DefaultDeletedDocsFormat::write(const storage::FSHandlerPtr& fs_ptr, const segme off = lseek(del_fd, 0, SEEK_END); if (off == -1) { std::string err_msg = "Failed to seek file: " + temp_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_WRITE_ERROR, err_msg); } if (::write(del_fd, deleted_docs_list.data(), sizeof(segment::offset_t) * deleted_docs->GetSize()) == -1) { std::string err_msg = "Failed to write to file" + temp_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_WRITE_ERROR, err_msg); } if (::close(del_fd) == -1) { std::string err_msg = "Failed to close file: " + temp_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_WRITE_ERROR, err_msg); } @@ -156,14 +156,14 @@ DefaultDeletedDocsFormat::readSize(const storage::FSHandlerPtr& fs_ptr, size_t& int del_fd = open(del_file_path.c_str(), O_RDONLY, 00664); if (del_fd == -1) { std::string err_msg = "Failed to open file: " + del_file_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_CANNOT_CREATE_FILE, err_msg); } size_t num_bytes; if (::read(del_fd, &num_bytes, sizeof(size_t)) == -1) { std::string err_msg = "Failed to read from file: " + del_file_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_WRITE_ERROR, err_msg); } @@ -171,7 +171,7 @@ DefaultDeletedDocsFormat::readSize(const storage::FSHandlerPtr& fs_ptr, size_t& if (::close(del_fd) == -1) { std::string err_msg = "Failed to close file: " + del_file_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_WRITE_ERROR, err_msg); } } diff --git a/core/src/codecs/default/DefaultIdBloomFilterFormat.cpp b/core/src/codecs/default/DefaultIdBloomFilterFormat.cpp index ed7c40908b43202032ee913067216d7c4c9be040..3222eac631cb6915c2f02daea205f8984bd638b8 100644 --- a/core/src/codecs/default/DefaultIdBloomFilterFormat.cpp +++ b/core/src/codecs/default/DefaultIdBloomFilterFormat.cpp @@ -40,7 +40,7 @@ DefaultIdBloomFilterFormat::read(const storage::FSHandlerPtr& fs_ptr, segment::I if (bloom_filter == nullptr) { std::string err_msg = "Failed to read bloom filter from file: " + bloom_filter_file_path + ". " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_UNEXPECTED_ERROR, err_msg); } id_bloom_filter_ptr = std::make_shared(bloom_filter); @@ -56,7 +56,7 @@ DefaultIdBloomFilterFormat::write(const storage::FSHandlerPtr& fs_ptr, if (scaling_bloom_flush(id_bloom_filter_ptr->GetBloomFilter()) == -1) { std::string err_msg = "Failed to write bloom filter to file: " + bloom_filter_file_path + ". " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_UNEXPECTED_ERROR, err_msg); } } @@ -71,7 +71,7 @@ DefaultIdBloomFilterFormat::create(const storage::FSHandlerPtr& fs_ptr, if (bloom_filter == nullptr) { std::string err_msg = "Failed to read bloom filter from file: " + bloom_filter_file_path + ". " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_UNEXPECTED_ERROR, err_msg); } id_bloom_filter_ptr = std::make_shared(bloom_filter); diff --git a/core/src/codecs/default/DefaultVectorIndexFormat.cpp b/core/src/codecs/default/DefaultVectorIndexFormat.cpp index 71df5ea73a16df89a02f51f2dd724ec729850ba9..84eed66316d237bb70b48e4eeba5acc3d1104620 100644 --- a/core/src/codecs/default/DefaultVectorIndexFormat.cpp +++ b/core/src/codecs/default/DefaultVectorIndexFormat.cpp @@ -37,13 +37,13 @@ DefaultVectorIndexFormat::read_internal(const storage::FSHandlerPtr& fs_ptr, con recorder.RecordSection("Start"); if (!fs_ptr->reader_ptr_->open(path)) { - ENGINE_LOG_ERROR << "Fail to open vector index: " << path; + LOG_ENGINE_ERROR_ << "Fail to open vector index: " << path; return nullptr; } int64_t length = fs_ptr->reader_ptr_->length(); if (length <= 0) { - ENGINE_LOG_ERROR << "Invalid vector index length: " << path; + LOG_ENGINE_ERROR_ << "Invalid vector index length: " << path; return nullptr; } @@ -55,7 +55,7 @@ DefaultVectorIndexFormat::read_internal(const storage::FSHandlerPtr& fs_ptr, con rp += sizeof(current_type); fs_ptr->reader_ptr_->seekg(rp); - ENGINE_LOG_DEBUG << "Start to read_index(" << path << ") length: " << length << " bytes"; + LOG_ENGINE_DEBUG_ << "Start to read_index(" << path << ") length: " << length << " bytes"; while (rp < length) { size_t meta_length; fs_ptr->reader_ptr_->read(&meta_length, sizeof(meta_length)); @@ -85,7 +85,7 @@ DefaultVectorIndexFormat::read_internal(const storage::FSHandlerPtr& fs_ptr, con double span = recorder.RecordSection("End"); double rate = length * 1000000.0 / span / 1024 / 1024; - ENGINE_LOG_DEBUG << "read_index(" << path << ") rate " << rate << "MB/s"; + LOG_ENGINE_DEBUG_ << "read_index(" << path << ") rate " << rate << "MB/s"; knowhere::VecIndexFactory& vec_index_factory = knowhere::VecIndexFactory::GetInstance(); auto index = @@ -94,7 +94,7 @@ DefaultVectorIndexFormat::read_internal(const storage::FSHandlerPtr& fs_ptr, con index->Load(load_data_list); index->SetIndexSize(length); } else { - ENGINE_LOG_ERROR << "Fail to create vector index: " << path; + LOG_ENGINE_ERROR_ << "Fail to create vector index: " << path; } return index; @@ -108,7 +108,7 @@ DefaultVectorIndexFormat::read(const storage::FSHandlerPtr& fs_ptr, const std::s std::string dir_path = fs_ptr->operation_ptr_->GetDirectory(); if (!boost::filesystem::is_directory(dir_path)) { std::string err_msg = "Directory: " + dir_path + "does not exist"; - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_INVALID_ARGUMENT, err_msg); } @@ -132,7 +132,7 @@ DefaultVectorIndexFormat::write(const storage::FSHandlerPtr& fs_ptr, const std:: recorder.RecordSection("Start"); if (!fs_ptr->writer_ptr_->open(location)) { - ENGINE_LOG_ERROR << "Fail to open vector index: " << location; + LOG_ENGINE_ERROR_ << "Fail to open vector index: " << location; return; } @@ -153,7 +153,7 @@ DefaultVectorIndexFormat::write(const storage::FSHandlerPtr& fs_ptr, const std:: double span = recorder.RecordSection("End"); double rate = fs_ptr->writer_ptr_->length() * 1000000.0 / span / 1024 / 1024; - ENGINE_LOG_DEBUG << "write_index(" << location << ") rate " << rate << "MB/s"; + LOG_ENGINE_DEBUG_ << "write_index(" << location << ") rate " << rate << "MB/s"; } } // namespace codec diff --git a/core/src/codecs/default/DefaultVectorsFormat.cpp b/core/src/codecs/default/DefaultVectorsFormat.cpp index 412049fc9a064f7f355d720d63b5fef6782f5045..6b00d2612581cfcafeda8fb3239e64a1c8c25c48 100644 --- a/core/src/codecs/default/DefaultVectorsFormat.cpp +++ b/core/src/codecs/default/DefaultVectorsFormat.cpp @@ -36,14 +36,14 @@ DefaultVectorsFormat::read_vectors_internal(const std::string& file_path, off_t int rv_fd = open(file_path.c_str(), O_RDONLY, 00664); if (rv_fd == -1) { std::string err_msg = "Failed to open file: " + file_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_CANNOT_CREATE_FILE, err_msg); } size_t num_bytes; if (::read(rv_fd, &num_bytes, sizeof(size_t)) == -1) { std::string err_msg = "Failed to read from file: " + file_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_WRITE_ERROR, err_msg); } @@ -53,20 +53,20 @@ DefaultVectorsFormat::read_vectors_internal(const std::string& file_path, off_t int off = lseek(rv_fd, offset, SEEK_SET); if (off == -1) { std::string err_msg = "Failed to seek file: " + file_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_WRITE_ERROR, err_msg); } raw_vectors.resize(num / sizeof(uint8_t)); if (::read(rv_fd, raw_vectors.data(), num) == -1) { std::string err_msg = "Failed to read from file: " + file_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_WRITE_ERROR, err_msg); } if (::close(rv_fd) == -1) { std::string err_msg = "Failed to close file: " + file_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_WRITE_ERROR, err_msg); } } @@ -76,27 +76,27 @@ DefaultVectorsFormat::read_uids_internal(const std::string& file_path, std::vect int uid_fd = open(file_path.c_str(), O_RDONLY, 00664); if (uid_fd == -1) { std::string err_msg = "Failed to open file: " + file_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_CANNOT_CREATE_FILE, err_msg); } size_t num_bytes; if (::read(uid_fd, &num_bytes, sizeof(size_t)) == -1) { std::string err_msg = "Failed to read from file: " + file_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_WRITE_ERROR, err_msg); } uids.resize(num_bytes / sizeof(segment::doc_id_t)); if (::read(uid_fd, uids.data(), num_bytes) == -1) { std::string err_msg = "Failed to read from file: " + file_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_WRITE_ERROR, err_msg); } if (::close(uid_fd) == -1) { std::string err_msg = "Failed to close file: " + file_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_WRITE_ERROR, err_msg); } } @@ -108,7 +108,7 @@ DefaultVectorsFormat::read(const storage::FSHandlerPtr& fs_ptr, segment::Vectors std::string dir_path = fs_ptr->operation_ptr_->GetDirectory(); if (!boost::filesystem::is_directory(dir_path)) { std::string err_msg = "Directory: " + dir_path + "does not exist"; - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_INVALID_ARGUMENT, err_msg); } @@ -147,24 +147,24 @@ DefaultVectorsFormat::write(const storage::FSHandlerPtr& fs_ptr, const segment:: int rv_fd = open(rv_file_path.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 00664); if (rv_fd == -1) { std::string err_msg = "Failed to open file: " + rv_file_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_CANNOT_CREATE_FILE, err_msg); } size_t rv_num_bytes = vectors->GetData().size() * sizeof(uint8_t); if (::write(rv_fd, &rv_num_bytes, sizeof(size_t)) == -1) { std::string err_msg = "Failed to write to file: " + rv_file_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_WRITE_ERROR, err_msg); } if (::write(rv_fd, vectors->GetData().data(), rv_num_bytes) == -1) { std::string err_msg = "Failed to write to file: " + rv_file_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_WRITE_ERROR, err_msg); } if (::close(rv_fd) == -1) { std::string err_msg = "Failed to close file: " + rv_file_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_WRITE_ERROR, err_msg); } @@ -173,23 +173,23 @@ DefaultVectorsFormat::write(const storage::FSHandlerPtr& fs_ptr, const segment:: int uid_fd = open(uid_file_path.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 00664); if (uid_fd == -1) { std::string err_msg = "Failed to open file: " + uid_file_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_CANNOT_CREATE_FILE, err_msg); } size_t uid_num_bytes = vectors->GetUids().size() * sizeof(segment::doc_id_t); if (::write(uid_fd, &uid_num_bytes, sizeof(size_t)) == -1) { std::string err_msg = "Failed to write to file" + rv_file_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_WRITE_ERROR, err_msg); } if (::write(uid_fd, vectors->GetUids().data(), uid_num_bytes) == -1) { std::string err_msg = "Failed to write to file" + uid_file_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_WRITE_ERROR, err_msg); } if (::close(uid_fd) == -1) { std::string err_msg = "Failed to close file: " + uid_file_path + ", error: " + std::strerror(errno); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_WRITE_ERROR, err_msg); } @@ -203,7 +203,7 @@ DefaultVectorsFormat::read_uids(const storage::FSHandlerPtr& fs_ptr, std::vector std::string dir_path = fs_ptr->operation_ptr_->GetDirectory(); if (!boost::filesystem::is_directory(dir_path)) { std::string err_msg = "Directory: " + dir_path + "does not exist"; - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_INVALID_ARGUMENT, err_msg); } @@ -228,7 +228,7 @@ DefaultVectorsFormat::read_vectors(const storage::FSHandlerPtr& fs_ptr, off_t of std::string dir_path = fs_ptr->operation_ptr_->GetDirectory(); if (!boost::filesystem::is_directory(dir_path)) { std::string err_msg = "Directory: " + dir_path + "does not exist"; - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_INVALID_ARGUMENT, err_msg); } diff --git a/core/src/config/Config.cpp b/core/src/config/Config.cpp index 726372aa7a29da3b50315717cdbe10610db0474b..964b55d017fc2238d4c54ad62768d73763abec21 100644 --- a/core/src/config/Config.cpp +++ b/core/src/config/Config.cpp @@ -668,7 +668,7 @@ Config::CheckConfigVersion(const std::string& value) { if (exist_error) { std::string msg = "Invalid config version: " + value + ". Expected config version: " + milvus_config_version_map.at(MILVUS_VERSION); - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_ARGUMENT, msg); } } diff --git a/core/src/config/ConfigNode.cpp b/core/src/config/ConfigNode.cpp index 6941b5bb02c4f308d52572adb6a9d1e2419e4f20..7125334cbf8e9cc259a07154a5ee85fd9b92c7f6 100644 --- a/core/src/config/ConfigNode.cpp +++ b/core/src/config/ConfigNode.cpp @@ -187,18 +187,18 @@ ConfigNode::ClearSequences() { void ConfigNode::PrintAll(const std::string& prefix) const { for (auto& elem : config_) { - SERVER_LOG_INFO << prefix << elem.first + ": " << elem.second; + LOG_SERVER_INFO_ << prefix << elem.first + ": " << elem.second; } for (auto& elem : sequences_) { - SERVER_LOG_INFO << prefix << elem.first << ": "; + LOG_SERVER_INFO_ << prefix << elem.first << ": "; for (auto& str : elem.second) { - SERVER_LOG_INFO << prefix << " - " << str; + LOG_SERVER_INFO_ << prefix << " - " << str; } } for (auto& elem : children_) { - SERVER_LOG_INFO << prefix << elem.first << ": "; + LOG_SERVER_INFO_ << prefix << elem.first << ": "; elem.second.PrintAll(prefix + " "); } } diff --git a/core/src/config/YamlConfigMgr.cpp b/core/src/config/YamlConfigMgr.cpp index 9d804069645e8659b2d71cbe51141c5538df6d6f..4c4eee639bc335b8e8c7ffd755c23dcd5c4ace1e 100644 --- a/core/src/config/YamlConfigMgr.cpp +++ b/core/src/config/YamlConfigMgr.cpp @@ -30,7 +30,7 @@ YamlConfigMgr::LoadConfigFile(const std::string& filename) { void YamlConfigMgr::Print() const { - SERVER_LOG_INFO << "System config content:"; + LOG_SERVER_INFO_ << "System config content:"; config_.PrintAll(); } diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 5cc90d2db78fbd44e805c15793fa70f9937ae23b..9fe5785aa2b98324619c9676ad5920f6290e39ba 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -96,7 +96,7 @@ DBImpl::Start() { return Status::OK(); } - // ENGINE_LOG_TRACE << "DB service start"; + // LOG_ENGINE_TRACE_ << "DB service start"; initialized_.store(true, std::memory_order_release); // wal @@ -184,7 +184,7 @@ DBImpl::Stop() { swn_metric_.Notify(); bg_metric_thread_.join(); - // ENGINE_LOG_TRACE << "DB service stop"; + // LOG_ENGINE_TRACE_ << "DB service stop"; return Status::OK(); } @@ -321,7 +321,7 @@ DBImpl::GetCollectionInfo(const std::string& collection_id, CollectionInfo& coll status = meta_ptr_->FilesByType(name_tag.first, file_types, collection_files); if (!status.ok()) { std::string err_msg = "Failed to get collection info: " + status.ToString(); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; return Status(DB_ERROR, err_msg); } @@ -375,8 +375,8 @@ DBImpl::PreloadCollection(const std::string& collection_id) { int64_t available_size = cache_total - cache_usage; // step 3: load file one by one - ENGINE_LOG_DEBUG << "Begin pre-load collection:" + collection_id + ", totally " << files_array.size() - << " files need to be pre-loaded"; + LOG_ENGINE_DEBUG_ << "Begin pre-load collection:" + collection_id + ", totally " << files_array.size() + << " files need to be pre-loaded"; TimeRecorderAuto rc("Pre-load collection:" + collection_id); for (auto& file : files_array) { EngineType engine_type; @@ -394,7 +394,7 @@ DBImpl::PreloadCollection(const std::string& collection_id) { EngineFactory::Build(file.dimension_, file.location_, engine_type, (MetricType)file.metric_type_, json); fiu_do_on("DBImpl.PreloadCollection.null_engine", engine = nullptr); if (engine == nullptr) { - ENGINE_LOG_ERROR << "Invalid engine type"; + LOG_ENGINE_ERROR_ << "Invalid engine type"; return Status(DB_ERROR, "Invalid engine type"); } @@ -408,12 +408,12 @@ DBImpl::PreloadCollection(const std::string& collection_id) { size += engine->Size(); if (size > available_size) { - ENGINE_LOG_DEBUG << "Pre-load cancelled since cache is almost full"; + LOG_ENGINE_DEBUG_ << "Pre-load cancelled since cache is almost full"; return Status(SERVER_CACHE_FULL, "Cache is full"); } } catch (std::exception& ex) { std::string msg = "Pre-load collection encounter exception: " + std::string(ex.what()); - ENGINE_LOG_ERROR << msg; + LOG_ENGINE_ERROR_ << msg; return Status(DB_ERROR, msg); } } @@ -460,7 +460,7 @@ DBImpl::DropPartition(const std::string& partition_name) { mem_mgr_->EraseMemVector(partition_name); // not allow insert auto status = meta_ptr_->DropPartition(partition_name); // soft delete collection if (!status.ok()) { - ENGINE_LOG_ERROR << status.message(); + LOG_ENGINE_ERROR_ << status.message(); return status; } @@ -482,7 +482,7 @@ DBImpl::DropPartitionByTag(const std::string& collection_id, const std::string& std::string partition_name; auto status = meta_ptr_->GetPartitionName(collection_id, partition_tag, partition_name); if (!status.ok()) { - ENGINE_LOG_ERROR << status.message(); + LOG_ENGINE_ERROR_ << status.message(); return status; } @@ -500,7 +500,7 @@ DBImpl::ShowPartitions(const std::string& collection_id, std::vectorFlush(collection_id); if (lsn != 0) { swn_wal_.Notify(); @@ -615,11 +615,11 @@ DBImpl::Flush(const std::string& collection_id) { } } else { - ENGINE_LOG_DEBUG << "MemTable flush"; + LOG_ENGINE_DEBUG_ << "MemTable flush"; InternalFlush(collection_id); } - ENGINE_LOG_DEBUG << "End flush collection: " << collection_id; + LOG_ENGINE_DEBUG_ << "End flush collection: " << collection_id; return status; } @@ -630,22 +630,22 @@ DBImpl::Flush() { return SHUTDOWN_ERROR; } - ENGINE_LOG_DEBUG << "Begin flush all collections"; + LOG_ENGINE_DEBUG_ << "Begin flush all collections"; Status status; if (options_.wal_enable_) { - ENGINE_LOG_DEBUG << "WAL flush"; + LOG_ENGINE_DEBUG_ << "WAL flush"; auto lsn = wal_mgr_->Flush(); if (lsn != 0) { swn_wal_.Notify(); flush_req_swn_.Wait(); } } else { - ENGINE_LOG_DEBUG << "MemTable flush"; + LOG_ENGINE_DEBUG_ << "MemTable flush"; InternalFlush(); } - ENGINE_LOG_DEBUG << "End flush all collections"; + LOG_ENGINE_DEBUG_ << "End flush all collections"; return status; } @@ -661,26 +661,26 @@ DBImpl::Compact(const std::string& collection_id) { auto status = DescribeCollection(collection_schema); if (!status.ok()) { if (status.code() == DB_NOT_FOUND) { - ENGINE_LOG_ERROR << "Collection to compact does not exist: " << collection_id; + LOG_ENGINE_ERROR_ << "Collection to compact does not exist: " << collection_id; return Status(DB_NOT_FOUND, "Collection to compact does not exist"); } else { return status; } } else { if (!collection_schema.owner_collection_.empty()) { - ENGINE_LOG_ERROR << "Collection to compact does not exist: " << collection_id; + LOG_ENGINE_ERROR_ << "Collection to compact does not exist: " << collection_id; return Status(DB_NOT_FOUND, "Collection to compact does not exist"); } } - ENGINE_LOG_DEBUG << "Before compacting, wait for build index thread to finish..."; + LOG_ENGINE_DEBUG_ << "Before compacting, wait for build index thread to finish..."; // WaitBuildIndexFinish(); const std::lock_guard index_lock(build_index_mutex_); const std::lock_guard merge_lock(flush_merge_compact_mutex_); - ENGINE_LOG_DEBUG << "Compacting collection: " << collection_id; + LOG_ENGINE_DEBUG_ << "Compacting collection: " << collection_id; // Get files to compact from meta. std::vector file_types{meta::SegmentSchema::FILE_TYPE::RAW, meta::SegmentSchema::FILE_TYPE::TO_INDEX, @@ -689,11 +689,11 @@ DBImpl::Compact(const std::string& collection_id) { status = meta_ptr_->FilesByType(collection_id, file_types, files_to_compact); if (!status.ok()) { std::string err_msg = "Failed to get files to compact: " + status.message(); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; return Status(DB_ERROR, err_msg); } - ENGINE_LOG_DEBUG << "Found " << files_to_compact.size() << " segment to compact"; + LOG_ENGINE_DEBUG_ << "Found " << files_to_compact.size() << " segment to compact"; OngoingFileChecker::GetInstance().MarkOngoingFiles(files_to_compact); @@ -719,18 +719,18 @@ DBImpl::Compact(const std::string& collection_id) { compact_status = CompactFile(collection_id, file, files_to_update); if (!compact_status.ok()) { - ENGINE_LOG_ERROR << "Compact failed for segment " << file.segment_id_ << ": " - << compact_status.message(); + LOG_ENGINE_ERROR_ << "Compact failed for segment " << file.segment_id_ << ": " + << compact_status.message(); OngoingFileChecker::GetInstance().UnmarkOngoingFile(file); continue; // skip this file and try compact next one } } else { OngoingFileChecker::GetInstance().UnmarkOngoingFile(file); - ENGINE_LOG_DEBUG << "Segment " << file.segment_id_ << " has no deleted data. No need to compact"; + LOG_ENGINE_DEBUG_ << "Segment " << file.segment_id_ << " has no deleted data. No need to compact"; continue; // skip this file and try compact next one } - ENGINE_LOG_DEBUG << "Updating meta after compaction..."; + LOG_ENGINE_DEBUG_ << "Updating meta after compaction..."; status = meta_ptr_->UpdateCollectionFiles(files_to_update); OngoingFileChecker::GetInstance().UnmarkOngoingFile(file); if (!status.ok()) { @@ -742,7 +742,7 @@ DBImpl::Compact(const std::string& collection_id) { OngoingFileChecker::GetInstance().UnmarkOngoingFiles(files_to_compact); if (compact_status.ok()) { - ENGINE_LOG_DEBUG << "Finished compacting collection: " << collection_id; + LOG_ENGINE_DEBUG_ << "Finished compacting collection: " << collection_id; } return compact_status; @@ -751,7 +751,7 @@ DBImpl::Compact(const std::string& collection_id) { Status DBImpl::CompactFile(const std::string& collection_id, const meta::SegmentSchema& file, meta::SegmentsSchema& files_to_update) { - ENGINE_LOG_DEBUG << "Compacting segment " << file.segment_id_ << " for collection: " << collection_id; + LOG_ENGINE_DEBUG_ << "Compacting segment " << file.segment_id_ << " for collection: " << collection_id; // Create new collection file meta::SegmentSchema compacted_file; @@ -761,7 +761,7 @@ DBImpl::CompactFile(const std::string& collection_id, const meta::SegmentSchema& Status status = meta_ptr_->CreateCollectionFile(compacted_file); if (!status.ok()) { - ENGINE_LOG_ERROR << "Failed to create collection file: " << status.message(); + LOG_ENGINE_ERROR_ << "Failed to create collection file: " << status.message(); return status; } @@ -774,18 +774,18 @@ DBImpl::CompactFile(const std::string& collection_id, const meta::SegmentSchema& std::string segment_dir_to_merge; utils::GetParentPath(file.location_, segment_dir_to_merge); - ENGINE_LOG_DEBUG << "Compacting begin..."; + LOG_ENGINE_DEBUG_ << "Compacting begin..."; segment_writer_ptr->Merge(segment_dir_to_merge, compacted_file.file_id_); // Serialize - ENGINE_LOG_DEBUG << "Serializing compacted segment..."; + LOG_ENGINE_DEBUG_ << "Serializing compacted segment..."; status = segment_writer_ptr->Serialize(); if (!status.ok()) { - ENGINE_LOG_ERROR << "Failed to serialize compacted segment: " << status.message(); + LOG_ENGINE_ERROR_ << "Failed to serialize compacted segment: " << status.message(); compacted_file.file_type_ = meta::SegmentSchema::TO_DELETE; auto mark_status = meta_ptr_->UpdateCollectionFile(compacted_file); if (mark_status.ok()) { - ENGINE_LOG_DEBUG << "Mark file: " << compacted_file.file_id_ << " to to_delete"; + LOG_ENGINE_DEBUG_ << "Mark file: " << compacted_file.file_id_ << " to to_delete"; } return status; } @@ -804,7 +804,7 @@ DBImpl::CompactFile(const std::string& collection_id, const meta::SegmentSchema& compacted_file.row_count_ = segment_writer_ptr->VectorCount(); if (compacted_file.row_count_ == 0) { - ENGINE_LOG_DEBUG << "Compacted segment is empty. Mark it as TO_DELETE"; + LOG_ENGINE_DEBUG_ << "Compacted segment is empty. Mark it as TO_DELETE"; compacted_file.file_type_ = meta::SegmentSchema::TO_DELETE; } @@ -822,9 +822,9 @@ DBImpl::CompactFile(const std::string& collection_id, const meta::SegmentSchema& files_to_update.emplace_back(f); } - ENGINE_LOG_DEBUG << "Compacted segment " << compacted_file.segment_id_ << " from " - << std::to_string(file.file_size_) << " bytes to " << std::to_string(compacted_file.file_size_) - << " bytes"; + LOG_ENGINE_DEBUG_ << "Compacted segment " << compacted_file.segment_id_ << " from " + << std::to_string(file.file_size_) << " bytes to " << std::to_string(compacted_file.file_size_) + << " bytes"; if (options_.insert_cache_immediately_) { segment_writer_ptr->Cache(); @@ -842,7 +842,7 @@ DBImpl::GetVectorByID(const std::string& collection_id, const IDNumber& vector_i bool has_collection; auto status = HasCollection(collection_id, has_collection); if (!has_collection) { - ENGINE_LOG_ERROR << "Collection " << collection_id << " does not exist: "; + LOG_ENGINE_ERROR_ << "Collection " << collection_id << " does not exist: "; return Status(DB_NOT_FOUND, "Collection does not exist"); } if (!status.ok()) { @@ -857,7 +857,7 @@ DBImpl::GetVectorByID(const std::string& collection_id, const IDNumber& vector_i status = meta_ptr_->FilesByType(collection_id, file_types, files_to_query); if (!status.ok()) { std::string err_msg = "Failed to get files for GetVectorByID: " + status.message(); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; return status; } @@ -868,7 +868,7 @@ DBImpl::GetVectorByID(const std::string& collection_id, const IDNumber& vector_i status = meta_ptr_->FilesByType(schema.collection_id_, file_types, files); if (!status.ok()) { std::string err_msg = "Failed to get files for GetVectorByID: " + status.message(); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; return status; } files_to_query.insert(files_to_query.end(), std::make_move_iterator(files.begin()), @@ -876,7 +876,7 @@ DBImpl::GetVectorByID(const std::string& collection_id, const IDNumber& vector_i } if (files_to_query.empty()) { - ENGINE_LOG_DEBUG << "No files to get vector by id from"; + LOG_ENGINE_DEBUG_ << "No files to get vector by id from"; return Status::OK(); } @@ -901,7 +901,7 @@ DBImpl::GetVectorIDs(const std::string& collection_id, const std::string& segmen bool has_collection; auto status = HasCollection(collection_id, has_collection); if (!has_collection) { - ENGINE_LOG_ERROR << "Collection " << collection_id << " does not exist: "; + LOG_ENGINE_ERROR_ << "Collection " << collection_id << " does not exist: "; return Status(DB_NOT_FOUND, "Collection does not exist"); } if (!status.ok()) { @@ -965,7 +965,7 @@ DBImpl::GetVectorIDs(const std::string& collection_id, const std::string& segmen Status DBImpl::GetVectorByIdHelper(const std::string& collection_id, IDNumber vector_id, VectorsData& vector, const meta::SegmentsSchema& files) { - ENGINE_LOG_DEBUG << "Getting vector by id in " << files.size() << " files"; + LOG_ENGINE_DEBUG_ << "Getting vector by id in " << files.size() << " files"; for (auto& file : files) { // Load bloom filter @@ -1046,7 +1046,7 @@ DBImpl::CreateIndex(const std::string& collection_id, const CollectionIndex& ind CollectionIndex old_index; status = DescribeIndex(collection_id, old_index); if (!status.ok()) { - ENGINE_LOG_ERROR << "Failed to get collection index info for collection: " << collection_id; + LOG_ENGINE_ERROR_ << "Failed to get collection index info for collection: " << collection_id; return status; } @@ -1087,7 +1087,7 @@ DBImpl::DropIndex(const std::string& collection_id) { return SHUTDOWN_ERROR; } - ENGINE_LOG_DEBUG << "Drop index for collection: " << collection_id; + LOG_ENGINE_DEBUG_ << "Drop index for collection: " << collection_id; return DropCollectionIndexRecursively(collection_id); } @@ -1218,7 +1218,7 @@ DBImpl::QueryAsync(const std::shared_ptr& context, const meta:: if (files.size() > milvus::scheduler::TASK_TABLE_MAX_COUNT) { std::string msg = "Search files count exceed scheduler limit: " + std::to_string(milvus::scheduler::TASK_TABLE_MAX_COUNT); - ENGINE_LOG_ERROR << msg; + LOG_ENGINE_ERROR_ << msg; return Status(DB_ERROR, msg); } @@ -1227,7 +1227,7 @@ DBImpl::QueryAsync(const std::shared_ptr& context, const meta:: // step 1: construct search job auto status = OngoingFileChecker::GetInstance().MarkOngoingFiles(files); - ENGINE_LOG_DEBUG << LogOut("Engine query begin, index file count: %ld", files.size()); + LOG_ENGINE_DEBUG_ << LogOut("Engine query begin, index file count: %ld", files.size()); scheduler::SearchJobPtr job = std::make_shared(tracer.Context(), k, extra_params, vectors); for (auto& file : files) { scheduler::SegmentSchemaPtr file_ptr = std::make_shared(file); @@ -1259,7 +1259,7 @@ DBImpl::BackgroundIndexThread() { WaitMergeFileFinish(); WaitBuildIndexFinish(); - ENGINE_LOG_DEBUG << "DB background thread exit"; + LOG_ENGINE_DEBUG_ << "DB background thread exit"; break; } @@ -1272,22 +1272,22 @@ DBImpl::BackgroundIndexThread() { void DBImpl::WaitMergeFileFinish() { - // ENGINE_LOG_DEBUG << "Begin WaitMergeFileFinish"; + // LOG_ENGINE_DEBUG_ << "Begin WaitMergeFileFinish"; std::lock_guard lck(merge_result_mutex_); for (auto& iter : merge_thread_results_) { iter.wait(); } - // ENGINE_LOG_DEBUG << "End WaitMergeFileFinish"; + // LOG_ENGINE_DEBUG_ << "End WaitMergeFileFinish"; } void DBImpl::WaitBuildIndexFinish() { - // ENGINE_LOG_DEBUG << "Begin WaitBuildIndexFinish"; + // LOG_ENGINE_DEBUG_ << "Begin WaitBuildIndexFinish"; std::lock_guard lck(index_result_mutex_); for (auto& iter : index_thread_results_) { iter.wait(); } - // ENGINE_LOG_DEBUG << "End WaitBuildIndexFinish"; + // LOG_ENGINE_DEBUG_ << "End WaitBuildIndexFinish"; } void @@ -1322,7 +1322,7 @@ DBImpl::StartMetricTask() { void DBImpl::StartMergeTask() { - // ENGINE_LOG_DEBUG << "Begin StartMergeTask"; + // LOG_ENGINE_DEBUG_ << "Begin StartMergeTask"; // merge task has been finished? { std::lock_guard lck(merge_result_mutex_); @@ -1356,14 +1356,14 @@ DBImpl::StartMergeTask() { } } - // ENGINE_LOG_DEBUG << "End StartMergeTask"; + // LOG_ENGINE_DEBUG_ << "End StartMergeTask"; } Status DBImpl::MergeFiles(const std::string& collection_id, const meta::SegmentsSchema& files) { // const std::lock_guard lock(flush_merge_compact_mutex_); - ENGINE_LOG_DEBUG << "Merge files for collection: " << collection_id; + LOG_ENGINE_DEBUG_ << "Merge files for collection: " << collection_id; // step 1: create collection file meta::SegmentSchema collection_file; @@ -1372,7 +1372,7 @@ DBImpl::MergeFiles(const std::string& collection_id, const meta::SegmentsSchema& Status status = meta_ptr_->CreateCollectionFile(collection_file); if (!status.ok()) { - ENGINE_LOG_ERROR << "Failed to create collection: " << status.ToString(); + LOG_ENGINE_ERROR_ << "Failed to create collection: " << status.ToString(); return status; } @@ -1409,19 +1409,19 @@ DBImpl::MergeFiles(const std::string& collection_id, const meta::SegmentsSchema& fiu_do_on("DBImpl.MergeFiles.Serialize_ErrorStatus", status = Status(DB_ERROR, "")); } catch (std::exception& ex) { std::string msg = "Serialize merged index encounter exception: " + std::string(ex.what()); - ENGINE_LOG_ERROR << msg; + LOG_ENGINE_ERROR_ << msg; status = Status(DB_ERROR, msg); } if (!status.ok()) { - ENGINE_LOG_ERROR << "Failed to persist merged segment: " << new_segment_dir << ". Error: " << status.message(); + LOG_ENGINE_ERROR_ << "Failed to persist merged segment: " << new_segment_dir << ". Error: " << status.message(); // if failed to serialize merge file to disk // typical error: out of disk space, out of memory or permission denied collection_file.file_type_ = meta::SegmentSchema::TO_DELETE; status = meta_ptr_->UpdateCollectionFile(collection_file); - ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << collection_file.file_id_ - << " to to_delete"; + LOG_ENGINE_DEBUG_ << "Failed to update file to index, mark file: " << collection_file.file_id_ + << " to to_delete"; return status; } @@ -1440,8 +1440,8 @@ DBImpl::MergeFiles(const std::string& collection_id, const meta::SegmentsSchema& collection_file.row_count_ = segment_writer_ptr->VectorCount(); updated.push_back(collection_file); status = meta_ptr_->UpdateCollectionFiles(updated); - ENGINE_LOG_DEBUG << "New merged segment " << collection_file.segment_id_ << " of size " - << segment_writer_ptr->Size() << " bytes"; + LOG_ENGINE_DEBUG_ << "New merged segment " << collection_file.segment_id_ << " of size " + << segment_writer_ptr->Size() << " bytes"; if (options_.insert_cache_immediately_) { segment_writer_ptr->Cache(); @@ -1457,12 +1457,12 @@ DBImpl::BackgroundMergeFiles(const std::string& collection_id) { meta::SegmentsSchema raw_files; auto status = meta_ptr_->FilesToMerge(collection_id, raw_files); if (!status.ok()) { - ENGINE_LOG_ERROR << "Failed to get merge files for collection: " << collection_id; + LOG_ENGINE_ERROR_ << "Failed to get merge files for collection: " << collection_id; return status; } if (raw_files.size() < options_.merge_trigger_number_) { - ENGINE_LOG_TRACE << "Files number not greater equal than merge trigger number, skip merge action"; + LOG_ENGINE_TRACE_ << "Files number not greater equal than merge trigger number, skip merge action"; return Status::OK(); } @@ -1471,7 +1471,7 @@ DBImpl::BackgroundMergeFiles(const std::string& collection_id) { status = OngoingFileChecker::GetInstance().UnmarkOngoingFiles(raw_files); if (!initialized_.load(std::memory_order_acquire)) { - ENGINE_LOG_DEBUG << "Server will shutdown, skip merge action for collection: " << collection_id; + LOG_ENGINE_DEBUG_ << "Server will shutdown, skip merge action for collection: " << collection_id; } return Status::OK(); @@ -1479,17 +1479,17 @@ DBImpl::BackgroundMergeFiles(const std::string& collection_id) { void DBImpl::BackgroundMerge(std::set collection_ids) { - // ENGINE_LOG_TRACE << " Background merge thread start"; + // LOG_ENGINE_TRACE_ << " Background merge thread start"; Status status; for (auto& collection_id : collection_ids) { status = BackgroundMergeFiles(collection_id); if (!status.ok()) { - ENGINE_LOG_ERROR << "Merge files for collection " << collection_id << " failed: " << status.ToString(); + LOG_ENGINE_ERROR_ << "Merge files for collection " << collection_id << " failed: " << status.ToString(); } if (!initialized_.load(std::memory_order_acquire)) { - ENGINE_LOG_DEBUG << "Server will shutdown, skip merge action"; + LOG_ENGINE_DEBUG_ << "Server will shutdown, skip merge action"; break; } } @@ -1505,7 +1505,7 @@ DBImpl::BackgroundMerge(std::set collection_ids) { meta_ptr_->CleanUpFilesWithTTL(ttl); } - // ENGINE_LOG_TRACE << " Background merge thread exit"; + // LOG_ENGINE_TRACE_ << " Background merge thread exit"; } void @@ -1538,7 +1538,7 @@ DBImpl::BackgroundBuildIndex() { Status status = index_failed_checker_.IgnoreFailedIndexFiles(to_index_files); if (!to_index_files.empty()) { - ENGINE_LOG_DEBUG << "Background build index thread begin"; + LOG_ENGINE_DEBUG_ << "Background build index thread begin"; status = OngoingFileChecker::GetInstance().MarkOngoingFiles(to_index_files); // step 2: put build index task to scheduler @@ -1558,18 +1558,18 @@ DBImpl::BackgroundBuildIndex() { job->WaitBuildIndexFinish(); if (!job->GetStatus().ok()) { Status status = job->GetStatus(); - ENGINE_LOG_ERROR << "Building index job " << job->id() << " failed: " << status.ToString(); + LOG_ENGINE_ERROR_ << "Building index job " << job->id() << " failed: " << status.ToString(); index_failed_checker_.MarkFailedIndexFile(file_schema, status.message()); } else { - ENGINE_LOG_DEBUG << "Building index job " << job->id() << " succeed."; + LOG_ENGINE_DEBUG_ << "Building index job " << job->id() << " succeed."; index_failed_checker_.MarkSucceedIndexFile(file_schema); } status = OngoingFileChecker::GetInstance().UnmarkOngoingFile(file_schema); } - ENGINE_LOG_DEBUG << "Background build index thread finished"; + LOG_ENGINE_DEBUG_ << "Background build index thread finished"; } } @@ -1594,7 +1594,7 @@ DBImpl::GetFilesToBuildIndex(const std::string& collection_id, const std::vector Status DBImpl::GetFilesToSearch(const std::string& collection_id, meta::SegmentsSchema& files) { - ENGINE_LOG_DEBUG << "Collect files from collection: " << collection_id; + LOG_ENGINE_DEBUG_ << "Collect files from collection: " << collection_id; meta::SegmentsSchema search_files; auto status = meta_ptr_->FilesToSearch(collection_id, search_files); @@ -1629,7 +1629,7 @@ DBImpl::GetPartitionByTag(const std::string& collection_id, const std::string& p status = meta_ptr_->GetPartitionName(collection_id, partition_tag, partition_name); if (!status.ok()) { - ENGINE_LOG_ERROR << status.message(); + LOG_ENGINE_ERROR_ << status.message(); } } @@ -1670,7 +1670,7 @@ DBImpl::GetPartitionsByTags(const std::string& collection_id, const std::vector< Status DBImpl::DropCollectionRecursively(const std::string& collection_id) { // dates partly delete files of the collection but currently we don't support - ENGINE_LOG_DEBUG << "Prepare to delete collection " << collection_id; + LOG_ENGINE_DEBUG_ << "Prepare to delete collection " << collection_id; Status status; if (options_.wal_enable_) { @@ -1708,7 +1708,7 @@ DBImpl::UpdateCollectionIndexRecursively(const std::string& collection_id, const fiu_do_on("DBImpl.UpdateCollectionIndexRecursively.fail_update_collection_index", status = Status(DB_META_TRANSACTION_FAILED, "")); if (!status.ok()) { - ENGINE_LOG_ERROR << "Failed to update collection index info for collection: " << collection_id; + LOG_ENGINE_ERROR_ << "Failed to update collection index info for collection: " << collection_id; return status; } @@ -1748,7 +1748,7 @@ DBImpl::WaitCollectionIndexRecursively(const std::string& collection_id, const C int times = 1; while (!collection_files.empty()) { - ENGINE_LOG_DEBUG << "Non index files detected! Will build index " << times; + LOG_ENGINE_DEBUG_ << "Non index files detected! Will build index " << times; if (!utils::IsRawIndexType(index.engine_type_)) { status = meta_ptr_->UpdateCollectionFilesToIndex(collection_id); } @@ -1785,7 +1785,7 @@ DBImpl::WaitCollectionIndexRecursively(const std::string& collection_id, const C Status DBImpl::DropCollectionIndexRecursively(const std::string& collection_id) { - ENGINE_LOG_DEBUG << "Drop index for collection: " << collection_id; + LOG_ENGINE_DEBUG_ << "Drop index for collection: " << collection_id; index_failed_checker_.CleanFailedIndexFileOfCollection(collection_id); auto status = meta_ptr_->DropCollectionIndex(collection_id); if (!status.ok()) { @@ -1868,7 +1868,7 @@ DBImpl::ExecWalRecord(const wal::MXLogRecord& record) { std::string target_collection_name; status = GetPartitionByTag(record.collection_id, record.partition_tag, target_collection_name); if (!status.ok()) { - WAL_LOG_ERROR << LogOut("[%s][%ld] ", "insert", 0) << "Get partition fail: " << status.message(); + LOG_WAL_ERROR_ << LogOut("[%s][%ld] ", "insert", 0) << "Get partition fail: " << status.message(); return status; } @@ -1888,7 +1888,7 @@ DBImpl::ExecWalRecord(const wal::MXLogRecord& record) { std::string target_collection_name; status = GetPartitionByTag(record.collection_id, record.partition_tag, target_collection_name); if (!status.ok()) { - WAL_LOG_ERROR << LogOut("[%s][%ld] ", "insert", 0) << "Get partition fail: " << status.message(); + LOG_WAL_ERROR_ << LogOut("[%s][%ld] ", "insert", 0) << "Get partition fail: " << status.message(); return status; } @@ -1994,6 +1994,7 @@ DBImpl::InternalFlush(const std::string& collection_id) { void DBImpl::BackgroundWalThread() { + SetThreadName("wal_thread"); server::SystemInfo::GetInstance().Init(); std::chrono::system_clock::time_point next_auto_flush_time; @@ -2015,7 +2016,7 @@ DBImpl::BackgroundWalThread() { wal::MXLogRecord record; auto error_code = wal_mgr_->GetNextRecord(record); if (error_code != WAL_SUCCESS) { - ENGINE_LOG_ERROR << "WAL background GetNextRecord error"; + LOG_ENGINE_ERROR_ << "WAL background GetNextRecord error"; break; } @@ -2037,7 +2038,7 @@ DBImpl::BackgroundWalThread() { flush_req_swn_.Notify(); WaitMergeFileFinish(); WaitBuildIndexFinish(); - ENGINE_LOG_DEBUG << "WAL background thread exit"; + LOG_ENGINE_DEBUG_ << "WAL background thread exit"; break; } @@ -2052,10 +2053,11 @@ DBImpl::BackgroundWalThread() { void DBImpl::BackgroundFlushThread() { + SetThreadName("flush_thread"); server::SystemInfo::GetInstance().Init(); while (true) { if (!initialized_.load(std::memory_order_acquire)) { - ENGINE_LOG_DEBUG << "DB background flush thread exit"; + LOG_ENGINE_DEBUG_ << "DB background flush thread exit"; break; } @@ -2073,7 +2075,7 @@ DBImpl::BackgroundMetricThread() { server::SystemInfo::GetInstance().Init(); while (true) { if (!initialized_.load(std::memory_order_acquire)) { - ENGINE_LOG_DEBUG << "DB background metric thread exit"; + LOG_ENGINE_DEBUG_ << "DB background metric thread exit"; break; } diff --git a/core/src/db/IDGenerator.cpp b/core/src/db/IDGenerator.cpp index 0f8bd1f634f4409dc56e142f8b497ba6a80ddda1..cbc38f31f099fe52c597f6effcd4864d5acca0db 100644 --- a/core/src/db/IDGenerator.cpp +++ b/core/src/db/IDGenerator.cpp @@ -100,7 +100,7 @@ Status SafeIDGenerator::NextIDNumbers(size_t n, IDNumbers& ids) { if (n <= 0 || n > MAX_IDS_PER_MICRO) { std::string msg = "Invalid ID number: " + std::to_string(n); - ENGINE_LOG_ERROR << msg; + LOG_ENGINE_ERROR_ << msg; return Status(SERVER_UNEXPECTED_ERROR, msg); } diff --git a/core/src/db/OngoingFileChecker.cpp b/core/src/db/OngoingFileChecker.cpp index b4a19f7f7f733a09036d87cb75b3f2ffa09331fa..def2ad9d95c545bc233ca7f12b042a0029edac0b 100644 --- a/core/src/db/OngoingFileChecker.cpp +++ b/core/src/db/OngoingFileChecker.cpp @@ -94,8 +94,8 @@ OngoingFileChecker::MarkOngoingFileNoLock(const meta::SegmentSchema& table_file) } } - ENGINE_LOG_DEBUG << "Mark ongoing file:" << table_file.file_id_ - << " refcount:" << ongoing_files_[table_file.collection_id_][table_file.file_id_]; + LOG_ENGINE_DEBUG_ << "Mark ongoing file:" << table_file.file_id_ + << " refcount:" << ongoing_files_[table_file.collection_id_][table_file.file_id_]; return Status::OK(); } @@ -112,7 +112,7 @@ OngoingFileChecker::UnmarkOngoingFileNoLock(const meta::SegmentSchema& table_fil if (it_file != iter->second.end()) { it_file->second--; - ENGINE_LOG_DEBUG << "Unmark ongoing file:" << table_file.file_id_ << " refcount:" << it_file->second; + LOG_ENGINE_DEBUG_ << "Unmark ongoing file:" << table_file.file_id_ << " refcount:" << it_file->second; if (it_file->second <= 0) { iter->second.erase(table_file.file_id_); diff --git a/core/src/db/Options.cpp b/core/src/db/Options.cpp index 3ea70d2e28ce2d7d8d33b9f0ea28a141b820d33d..e2dbfdfcf000a0669707fd98184880b572dfd5ec 100644 --- a/core/src/db/Options.cpp +++ b/core/src/db/Options.cpp @@ -54,11 +54,11 @@ ArchiveConf::ParseCritirias(const std::string& criterias) { std::vector kv; boost::algorithm::split(kv, token, boost::is_any_of(":")); if (kv.size() != 2) { - ENGINE_LOG_WARNING << "Invalid ArchiveConf Criterias: " << token << " Ignore!"; + LOG_ENGINE_WARNING_ << "Invalid ArchiveConf Criterias: " << token << " Ignore!"; continue; } if (kv[0] != "disk" && kv[0] != "days") { - ENGINE_LOG_WARNING << "Invalid ArchiveConf Criterias: " << token << " Ignore!"; + LOG_ENGINE_WARNING_ << "Invalid ArchiveConf Criterias: " << token << " Ignore!"; continue; } try { @@ -68,11 +68,11 @@ ArchiveConf::ParseCritirias(const std::string& criterias) { criterias_[kv[0]] = value; } catch (std::out_of_range&) { std::string msg = "Out of range: '" + kv[1] + "'"; - ENGINE_LOG_ERROR << msg; + LOG_ENGINE_ERROR_ << msg; throw InvalidArgumentException(msg); } catch (...) { std::string msg = "Invalid argument: '" + kv[1] + "'"; - ENGINE_LOG_ERROR << msg; + LOG_ENGINE_ERROR_ << msg; throw InvalidArgumentException(msg); } } diff --git a/core/src/db/Utils.cpp b/core/src/db/Utils.cpp index 2c2a767b4924252915a668c45b7329ae5b473cfe..b79305220bed4f23a8862a7c081082dc0624642f 100644 --- a/core/src/db/Utils.cpp +++ b/core/src/db/Utils.cpp @@ -84,7 +84,7 @@ CreateCollectionPath(const DBMetaOptions& options, const std::string& collection std::string table_path = db_path + TABLES_FOLDER + collection_id; auto status = server::CommonUtil::CreateDirectory(table_path); if (!status.ok()) { - ENGINE_LOG_ERROR << status.message(); + LOG_ENGINE_ERROR_ << status.message(); return status; } @@ -93,7 +93,7 @@ CreateCollectionPath(const DBMetaOptions& options, const std::string& collection status = server::CommonUtil::CreateDirectory(table_path); fiu_do_on("CreateCollectionPath.creat_slave_path", status = Status(DB_INVALID_PATH, "")); if (!status.ok()) { - ENGINE_LOG_ERROR << status.message(); + LOG_ENGINE_ERROR_ << status.message(); return status; } } @@ -110,10 +110,10 @@ DeleteCollectionPath(const DBMetaOptions& options, const std::string& collection std::string table_path = path + TABLES_FOLDER + collection_id; if (force) { boost::filesystem::remove_all(table_path); - ENGINE_LOG_DEBUG << "Remove collection folder: " << table_path; + LOG_ENGINE_DEBUG_ << "Remove collection folder: " << table_path; } else if (boost::filesystem::exists(table_path) && boost::filesystem::is_empty(table_path)) { boost::filesystem::remove_all(table_path); - ENGINE_LOG_DEBUG << "Remove collection folder: " << table_path; + LOG_ENGINE_DEBUG_ << "Remove collection folder: " << table_path; } } @@ -141,7 +141,7 @@ CreateCollectionFilePath(const DBMetaOptions& options, meta::SegmentSchema& tabl auto status = server::CommonUtil::CreateDirectory(parent_path); fiu_do_on("CreateCollectionFilePath.fail_create", status = Status(DB_INVALID_PATH, "")); if (!status.ok()) { - ENGINE_LOG_ERROR << status.message(); + LOG_ENGINE_ERROR_ << status.message(); return status; } @@ -181,7 +181,7 @@ GetCollectionFilePath(const DBMetaOptions& options, meta::SegmentSchema& table_f std::string msg = "Collection file doesn't exist: " + file_path; if (table_file.file_size_ > 0) { // no need to pop error for empty file - ENGINE_LOG_ERROR << msg << " in path: " << options.path_ << " for collection: " << table_file.collection_id_; + LOG_ENGINE_ERROR_ << msg << " in path: " << options.path_ << " for collection: " << table_file.collection_id_; } return Status(DB_ERROR, msg); diff --git a/core/src/db/engine/EngineFactory.cpp b/core/src/db/engine/EngineFactory.cpp index 285c028d16aa3a8ee16e1031838bb4f19ccf0050..a3c50c89a81351e632a9e453f7bcf76bf4b532fa 100644 --- a/core/src/db/engine/EngineFactory.cpp +++ b/core/src/db/engine/EngineFactory.cpp @@ -22,11 +22,11 @@ ExecutionEnginePtr EngineFactory::Build(uint16_t dimension, const std::string& location, EngineType index_type, MetricType metric_type, const milvus::json& index_params) { if (index_type == EngineType::INVALID) { - ENGINE_LOG_ERROR << "Unsupported engine type"; + LOG_ENGINE_ERROR_ << "Unsupported engine type"; return nullptr; } - ENGINE_LOG_DEBUG << "EngineFactory index type: " << (int)index_type; + LOG_ENGINE_DEBUG_ << "EngineFactory index type: " << (int)index_type; ExecutionEnginePtr execution_engine_ptr = std::make_shared(dimension, location, index_type, metric_type, index_params); diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index efdedae6f300801d88b43ed15acbbad9c1496a68..d47fa327ad2baa96eec7ab952d224800a956296c 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -131,7 +131,7 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, const std::string& conf[knowhere::meta::DEVICEID] = gpu_num_; conf[knowhere::meta::DIM] = dimension; MappingMetricType(metric_type, conf); - ENGINE_LOG_DEBUG << "Index params: " << conf.dump(); + LOG_ENGINE_DEBUG_ << "Index params: " << conf.dump(); auto adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(index_->index_type()); if (!adapter->CheckTrain(conf, index_->index_mode())) { throw Exception(DB_ERROR, "Illegal index params"); @@ -223,13 +223,13 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) { break; } default: { - ENGINE_LOG_ERROR << "Unsupported index type " << (int)type; + LOG_ENGINE_ERROR_ << "Unsupported index type " << (int)type; return nullptr; } } if (index == nullptr) { std::string err_msg = "Invalid index type " + std::to_string((int)type) + " mod " + std::to_string((int)mode); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(DB_ERROR, err_msg); } return index; @@ -240,7 +240,7 @@ ExecutionEngineImpl::HybridLoad() const { #ifdef MILVUS_GPU_VERSION auto hybrid_index = std::dynamic_pointer_cast(index_); if (hybrid_index == nullptr) { - ENGINE_LOG_WARNING << "HybridLoad only support with IVFSQHybrid"; + LOG_ENGINE_WARNING_ << "HybridLoad only support with IVFSQHybrid"; return; } @@ -250,7 +250,7 @@ ExecutionEngineImpl::HybridLoad() const { std::vector gpus; Status s = config.GetGpuResourceConfigSearchResources(gpus); if (!s.ok()) { - ENGINE_LOG_ERROR << s.message(); + LOG_ENGINE_ERROR_ << s.message(); return; } @@ -289,9 +289,9 @@ ExecutionEngineImpl::HybridLoad() const { milvus::json quantizer_conf{{knowhere::meta::DEVICEID, best_device_id}, {"mode", 1}}; auto quantizer = hybrid_index->LoadQuantizer(quantizer_conf); - ENGINE_LOG_DEBUG << "Quantizer params: " << quantizer_conf.dump(); + LOG_ENGINE_DEBUG_ << "Quantizer params: " << quantizer_conf.dump(); if (quantizer == nullptr) { - ENGINE_LOG_ERROR << "quantizer is nullptr"; + LOG_ENGINE_ERROR_ << "quantizer is nullptr"; } hybrid_index->SetQuantizer(quantizer); auto cache_quantizer = std::make_shared(quantizer); @@ -328,7 +328,7 @@ ExecutionEngineImpl::AddWithIds(int64_t n, const uint8_t* xdata, const int64_t* size_t ExecutionEngineImpl::Count() const { if (index_ == nullptr) { - ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, return count 0"; + LOG_ENGINE_ERROR_ << "ExecutionEngineImpl: index is null, return count 0"; return 0; } return index_->Count(); @@ -337,7 +337,7 @@ ExecutionEngineImpl::Count() const { size_t ExecutionEngineImpl::Dimension() const { if (index_ == nullptr) { - ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, return dimension " << dim_; + LOG_ENGINE_ERROR_ << "ExecutionEngineImpl: index is null, return dimension " << dim_; return dim_; } return index_->Dim(); @@ -346,7 +346,7 @@ ExecutionEngineImpl::Dimension() const { size_t ExecutionEngineImpl::Size() const { if (index_ == nullptr) { - ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, return size 0"; + LOG_ENGINE_ERROR_ << "ExecutionEngineImpl: index is null, return size 0"; return 0; } return index_->Size(); @@ -363,7 +363,7 @@ ExecutionEngineImpl::Serialize() { // here we reset index size by file size, // since some index type(such as SQ8) data size become smaller after serialized index_->SetIndexSize(server::CommonUtil::GetFileSize(location_)); - ENGINE_LOG_DEBUG << "Finish serialize index file: " << location_ << " size: " << index_->Size(); + LOG_ENGINE_DEBUG_ << "Finish serialize index file: " << location_ << " size: " << index_->Size(); if (index_->Size() == 0) { std::string msg = "Failed to serialize file: " + location_ + " reason: out of disk space or memory"; @@ -394,7 +394,7 @@ ExecutionEngineImpl::Load(bool to_cache) { milvus::json conf{{knowhere::meta::DEVICEID, gpu_num_}, {knowhere::meta::DIM, dim_}}; MappingMetricType(metric_type_, conf); auto adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(index_->index_type()); - ENGINE_LOG_DEBUG << "Index params: " << conf.dump(); + LOG_ENGINE_DEBUG_ << "Index params: " << conf.dump(); if (!adapter->CheckTrain(conf, index_->index_mode())) { throw Exception(DB_ERROR, "Illegal index params"); } @@ -402,7 +402,7 @@ ExecutionEngineImpl::Load(bool to_cache) { auto status = segment_reader_ptr->Load(); if (!status.ok()) { std::string msg = "Failed to load segment from " + location_; - ENGINE_LOG_ERROR << msg; + LOG_ENGINE_ERROR_ << msg; return Status(DB_ERROR, msg); } @@ -413,7 +413,7 @@ ExecutionEngineImpl::Load(bool to_cache) { auto vectors_uids = vectors->GetUids(); index_->SetUids(vectors_uids); - ENGINE_LOG_DEBUG << "set uids " << index_->GetUids().size() << " for index " << location_; + LOG_ENGINE_DEBUG_ << "set uids " << index_->GetUids().size() << " for index " << location_; auto vectors_data = vectors->GetData(); @@ -438,7 +438,7 @@ ExecutionEngineImpl::Load(bool to_cache) { bin_bf_index->SetBlacklist(concurrent_bitset_ptr); } - ENGINE_LOG_DEBUG << "Finished loading raw data from segment " << segment_dir; + LOG_ENGINE_DEBUG_ << "Finished loading raw data from segment " << segment_dir; } else { try { segment::SegmentPtr segment_ptr; @@ -448,14 +448,14 @@ ExecutionEngineImpl::Load(bool to_cache) { if (index_ == nullptr) { std::string msg = "Failed to load index from " + location_; - ENGINE_LOG_ERROR << msg; + LOG_ENGINE_ERROR_ << msg; return Status(DB_ERROR, msg); } else { segment::DeletedDocsPtr deleted_docs_ptr; auto status = segment_reader_ptr->LoadDeletedDocs(deleted_docs_ptr); if (!status.ok()) { std::string msg = "Failed to load deleted docs from " + location_; - ENGINE_LOG_ERROR << msg; + LOG_ENGINE_ERROR_ << msg; return Status(DB_ERROR, msg); } auto& deleted_docs = deleted_docs_ptr->GetDeletedDocs(); @@ -473,12 +473,12 @@ ExecutionEngineImpl::Load(bool to_cache) { std::vector uids; segment_reader_ptr->LoadUids(uids); index_->SetUids(uids); - ENGINE_LOG_DEBUG << "set uids " << index_->GetUids().size() << " for index " << location_; + LOG_ENGINE_DEBUG_ << "set uids " << index_->GetUids().size() << " for index " << location_; - ENGINE_LOG_DEBUG << "Finished loading index file from segment " << segment_dir; + LOG_ENGINE_DEBUG_ << "Finished loading index file from segment " << segment_dir; } } catch (std::exception& e) { - ENGINE_LOG_ERROR << e.what(); + LOG_ENGINE_ERROR_ << e.what(); return Status(DB_ERROR, e.what()); } } @@ -552,7 +552,7 @@ ExecutionEngineImpl::CopyToGpu(uint64_t device_id, bool hybrid) { index_ = index; } else { if (index_ == nullptr) { - ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to copy to gpu"; + LOG_ENGINE_ERROR_ << "ExecutionEngineImpl: index is null, failed to copy to gpu"; return Status(DB_ERROR, "index is null"); } @@ -561,14 +561,14 @@ ExecutionEngineImpl::CopyToGpu(uint64_t device_id, bool hybrid) { * Add lock here to avoid multiple INDEX are copied to one GPU card at same time. * And reserve space to avoid GPU out of memory issue. */ - ENGINE_LOG_DEBUG << "CPU to GPU" << device_id << " start"; + LOG_ENGINE_DEBUG_ << "CPU to GPU" << device_id << " start"; auto gpu_cache_mgr = cache::GpuCacheMgr::GetInstance(device_id); // gpu_cache_mgr->Reserve(index_->Size()); index_ = knowhere::cloner::CopyCpuToGpu(index_, device_id, knowhere::Config()); // gpu_cache_mgr->InsertItem(location_, std::static_pointer_cast(index_)); - ENGINE_LOG_DEBUG << "CPU to GPU" << device_id << " finished"; + LOG_ENGINE_DEBUG_ << "CPU to GPU" << device_id << " finished"; } catch (std::exception& e) { - ENGINE_LOG_ERROR << e.what(); + LOG_ENGINE_ERROR_ << e.what(); return Status(DB_ERROR, e.what()); } } @@ -599,15 +599,15 @@ ExecutionEngineImpl::CopyToCpu() { index_ = index; } else { if (index_ == nullptr) { - ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to copy to cpu"; + LOG_ENGINE_ERROR_ << "ExecutionEngineImpl: index is null, failed to copy to cpu"; return Status(DB_ERROR, "index is null"); } try { index_ = knowhere::cloner::CopyGpuToCpu(index_, knowhere::Config()); - ENGINE_LOG_DEBUG << "GPU to CPU"; + LOG_ENGINE_DEBUG_ << "GPU to CPU"; } catch (std::exception& e) { - ENGINE_LOG_ERROR << e.what(); + LOG_ENGINE_ERROR_ << e.what(); return Status(DB_ERROR, e.what()); } } @@ -617,19 +617,19 @@ ExecutionEngineImpl::CopyToCpu() { } return Status::OK(); #else - ENGINE_LOG_ERROR << "Calling ExecutionEngineImpl::CopyToCpu when using CPU version"; + LOG_ENGINE_ERROR_ << "Calling ExecutionEngineImpl::CopyToCpu when using CPU version"; return Status(DB_ERROR, "Calling ExecutionEngineImpl::CopyToCpu when using CPU version"); #endif } ExecutionEnginePtr ExecutionEngineImpl::BuildIndex(const std::string& location, EngineType engine_type) { - ENGINE_LOG_DEBUG << "Build index file: " << location << " from: " << location_; + LOG_ENGINE_DEBUG_ << "Build index file: " << location << " from: " << location_; auto from_index = std::dynamic_pointer_cast(index_); auto bin_from_index = std::dynamic_pointer_cast(index_); if (from_index == nullptr && bin_from_index == nullptr) { - ENGINE_LOG_ERROR << "ExecutionEngineImpl: from_index is null, failed to build index"; + LOG_ENGINE_ERROR_ << "ExecutionEngineImpl: from_index is null, failed to build index"; return nullptr; } @@ -643,12 +643,12 @@ ExecutionEngineImpl::BuildIndex(const std::string& location, EngineType engine_t conf[knowhere::meta::ROWS] = Count(); conf[knowhere::meta::DEVICEID] = gpu_num_; MappingMetricType(metric_type_, conf); - ENGINE_LOG_DEBUG << "Index params: " << conf.dump(); + LOG_ENGINE_DEBUG_ << "Index params: " << conf.dump(); auto adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(to_index->index_type()); if (!adapter->CheckTrain(conf, to_index->index_mode())) { throw Exception(DB_ERROR, "Illegal index params"); } - ENGINE_LOG_DEBUG << "Index config: " << conf.dump(); + LOG_ENGINE_DEBUG_ << "Index config: " << conf.dump(); std::vector uids; faiss::ConcurrentBitsetPtr blacklist; @@ -675,13 +675,13 @@ ExecutionEngineImpl::BuildIndex(const std::string& location, EngineType engine_t #endif to_index->SetUids(uids); - ENGINE_LOG_DEBUG << "Set " << to_index->GetUids().size() << "uids for " << location; + LOG_ENGINE_DEBUG_ << "Set " << to_index->GetUids().size() << "uids for " << location; if (blacklist != nullptr) { to_index->SetBlacklist(blacklist); - ENGINE_LOG_DEBUG << "Set blacklist for index " << location; + LOG_ENGINE_DEBUG_ << "Set blacklist for index " << location; } - ENGINE_LOG_DEBUG << "Finish build index: " << location; + LOG_ENGINE_DEBUG_ << "Finish build index: " << location; return std::make_shared(to_index, location, engine_type, metric_type_, index_params_); } @@ -766,7 +766,7 @@ ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, const milvu TimeRecorder rc(LogOut("[%s][%ld] ExecutionEngineImpl::Search float", "search", 0)); if (index_ == nullptr) { - ENGINE_LOG_ERROR << LogOut("[%s][%ld] ExecutionEngineImpl: index is null, failed to search", "search", 0); + LOG_ENGINE_ERROR_ << LogOut("[%s][%ld] ExecutionEngineImpl: index is null, failed to search", "search", 0); return Status(DB_ERROR, "index is null"); } @@ -774,7 +774,7 @@ ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, const milvu conf[knowhere::meta::TOPK] = k; auto adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(index_->index_type()); if (!adapter->CheckSearch(conf, index_->index_type(), index_->index_mode())) { - ENGINE_LOG_ERROR << LogOut("[%s][%ld] Illegal search params", "search", 0); + LOG_ENGINE_ERROR_ << LogOut("[%s][%ld] Illegal search params", "search", 0); throw Exception(DB_ERROR, "Illegal search params"); } @@ -787,8 +787,8 @@ ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, const milvu auto result = index_->Query(dataset, conf); rc.RecordSection("query done"); - ENGINE_LOG_DEBUG << LogOut("[%s][%ld] get %ld uids from index %s", "search", 0, index_->GetUids().size(), - location_.c_str()); + LOG_ENGINE_DEBUG_ << LogOut("[%s][%ld] get %ld uids from index %s", "search", 0, index_->GetUids().size(), + location_.c_str()); MapAndCopyResult(result, index_->GetUids(), n, k, distances, labels); rc.RecordSection("map uids " + std::to_string(n * k)); @@ -805,7 +805,7 @@ ExecutionEngineImpl::Search(int64_t n, const uint8_t* data, int64_t k, const mil TimeRecorder rc(LogOut("[%s][%ld] ExecutionEngineImpl::Search uint8", "search", 0)); if (index_ == nullptr) { - ENGINE_LOG_ERROR << LogOut("[%s][%ld] ExecutionEngineImpl: index is null, failed to search", "search", 0); + LOG_ENGINE_ERROR_ << LogOut("[%s][%ld] ExecutionEngineImpl: index is null, failed to search", "search", 0); return Status(DB_ERROR, "index is null"); } @@ -813,7 +813,7 @@ ExecutionEngineImpl::Search(int64_t n, const uint8_t* data, int64_t k, const mil conf[knowhere::meta::TOPK] = k; auto adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(index_->index_type()); if (!adapter->CheckSearch(conf, index_->index_type(), index_->index_mode())) { - ENGINE_LOG_ERROR << LogOut("[%s][%ld] Illegal search params", "search", 0); + LOG_ENGINE_ERROR_ << LogOut("[%s][%ld] Illegal search params", "search", 0); throw Exception(DB_ERROR, "Illegal search params"); } @@ -826,8 +826,8 @@ ExecutionEngineImpl::Search(int64_t n, const uint8_t* data, int64_t k, const mil auto result = index_->Query(dataset, conf); rc.RecordSection("query done"); - ENGINE_LOG_DEBUG << LogOut("[%s][%ld] get %ld uids from index %s", "search", 0, index_->GetUids().size(), - location_.c_str()); + LOG_ENGINE_DEBUG_ << LogOut("[%s][%ld] get %ld uids from index %s", "search", 0, index_->GetUids().size(), + location_.c_str()); MapAndCopyResult(result, index_->GetUids(), n, k, distances, labels); rc.RecordSection("map uids " + std::to_string(n * k)); @@ -844,7 +844,7 @@ ExecutionEngineImpl::Search(int64_t n, const std::vector& ids, int64_t TimeRecorder rc(LogOut("[%s][%ld] ExecutionEngineImpl::Search vector of ids", "search", 0)); if (index_ == nullptr) { - ENGINE_LOG_ERROR << LogOut("[%s][%ld] ExecutionEngineImpl: index is null, failed to search", "search", 0); + LOG_ENGINE_ERROR_ << LogOut("[%s][%ld] ExecutionEngineImpl: index is null, failed to search", "search", 0); return Status(DB_ERROR, "index is null"); } @@ -852,7 +852,7 @@ ExecutionEngineImpl::Search(int64_t n, const std::vector& ids, int64_t conf[knowhere::meta::TOPK] = k; auto adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(index_->index_type()); if (!adapter->CheckSearch(conf, index_->index_type(), index_->index_mode())) { - ENGINE_LOG_ERROR << LogOut("[%s][%ld] Illegal search params", "search", 0); + LOG_ENGINE_ERROR_ << LogOut("[%s][%ld] Illegal search params", "search", 0); throw Exception(DB_ERROR, "Illegal search params"); } @@ -906,8 +906,8 @@ ExecutionEngineImpl::Search(int64_t n, const std::vector& ids, int64_t auto result = index_->QueryById(dataset, conf); rc.RecordSection("query by id done"); - ENGINE_LOG_DEBUG << LogOut("[%s][%ld] get %ld uids from index %s", "search", 0, index_->GetUids().size(), - location_.c_str()); + LOG_ENGINE_DEBUG_ << LogOut("[%s][%ld] get %ld uids from index %s", "search", 0, index_->GetUids().size(), + location_.c_str()); MapAndCopyResult(result, uids, offsets.size(), k, distances, labels); rc.RecordSection("map uids " + std::to_string(offsets.size() * k)); } @@ -922,7 +922,7 @@ ExecutionEngineImpl::Search(int64_t n, const std::vector& ids, int64_t Status ExecutionEngineImpl::GetVectorByID(const int64_t& id, float* vector, bool hybrid) { if (index_ == nullptr) { - ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to search"; + LOG_ENGINE_ERROR_ << "ExecutionEngineImpl: index is null, failed to search"; return Status(DB_ERROR, "index is null"); } @@ -947,11 +947,11 @@ ExecutionEngineImpl::GetVectorByID(const int64_t& id, float* vector, bool hybrid Status ExecutionEngineImpl::GetVectorByID(const int64_t& id, uint8_t* vector, bool hybrid) { if (index_ == nullptr) { - ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to search"; + LOG_ENGINE_ERROR_ << "ExecutionEngineImpl: index is null, failed to search"; return Status(DB_ERROR, "index is null"); } - ENGINE_LOG_DEBUG << "Get binary vector by id: " << id; + LOG_ENGINE_DEBUG_ << "Get binary vector by id: " << id; if (hybrid) { HybridLoad(); diff --git a/core/src/db/insert/MemManagerImpl.cpp b/core/src/db/insert/MemManagerImpl.cpp index c35fc42ba83b1902072be50c04580f97caa8d6de..13a827e692257ace4a8a55cc524c25fd8f3002c9 100644 --- a/core/src/db/insert/MemManagerImpl.cpp +++ b/core/src/db/insert/MemManagerImpl.cpp @@ -36,7 +36,7 @@ MemManagerImpl::InsertVectors(const std::string& collection_id, int64_t length, const float* vectors, uint64_t lsn, std::set& flushed_tables) { flushed_tables.clear(); if (GetCurrentMem() > options_.insert_buffer_size_) { - ENGINE_LOG_DEBUG << "Insert buffer size exceeds limit. Performing force flush"; + LOG_ENGINE_DEBUG_ << "Insert buffer size exceeds limit. Performing force flush"; // TODO(zhiru): Don't apply delete here in order to avoid possible concurrency issues with Merge auto status = Flush(flushed_tables, false); if (!status.ok()) { @@ -62,12 +62,12 @@ MemManagerImpl::InsertVectors(const std::string& collection_id, int64_t length, const uint8_t* vectors, uint64_t lsn, std::set& flushed_tables) { flushed_tables.clear(); if (GetCurrentMem() > options_.insert_buffer_size_) { - ENGINE_LOG_DEBUG << LogOut("[%s][%ld] ", "insert", 0) - << "Insert buffer size exceeds limit. Performing force flush"; + LOG_ENGINE_DEBUG_ << LogOut("[%s][%ld] ", "insert", 0) + << "Insert buffer size exceeds limit. Performing force flush"; // TODO(zhiru): Don't apply delete here in order to avoid possible concurrency issues with Merge auto status = Flush(flushed_tables, false); if (!status.ok()) { - ENGINE_LOG_DEBUG << LogOut("[%s][%ld] ", "insert", 0) << "Flush fail: " << status.message(); + LOG_ENGINE_DEBUG_ << LogOut("[%s][%ld] ", "insert", 0) << "Flush fail: " << status.message(); return status; } } @@ -143,13 +143,13 @@ MemManagerImpl::Flush(const std::string& collection_id, bool apply_delete) { std::unique_lock lock(serialization_mtx_); auto max_lsn = GetMaxLSN(temp_immutable_list); for (auto& mem : temp_immutable_list) { - ENGINE_LOG_DEBUG << "Flushing collection: " << mem->GetTableId(); + LOG_ENGINE_DEBUG_ << "Flushing collection: " << mem->GetTableId(); auto status = mem->Serialize(max_lsn, apply_delete); if (!status.ok()) { - ENGINE_LOG_ERROR << "Flush collection " << mem->GetTableId() << " failed"; + LOG_ENGINE_ERROR_ << "Flush collection " << mem->GetTableId() << " failed"; return status; } - ENGINE_LOG_DEBUG << "Flushed collection: " << mem->GetTableId(); + LOG_ENGINE_DEBUG_ << "Flushed collection: " << mem->GetTableId(); } return Status::OK(); @@ -169,14 +169,14 @@ MemManagerImpl::Flush(std::set& table_ids, bool apply_delete) { table_ids.clear(); auto max_lsn = GetMaxLSN(temp_immutable_list); for (auto& mem : temp_immutable_list) { - ENGINE_LOG_DEBUG << "Flushing collection: " << mem->GetTableId(); + LOG_ENGINE_DEBUG_ << "Flushing collection: " << mem->GetTableId(); auto status = mem->Serialize(max_lsn, apply_delete); if (!status.ok()) { - ENGINE_LOG_ERROR << "Flush collection " << mem->GetTableId() << " failed"; + LOG_ENGINE_ERROR_ << "Flush collection " << mem->GetTableId() << " failed"; return status; } table_ids.insert(mem->GetTableId()); - ENGINE_LOG_DEBUG << "Flushed collection: " << mem->GetTableId(); + LOG_ENGINE_DEBUG_ << "Flushed collection: " << mem->GetTableId(); } meta_->SetGlobalLastLSN(max_lsn); @@ -194,7 +194,7 @@ MemManagerImpl::ToImmutable(const std::string& collection_id) { mem_id_map_.erase(memIt); } // std::string err_msg = "Could not find collection = " + collection_id + " to flush"; - // ENGINE_LOG_ERROR << err_msg; + // LOG_ENGINE_ERROR_ << err_msg; // return Status(DB_NOT_FOUND, err_msg); } diff --git a/core/src/db/insert/MemTable.cpp b/core/src/db/insert/MemTable.cpp index 6a0d77bced6f4b9534c82e05cdc99f37375af307..a7066b176d8242cee683c060ca6fc7f02b077553 100644 --- a/core/src/db/insert/MemTable.cpp +++ b/core/src/db/insert/MemTable.cpp @@ -53,7 +53,7 @@ MemTable::Add(const VectorSourcePtr& source) { if (!status.ok()) { std::string err_msg = "Insert failed: " + status.ToString(); - ENGINE_LOG_ERROR << LogOut("[%s][%ld] ", "insert", 0) << err_msg; + LOG_ENGINE_ERROR_ << LogOut("[%s][%ld] ", "insert", 0) << err_msg; return Status(DB_ERROR, err_msg); } } @@ -113,7 +113,7 @@ MemTable::Serialize(uint64_t wal_lsn, bool apply_delete) { return status; } - ENGINE_LOG_DEBUG << "Flushed segment " << (*mem_table_file)->GetSegmentId(); + LOG_ENGINE_DEBUG_ << "Flushed segment " << (*mem_table_file)->GetSegmentId(); { std::lock_guard lock(mutex_); @@ -125,13 +125,13 @@ MemTable::Serialize(uint64_t wal_lsn, bool apply_delete) { auto status = meta_->UpdateCollectionFlushLSN(collection_id_, wal_lsn); if (!status.ok()) { std::string err_msg = "Failed to write flush lsn to meta: " + status.ToString(); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; return Status(DB_ERROR, err_msg); } auto end = std::chrono::high_resolution_clock::now(); std::chrono::duration diff = end - start; - ENGINE_LOG_DEBUG << "Finished flushing for collection " << collection_id_ << " in " << diff.count() << " s"; + LOG_ENGINE_DEBUG_ << "Finished flushing for collection " << collection_id_ << " in " << diff.count() << " s"; return Status::OK(); } @@ -173,7 +173,7 @@ MemTable::ApplyDeletes() { // Serialize segment's deletedDoc TODO(zhiru): append directly to previous file for now, may have duplicates // Serialize bloom filter - ENGINE_LOG_DEBUG << "Applying " << doc_ids_to_delete_.size() << " deletes in collection: " << collection_id_; + LOG_ENGINE_DEBUG_ << "Applying " << doc_ids_to_delete_.size() << " deletes in collection: " << collection_id_; auto start_total = std::chrono::high_resolution_clock::now(); @@ -185,7 +185,7 @@ MemTable::ApplyDeletes() { auto status = meta_->FilesByType(collection_id_, file_types, table_files); if (!status.ok()) { std::string err_msg = "Failed to apply deletes: " + status.ToString(); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; return Status(DB_ERROR, err_msg); } @@ -220,13 +220,14 @@ MemTable::ApplyDeletes() { auto time0 = std::chrono::high_resolution_clock::now(); std::chrono::duration diff0 = time0 - start_total; - ENGINE_LOG_DEBUG << "Found " << ids_to_check_map.size() << " segment to apply deletes in " << diff0.count() << " s"; + LOG_ENGINE_DEBUG_ << "Found " << ids_to_check_map.size() << " segment to apply deletes in " << diff0.count() + << " s"; meta::SegmentsSchema table_files_to_update; for (auto& kv : ids_to_check_map) { auto& table_file = table_files[kv.first]; - ENGINE_LOG_DEBUG << "Applying deletes in segment: " << table_file.segment_id_; + LOG_ENGINE_DEBUG_ << "Applying deletes in segment: " << table_file.segment_id_; auto time1 = std::chrono::high_resolution_clock::now(); @@ -273,13 +274,13 @@ MemTable::ApplyDeletes() { auto time2 = std::chrono::high_resolution_clock::now(); std::chrono::duration diff1 = time2 - time1; - ENGINE_LOG_DEBUG << "Loading uids and deleted docs took " << diff1.count() << " s"; + LOG_ENGINE_DEBUG_ << "Loading uids and deleted docs took " << diff1.count() << " s"; std::sort(ids_to_check.begin(), ids_to_check.end()); auto time3 = std::chrono::high_resolution_clock::now(); std::chrono::duration diff2 = time3 - time2; - ENGINE_LOG_DEBUG << "Sorting " << ids_to_check.size() << " ids took " << diff2.count() << " s"; + LOG_ENGINE_DEBUG_ << "Sorting " << ids_to_check.size() << " ids took " << diff2.count() << " s"; size_t delete_count = 0; auto find_diff = std::chrono::duration::zero(); @@ -315,9 +316,9 @@ MemTable::ApplyDeletes() { } } - ENGINE_LOG_DEBUG << "Finding " << ids_to_check.size() << " uids in " << uids.size() << " uids took " - << find_diff.count() << " s in total"; - ENGINE_LOG_DEBUG << "Setting deleted docs and bloom filter took " << set_diff.count() << " s in total"; + LOG_ENGINE_DEBUG_ << "Finding " << ids_to_check.size() << " uids in " << uids.size() << " uids took " + << find_diff.count() << " s in total"; + LOG_ENGINE_DEBUG_ << "Setting deleted docs and bloom filter took " << set_diff.count() << " s in total"; auto time4 = std::chrono::high_resolution_clock::now(); @@ -336,9 +337,9 @@ MemTable::ApplyDeletes() { auto time5 = std::chrono::high_resolution_clock::now(); std::chrono::duration diff4 = time5 - time4; - ENGINE_LOG_DEBUG << "Appended " << deleted_docs->GetSize() - << " offsets to deleted docs in segment: " << table_file.segment_id_ << " in " << diff4.count() - << " s"; + LOG_ENGINE_DEBUG_ << "Appended " << deleted_docs->GetSize() + << " offsets to deleted docs in segment: " << table_file.segment_id_ << " in " + << diff4.count() << " s"; // start = std::chrono::high_resolution_clock::now(); @@ -348,8 +349,8 @@ MemTable::ApplyDeletes() { } auto time6 = std::chrono::high_resolution_clock::now(); std::chrono::duration diff5 = time6 - time5; - ENGINE_LOG_DEBUG << "Updated bloom filter in segment: " << table_file.segment_id_ << " in " << diff5.count() - << " s"; + LOG_ENGINE_DEBUG_ << "Updated bloom filter in segment: " << table_file.segment_id_ << " in " << diff5.count() + << " s"; // Update collection file row count for (auto& file : segment_files) { @@ -362,8 +363,8 @@ MemTable::ApplyDeletes() { auto time7 = std::chrono::high_resolution_clock::now(); std::chrono::duration diff6 = time7 - time6; diff6 = time6 - time5; - ENGINE_LOG_DEBUG << "Update collection file row count in vector of segment: " << table_file.segment_id_ - << " in " << diff6.count() << " s"; + LOG_ENGINE_DEBUG_ << "Update collection file row count in vector of segment: " << table_file.segment_id_ + << " in " << diff6.count() << " s"; } auto time7 = std::chrono::high_resolution_clock::now(); @@ -372,7 +373,7 @@ MemTable::ApplyDeletes() { if (!status.ok()) { std::string err_msg = "Failed to apply deletes: " + status.ToString(); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; return Status(DB_ERROR, err_msg); } @@ -380,9 +381,9 @@ MemTable::ApplyDeletes() { auto end_total = std::chrono::high_resolution_clock::now(); std::chrono::duration diff7 = end_total - time7; - ENGINE_LOG_DEBUG << "Update deletes to meta in collection " << collection_id_ << " in " << diff7.count() << " s"; + LOG_ENGINE_DEBUG_ << "Update deletes to meta in collection " << collection_id_ << " in " << diff7.count() << " s"; std::chrono::duration diff_total = end_total - start_total; - ENGINE_LOG_DEBUG << "Finished deletes in collection " << collection_id_ << " in " << diff_total.count() << " s"; + LOG_ENGINE_DEBUG_ << "Finished deletes in collection " << collection_id_ << " in " << diff_total.count() << " s"; OngoingFileChecker::GetInstance().UnmarkOngoingFiles(files_to_check); diff --git a/core/src/db/insert/MemTableFile.cpp b/core/src/db/insert/MemTableFile.cpp index 48b21c6b8b42fc74bea86330bc46d8b3cc106199..1e11e763fddc6bb2dc37252798fc55891fd3f9c0 100644 --- a/core/src/db/insert/MemTableFile.cpp +++ b/core/src/db/insert/MemTableFile.cpp @@ -54,7 +54,7 @@ MemTableFile::CreateCollectionFile() { table_file_schema_ = table_file_schema; } else { std::string err_msg = "MemTableFile::CreateCollectionFile failed: " + status.ToString(); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; } return status; } @@ -65,7 +65,7 @@ MemTableFile::Add(const VectorSourcePtr& source) { std::string err_msg = "MemTableFile::Add: table_file_schema dimension = " + std::to_string(table_file_schema_.dimension_) + ", collection_id = " + table_file_schema_.collection_id_; - ENGINE_LOG_ERROR << LogOut("[%s][%ld]", "insert", 0) << err_msg; + LOG_ENGINE_ERROR_ << LogOut("[%s][%ld]", "insert", 0) << err_msg; return Status(DB_ERROR, "Not able to create collection file"); } @@ -160,7 +160,7 @@ MemTableFile::Serialize(uint64_t wal_lsn) { auto status = segment_writer_ptr_->Serialize(); if (!status.ok()) { - ENGINE_LOG_ERROR << "Failed to serialize segment: " << table_file_schema_.segment_id_; + LOG_ENGINE_ERROR_ << "Failed to serialize segment: " << table_file_schema_.segment_id_; /* Can't mark it as to_delete because data is stored in this mem collection file. Any further flush * will try to serialize the same mem collection file and it won't be able to find the directory @@ -168,7 +168,7 @@ MemTableFile::Serialize(uint64_t wal_lsn) { * table_file_schema_.file_type_ = meta::SegmentSchema::TO_DELETE; meta_->UpdateCollectionFile(table_file_schema_); - ENGINE_LOG_DEBUG << "Failed to serialize segment, mark file: " << table_file_schema_.file_id_ + LOG_ENGINE_DEBUG_ << "Failed to serialize segment, mark file: " << table_file_schema_.file_id_ << " to to_delete"; */ return status; @@ -194,8 +194,8 @@ MemTableFile::Serialize(uint64_t wal_lsn) { status = meta_->UpdateCollectionFile(table_file_schema_); - ENGINE_LOG_DEBUG << "New " << ((table_file_schema_.file_type_ == meta::SegmentSchema::RAW) ? "raw" : "to_index") - << " file " << table_file_schema_.file_id_ << " of size " << size << " bytes, lsn = " << wal_lsn; + LOG_ENGINE_DEBUG_ << "New " << ((table_file_schema_.file_type_ == meta::SegmentSchema::RAW) ? "raw" : "to_index") + << " file " << table_file_schema_.file_id_ << " of size " << size << " bytes, lsn = " << wal_lsn; // TODO(zhiru): cache /* diff --git a/core/src/db/insert/VectorSource.cpp b/core/src/db/insert/VectorSource.cpp index d738787d2680d736d758911a82a94ef649960758..49bb59b7f12254dab6119303d7043a7dd4322758 100644 --- a/core/src/db/insert/VectorSource.cpp +++ b/core/src/db/insert/VectorSource.cpp @@ -40,7 +40,7 @@ VectorSource::Add(/*const ExecutionEnginePtr& execution_engine,*/ const segment: SafeIDGenerator& id_generator = SafeIDGenerator::GetInstance(); Status status = id_generator.GetNextIDNumbers(num_vectors_added, vector_ids_to_add); if (!status.ok()) { - ENGINE_LOG_ERROR << LogOut("[%s][%ld]", "insert", 0) << "Generate ids fail: " << status.message(); + LOG_ENGINE_ERROR_ << LogOut("[%s][%ld]", "insert", 0) << "Generate ids fail: " << status.message(); return status; } } else { @@ -62,7 +62,7 @@ VectorSource::Add(/*const ExecutionEnginePtr& execution_engine,*/ const segment: vectors.resize(size); memcpy(vectors.data(), vectors_.float_data_.data() + current_num_vectors_added * table_file_schema.dimension_, size); - ENGINE_LOG_DEBUG << LogOut("[%s][%ld]", "insert", 0) << "Insert into segment"; + LOG_ENGINE_DEBUG_ << LogOut("[%s][%ld]", "insert", 0) << "Insert into segment"; status = segment_writer_ptr->AddVectors(table_file_schema.file_id_, vectors, vector_ids_to_add); } else if (!vectors_.binary_data_.empty()) { @@ -79,7 +79,7 @@ VectorSource::Add(/*const ExecutionEnginePtr& execution_engine,*/ const segment: vectors.data(), vectors_.binary_data_.data() + current_num_vectors_added * SingleVectorSize(table_file_schema.dimension_), size); - ENGINE_LOG_DEBUG << LogOut("[%s][%ld]", "insert", 0) << "Insert into segment"; + LOG_ENGINE_DEBUG_ << LogOut("[%s][%ld]", "insert", 0) << "Insert into segment"; status = segment_writer_ptr->AddVectors(table_file_schema.file_id_, vectors, vector_ids_to_add); } @@ -90,7 +90,7 @@ VectorSource::Add(/*const ExecutionEnginePtr& execution_engine,*/ const segment: vector_ids_.insert(vector_ids_.end(), std::make_move_iterator(vector_ids_to_add.begin()), std::make_move_iterator(vector_ids_to_add.end())); } else { - ENGINE_LOG_ERROR << LogOut("[%s][%ld]", "insert", 0) << "VectorSource::Add failed: " + status.ToString(); + LOG_ENGINE_ERROR_ << LogOut("[%s][%ld]", "insert", 0) << "VectorSource::Add failed: " + status.ToString(); } return status; diff --git a/core/src/db/meta/MetaFactory.cpp b/core/src/db/meta/MetaFactory.cpp index e27828daa538bf9a14fcce215c20b191eda8ae89..03777581fbb963faae889f6baf52ec17ca67c5f0 100644 --- a/core/src/db/meta/MetaFactory.cpp +++ b/core/src/db/meta/MetaFactory.cpp @@ -50,18 +50,18 @@ MetaFactory::Build(const DBMetaOptions& metaOptions, const int& mode) { utils::MetaUriInfo uri_info; auto status = utils::ParseMetaUri(uri, uri_info); if (!status.ok()) { - ENGINE_LOG_ERROR << "Wrong URI format: URI = " << uri; + LOG_ENGINE_ERROR_ << "Wrong URI format: URI = " << uri; throw InvalidArgumentException("Wrong URI format "); } if (strcasecmp(uri_info.dialect_.c_str(), "mysql") == 0) { - ENGINE_LOG_INFO << "Using MySQL"; + LOG_ENGINE_INFO_ << "Using MySQL"; return std::make_shared(metaOptions, mode); } else if (strcasecmp(uri_info.dialect_.c_str(), "sqlite") == 0) { - ENGINE_LOG_INFO << "Using SQLite"; + LOG_ENGINE_INFO_ << "Using SQLite"; return std::make_shared(metaOptions); } else { - ENGINE_LOG_ERROR << "Invalid dialect in URI: dialect = " << uri_info.dialect_; + LOG_ENGINE_ERROR_ << "Invalid dialect in URI: dialect = " << uri_info.dialect_; throw InvalidArgumentException("URI dialect is not mysql / sqlite"); } } diff --git a/core/src/db/meta/MySQLConnectionPool.cpp b/core/src/db/meta/MySQLConnectionPool.cpp index 3320f5168fe59a9eea15508f6e2ee9585fe75bd1..c28da4b097ef21cbe29f6b2f44e7dcc89c5a7cb1 100644 --- a/core/src/db/meta/MySQLConnectionPool.cpp +++ b/core/src/db/meta/MySQLConnectionPool.cpp @@ -36,8 +36,8 @@ void MySQLConnectionPool::release(const mysqlpp::Connection* pc) { mysqlpp::ConnectionPool::release(pc); if (conns_in_use_ <= 0) { - ENGINE_LOG_WARNING << "MySQLConnetionPool::release: conns_in_use_ is less than zero. conns_in_use_ = " - << conns_in_use_; + LOG_ENGINE_WARNING_ << "MySQLConnetionPool::release: conns_in_use_ is less than zero. conns_in_use_ = " + << conns_in_use_; } else { --conns_in_use_; } @@ -70,8 +70,8 @@ MySQLConnectionPool::create() { user_.empty() ? 0 : user_.c_str(), password_.empty() ? 0 : password_.c_str(), port_); return conn; } catch (const mysqlpp::ConnectionFailed& er) { - ENGINE_LOG_ERROR << "Failed to connect to database server" - << ": " << er.what(); + LOG_ENGINE_ERROR_ << "Failed to connect to database server" + << ": " << er.what(); return nullptr; } } diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index 1cb84952732ae185cca99548df8fa7e7838cb1a7..d974cc7b7407e3c18b6ccfb2a5b6431c87cfda8b 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -49,12 +49,12 @@ namespace { Status HandleException(const std::string& desc, const char* what = nullptr) { if (what == nullptr) { - ENGINE_LOG_ERROR << desc; + LOG_ENGINE_ERROR_ << desc; return Status(DB_META_TRANSACTION_FAILED, desc); } std::string msg = desc + ":" + what; - ENGINE_LOG_ERROR << msg; + LOG_ENGINE_ERROR_ << msg; return Status(DB_META_TRANSACTION_FAILED, msg); } @@ -236,7 +236,7 @@ MySQLMetaImpl::ValidateMetaSchema() { exist_fields.push_back(MetaField(name, type, "")); } } catch (std::exception& e) { - ENGINE_LOG_DEBUG << "Meta collection '" << schema.name() << "' not exist and will be created"; + LOG_ENGINE_DEBUG_ << "Meta collection '" << schema.name() << "' not exist and will be created"; } if (exist_fields.empty()) { @@ -270,7 +270,7 @@ MySQLMetaImpl::Initialize() { fiu_do_on("MySQLMetaImpl.Initialize.fail_create_directory", ret = false); if (!ret) { std::string msg = "Failed to create db directory " + options_.path_; - ENGINE_LOG_ERROR << msg; + LOG_ENGINE_ERROR_ << msg; throw Exception(DB_META_TRANSACTION_FAILED, msg); } } @@ -282,13 +282,13 @@ MySQLMetaImpl::Initialize() { auto status = utils::ParseMetaUri(uri, uri_info); if (!status.ok()) { std::string msg = "Wrong URI format: " + uri; - ENGINE_LOG_ERROR << msg; + LOG_ENGINE_ERROR_ << msg; throw Exception(DB_INVALID_META_URI, msg); } if (strcasecmp(uri_info.dialect_.c_str(), "mysql") != 0) { std::string msg = "URI's dialect is not MySQL"; - ENGINE_LOG_ERROR << msg; + LOG_ENGINE_ERROR_ << msg; throw Exception(DB_INVALID_META_URI, msg); } @@ -302,7 +302,7 @@ MySQLMetaImpl::Initialize() { mysql_connection_pool_ = std::make_shared( uri_info.db_name_, uri_info.username_, uri_info.password_, uri_info.host_, port, max_pool_size); - ENGINE_LOG_DEBUG << "MySQL connection pool: maximum pool size = " << std::to_string(max_pool_size); + LOG_ENGINE_DEBUG_ << "MySQL connection pool: maximum pool size = " << std::to_string(max_pool_size); // step 4: validate to avoid open old version schema ValidateMetaSchema(); @@ -317,7 +317,7 @@ MySQLMetaImpl::Initialize() { if (connectionPtr == nullptr) { std::string msg = "Failed to connect MySQL meta server: " + uri; - ENGINE_LOG_ERROR << msg; + LOG_ENGINE_ERROR_ << msg; throw Exception(DB_INVALID_META_URI, msg); } @@ -326,7 +326,7 @@ MySQLMetaImpl::Initialize() { if (!is_thread_aware) { std::string msg = "Failed to initialize MySQL meta backend: MySQL client component wasn't built with thread awareness"; - ENGINE_LOG_ERROR << msg; + LOG_ENGINE_ERROR_ << msg; throw Exception(DB_INVALID_META_URI, msg); } @@ -335,13 +335,13 @@ MySQLMetaImpl::Initialize() { InitializeQuery << "CREATE TABLE IF NOT EXISTS " << TABLES_SCHEMA.name() << " (" << TABLES_SCHEMA.ToString() + ");"; - ENGINE_LOG_DEBUG << "Initialize: " << InitializeQuery.str(); + LOG_ENGINE_DEBUG_ << "Initialize: " << InitializeQuery.str(); bool initialize_query_exec = InitializeQuery.exec(); fiu_do_on("MySQLMetaImpl.Initialize.fail_create_collection_scheme", initialize_query_exec = false); if (!initialize_query_exec) { std::string msg = "Failed to create meta collection 'Tables' in MySQL"; - ENGINE_LOG_ERROR << msg; + LOG_ENGINE_ERROR_ << msg; throw Exception(DB_META_TRANSACTION_FAILED, msg); } @@ -349,13 +349,13 @@ MySQLMetaImpl::Initialize() { InitializeQuery << "CREATE TABLE IF NOT EXISTS " << TABLEFILES_SCHEMA.name() << " (" << TABLEFILES_SCHEMA.ToString() + ");"; - ENGINE_LOG_DEBUG << "Initialize: " << InitializeQuery.str(); + LOG_ENGINE_DEBUG_ << "Initialize: " << InitializeQuery.str(); initialize_query_exec = InitializeQuery.exec(); fiu_do_on("MySQLMetaImpl.Initialize.fail_create_collection_files", initialize_query_exec = false); if (!initialize_query_exec) { std::string msg = "Failed to create meta collection 'TableFiles' in MySQL"; - ENGINE_LOG_ERROR << msg; + LOG_ENGINE_ERROR_ << msg; throw Exception(DB_META_TRANSACTION_FAILED, msg); } @@ -363,12 +363,12 @@ MySQLMetaImpl::Initialize() { InitializeQuery << "CREATE TABLE IF NOT EXISTS " << ENVIRONMENT_SCHEMA.name() << " (" << ENVIRONMENT_SCHEMA.ToString() + ");"; - ENGINE_LOG_DEBUG << "Initialize: " << InitializeQuery.str(); + LOG_ENGINE_DEBUG_ << "Initialize: " << InitializeQuery.str(); initialize_query_exec = InitializeQuery.exec(); if (!initialize_query_exec) { std::string msg = "Failed to create meta table 'Environment' in MySQL"; - ENGINE_LOG_ERROR << msg; + LOG_ENGINE_ERROR_ << msg; throw Exception(DB_META_TRANSACTION_FAILED, msg); } @@ -397,7 +397,7 @@ MySQLMetaImpl::CreateCollection(CollectionSchema& collection_schema) { statement << "SELECT state FROM " << META_TABLES << " WHERE table_id = " << mysqlpp::quote << collection_schema.collection_id_ << ";"; - ENGINE_LOG_DEBUG << "CreateCollection: " << statement.str(); + LOG_ENGINE_DEBUG_ << "CreateCollection: " << statement.str(); mysqlpp::StoreQueryResult res = statement.store(); @@ -437,7 +437,7 @@ MySQLMetaImpl::CreateCollection(CollectionSchema& collection_schema) { << metric_type << ", " << mysqlpp::quote << owner_collection << ", " << mysqlpp::quote << partition_tag << ", " << mysqlpp::quote << version << ", " << flush_lsn << ");"; - ENGINE_LOG_DEBUG << "CreateCollection: " << statement.str(); + LOG_ENGINE_DEBUG_ << "CreateCollection: " << statement.str(); if (mysqlpp::SimpleResult res = statement.execute()) { collection_schema.id_ = res.insert_id(); // Might need to use SELECT LAST_INSERT_ID()? @@ -448,7 +448,7 @@ MySQLMetaImpl::CreateCollection(CollectionSchema& collection_schema) { } } // Scoped Connection - ENGINE_LOG_DEBUG << "Successfully create collection: " << collection_schema.collection_id_; + LOG_ENGINE_DEBUG_ << "Successfully create collection: " << collection_schema.collection_id_; return utils::CreateCollectionPath(options_, collection_schema.collection_id_); } catch (std::exception& e) { return HandleException("Failed to create collection", e.what()); @@ -477,7 +477,7 @@ MySQLMetaImpl::DescribeCollection(CollectionSchema& collection_schema) { << collection_schema.collection_id_ << " AND state <> " << std::to_string(CollectionSchema::TO_DELETE) << ";"; - ENGINE_LOG_DEBUG << "DescribeCollection: " << statement.str(); + LOG_ENGINE_DEBUG_ << "DescribeCollection: " << statement.str(); res = statement.store(); } // Scoped Connection @@ -531,7 +531,7 @@ MySQLMetaImpl::HasCollection(const std::string& collection_id, bool& has_or_not) << " AS " << mysqlpp::quote << "check" << ";"; - ENGINE_LOG_DEBUG << "HasCollection: " << HasCollectionQuery.str(); + LOG_ENGINE_DEBUG_ << "HasCollection: " << HasCollectionQuery.str(); res = HasCollectionQuery.store(); } // Scoped Connection @@ -566,7 +566,7 @@ MySQLMetaImpl::AllCollections(std::vector& collection_schema_a << " FROM " << META_TABLES << " WHERE state <> " << std::to_string(CollectionSchema::TO_DELETE) << " AND owner_table = \"\";"; - ENGINE_LOG_DEBUG << "AllCollections: " << statement.str(); + LOG_ENGINE_DEBUG_ << "AllCollections: " << statement.str(); res = statement.store(); } // Scoped Connection @@ -615,7 +615,7 @@ MySQLMetaImpl::DropCollection(const std::string& collection_id) { statement << "UPDATE " << META_TABLES << " SET state = " << std::to_string(CollectionSchema::TO_DELETE) << " WHERE table_id = " << mysqlpp::quote << collection_id << ";"; - ENGINE_LOG_DEBUG << "DeleteCollection: " << statement.str(); + LOG_ENGINE_DEBUG_ << "DeleteCollection: " << statement.str(); if (!statement.exec()) { return HandleException("Failed to drop collection", statement.error()); @@ -628,7 +628,7 @@ MySQLMetaImpl::DropCollection(const std::string& collection_id) { DeleteCollectionFiles(collection_id); } - ENGINE_LOG_DEBUG << "Successfully delete collection: " << collection_id; + LOG_ENGINE_DEBUG_ << "Successfully delete collection: " << collection_id; } catch (std::exception& e) { return HandleException("Failed to drop collection", e.what()); } @@ -659,14 +659,14 @@ MySQLMetaImpl::DeleteCollectionFiles(const std::string& collection_id) { << " WHERE table_id = " << mysqlpp::quote << collection_id << " AND file_type <> " << std::to_string(SegmentSchema::TO_DELETE) << ";"; - ENGINE_LOG_DEBUG << "DeleteCollectionFiles: " << statement.str(); + LOG_ENGINE_DEBUG_ << "DeleteCollectionFiles: " << statement.str(); if (!statement.exec()) { return HandleException("Failed to delete colletion files", statement.error()); } } // Scoped Connection - ENGINE_LOG_DEBUG << "Successfully delete collection files from " << collection_id; + LOG_ENGINE_DEBUG_ << "Successfully delete collection files from " << collection_id; } catch (std::exception& e) { return HandleException("Failed to delete colletion files", e.what()); } @@ -733,7 +733,7 @@ MySQLMetaImpl::CreateCollectionFile(SegmentSchema& file_schema) { << mysqlpp::quote << file_id << ", " << file_type << ", " << file_size << ", " << row_count << ", " << updated_time << ", " << created_on << ", " << date << ", " << flush_lsn << ");"; - ENGINE_LOG_DEBUG << "CreateCollectionFile: " << statement.str(); + LOG_ENGINE_DEBUG_ << "CreateCollectionFile: " << statement.str(); if (mysqlpp::SimpleResult res = statement.execute()) { file_schema.id_ = res.insert_id(); // Might need to use SELECT LAST_INSERT_ID()? @@ -744,7 +744,7 @@ MySQLMetaImpl::CreateCollectionFile(SegmentSchema& file_schema) { } } // Scoped Connection - ENGINE_LOG_DEBUG << "Successfully create collection file, file id = " << file_schema.file_id_; + LOG_ENGINE_DEBUG_ << "Successfully create collection file, file id = " << file_schema.file_id_; return utils::CreateCollectionFilePath(options_, file_schema); } catch (std::exception& e) { return HandleException("Failed to create collection file", e.what()); @@ -784,7 +784,7 @@ MySQLMetaImpl::GetCollectionFiles(const std::string& collection_id, const std::v << idStr << ")" << " AND file_type <> " << std::to_string(SegmentSchema::TO_DELETE) << ";"; - ENGINE_LOG_DEBUG << "GetCollectionFiles: " << statement.str(); + LOG_ENGINE_DEBUG_ << "GetCollectionFiles: " << statement.str(); res = statement.store(); } // Scoped Connection @@ -815,7 +815,7 @@ MySQLMetaImpl::GetCollectionFiles(const std::string& collection_id, const std::v collection_files.emplace_back(file_schema); } - ENGINE_LOG_DEBUG << "Get collection files by id"; + LOG_ENGINE_DEBUG_ << "Get collection files by id"; return ret; } catch (std::exception& e) { return HandleException("Failed to get collection files", e.what()); @@ -840,7 +840,7 @@ MySQLMetaImpl::GetCollectionFilesBySegmentId(const std::string& segment_id, << " FROM " << META_TABLEFILES << " WHERE segment_id = " << mysqlpp::quote << segment_id << " AND file_type <> " << std::to_string(SegmentSchema::TO_DELETE) << ";"; - ENGINE_LOG_DEBUG << "GetCollectionFilesBySegmentId: " << statement.str(); + LOG_ENGINE_DEBUG_ << "GetCollectionFilesBySegmentId: " << statement.str(); res = statement.store(); } // Scoped Connection @@ -875,7 +875,7 @@ MySQLMetaImpl::GetCollectionFilesBySegmentId(const std::string& segment_id, } } - ENGINE_LOG_DEBUG << "Get collection files by segment id"; + LOG_ENGINE_DEBUG_ << "Get collection files by segment id"; return Status::OK(); } catch (std::exception& e) { return HandleException("Failed to get collection files by segment id", e.what()); @@ -903,7 +903,7 @@ MySQLMetaImpl::UpdateCollectionIndex(const std::string& collection_id, const Col << collection_id << " AND state <> " << std::to_string(CollectionSchema::TO_DELETE) << ";"; - ENGINE_LOG_DEBUG << "UpdateCollectionIndex: " << updateCollectionIndexParamQuery.str(); + LOG_ENGINE_DEBUG_ << "UpdateCollectionIndex: " << updateCollectionIndexParamQuery.str(); mysqlpp::StoreQueryResult res = updateCollectionIndexParamQuery.store(); @@ -922,7 +922,7 @@ MySQLMetaImpl::UpdateCollectionIndex(const std::string& collection_id, const Col << index.extra_params_.dump() << " ,metric_type = " << index.metric_type_ << " WHERE table_id = " << mysqlpp::quote << collection_id << ";"; - ENGINE_LOG_DEBUG << "UpdateCollectionIndex: " << updateCollectionIndexParamQuery.str(); + LOG_ENGINE_DEBUG_ << "UpdateCollectionIndex: " << updateCollectionIndexParamQuery.str(); if (!updateCollectionIndexParamQuery.exec()) { return HandleException("Failed to update collection index", @@ -933,7 +933,7 @@ MySQLMetaImpl::UpdateCollectionIndex(const std::string& collection_id, const Col } } // Scoped Connection - ENGINE_LOG_DEBUG << "Successfully update collection index for " << collection_id; + LOG_ENGINE_DEBUG_ << "Successfully update collection index for " << collection_id; } catch (std::exception& e) { return HandleException("Failed to update collection index", e.what()); } @@ -960,14 +960,14 @@ MySQLMetaImpl::UpdateCollectionFlag(const std::string& collection_id, int64_t fl statement << "UPDATE " << META_TABLES << " SET flag = " << flag << " WHERE table_id = " << mysqlpp::quote << collection_id << ";"; - ENGINE_LOG_DEBUG << "UpdateCollectionFlag: " << statement.str(); + LOG_ENGINE_DEBUG_ << "UpdateCollectionFlag: " << statement.str(); if (!statement.exec()) { return HandleException("Failed to update collection flag", statement.error()); } } // Scoped Connection - ENGINE_LOG_DEBUG << "Successfully update collection flag for " << collection_id; + LOG_ENGINE_DEBUG_ << "Successfully update collection flag for " << collection_id; } catch (std::exception& e) { return HandleException("Failed to update collection flag", e.what()); } @@ -991,14 +991,14 @@ MySQLMetaImpl::UpdateCollectionFlushLSN(const std::string& collection_id, uint64 statement << "UPDATE " << META_TABLES << " SET flush_lsn = " << flush_lsn << " WHERE table_id = " << mysqlpp::quote << collection_id << ";"; - ENGINE_LOG_DEBUG << "UpdateCollectionFlushLSN: " << statement.str(); + LOG_ENGINE_DEBUG_ << "UpdateCollectionFlushLSN: " << statement.str(); if (!statement.exec()) { return HandleException("Failed to update collection lsn", statement.error()); } } // Scoped Connection - ENGINE_LOG_DEBUG << "Successfully update collection flush_lsn for " << collection_id; + LOG_ENGINE_DEBUG_ << "Successfully update collection flush_lsn for " << collection_id; } catch (std::exception& e) { return HandleException("Failed to update collection lsn", e.what()); } @@ -1022,7 +1022,7 @@ MySQLMetaImpl::GetCollectionFlushLSN(const std::string& collection_id, uint64_t& statement << "SELECT flush_lsn FROM " << META_TABLES << " WHERE collection_id = " << mysqlpp::quote << collection_id << ";"; - ENGINE_LOG_DEBUG << "GetCollectionFlushLSN: " << statement.str(); + LOG_ENGINE_DEBUG_ << "GetCollectionFlushLSN: " << statement.str(); res = statement.store(); } // Scoped Connection @@ -1061,7 +1061,7 @@ MySQLMetaImpl::UpdateCollectionFile(SegmentSchema& file_schema) { statement << "SELECT state FROM " << META_TABLES << " WHERE table_id = " << mysqlpp::quote << file_schema.collection_id_ << ";"; - ENGINE_LOG_DEBUG << "UpdateCollectionFile: " << statement.str(); + LOG_ENGINE_DEBUG_ << "UpdateCollectionFile: " << statement.str(); mysqlpp::StoreQueryResult res = statement.store(); @@ -1091,16 +1091,16 @@ MySQLMetaImpl::UpdateCollectionFile(SegmentSchema& file_schema) { << " ,updated_time = " << updated_time << " ,created_on = " << created_on << " ,date = " << date << " WHERE id = " << id << ";"; - ENGINE_LOG_DEBUG << "UpdateCollectionFile: " << statement.str(); + LOG_ENGINE_DEBUG_ << "UpdateCollectionFile: " << statement.str(); if (!statement.exec()) { - ENGINE_LOG_DEBUG << "collection_id= " << file_schema.collection_id_ - << " file_id=" << file_schema.file_id_; + LOG_ENGINE_DEBUG_ << "collection_id= " << file_schema.collection_id_ + << " file_id=" << file_schema.file_id_; return HandleException("Failed to update collection file", statement.error()); } } // Scoped Connection - ENGINE_LOG_DEBUG << "Update single collection file, file id: " << file_schema.file_id_; + LOG_ENGINE_DEBUG_ << "Update single collection file, file id: " << file_schema.file_id_; } catch (std::exception& e) { return HandleException("Failed to update collection file", e.what()); } @@ -1127,13 +1127,13 @@ MySQLMetaImpl::UpdateCollectionFilesToIndex(const std::string& collection_id) { << " AND row_count >= " << std::to_string(meta::BUILD_INDEX_THRESHOLD) << " AND file_type = " << std::to_string(SegmentSchema::RAW) << ";"; - ENGINE_LOG_DEBUG << "UpdateCollectionFilesToIndex: " << statement.str(); + LOG_ENGINE_DEBUG_ << "UpdateCollectionFilesToIndex: " << statement.str(); if (!statement.exec()) { return HandleException("Failed to update collection files to index", statement.error()); } - ENGINE_LOG_DEBUG << "Update files to to_index for " << collection_id; + LOG_ENGINE_DEBUG_ << "Update files to to_index for " << collection_id; } catch (std::exception& e) { return HandleException("Failed to update collection files to index", e.what()); } @@ -1170,7 +1170,7 @@ MySQLMetaImpl::UpdateCollectionFiles(SegmentsSchema& files) { << " AS " << mysqlpp::quote << "check" << ";"; - ENGINE_LOG_DEBUG << "UpdateCollectionFiles: " << statement.str(); + LOG_ENGINE_DEBUG_ << "UpdateCollectionFiles: " << statement.str(); mysqlpp::StoreQueryResult res = statement.store(); @@ -1201,7 +1201,7 @@ MySQLMetaImpl::UpdateCollectionFiles(SegmentsSchema& files) { << " ,row_count = " << row_count << " ,updated_time = " << updated_time << " ,created_on = " << created_on << " ,date = " << date << " WHERE id = " << id << ";"; - ENGINE_LOG_DEBUG << "UpdateCollectionFiles: " << statement.str(); + LOG_ENGINE_DEBUG_ << "UpdateCollectionFiles: " << statement.str(); if (!statement.exec()) { return HandleException("Failed to update collection files", statement.error()); @@ -1209,7 +1209,7 @@ MySQLMetaImpl::UpdateCollectionFiles(SegmentsSchema& files) { } } // Scoped Connection - ENGINE_LOG_DEBUG << "Update " << files.size() << " collection files"; + LOG_ENGINE_DEBUG_ << "Update " << files.size() << " collection files"; } catch (std::exception& e) { return HandleException("Failed to update collection files", e.what()); } @@ -1238,17 +1238,17 @@ MySQLMetaImpl::UpdateCollectionFilesRowCount(SegmentsSchema& files) { statement << "UPDATE " << META_TABLEFILES << " SET row_count = " << row_count << " , updated_time = " << updated_time << " WHERE file_id = " << file.file_id_ << ";"; - ENGINE_LOG_DEBUG << "UpdateCollectionFilesRowCount: " << statement.str(); + LOG_ENGINE_DEBUG_ << "UpdateCollectionFilesRowCount: " << statement.str(); if (!statement.exec()) { return HandleException("Failed to update collection files row count", statement.error()); } - ENGINE_LOG_DEBUG << "Update file " << file.file_id_ << " row count to " << file.row_count_; + LOG_ENGINE_DEBUG_ << "Update file " << file.file_id_ << " row count to " << file.row_count_; } } // Scoped Connection - ENGINE_LOG_DEBUG << "Update " << files.size() << " collection files"; + LOG_ENGINE_DEBUG_ << "Update " << files.size() << " collection files"; } catch (std::exception& e) { return HandleException("Failed to update collection files row count", e.what()); } @@ -1276,7 +1276,7 @@ MySQLMetaImpl::DescribeCollectionIndex(const std::string& collection_id, Collect << " FROM " << META_TABLES << " WHERE table_id = " << mysqlpp::quote << collection_id << " AND state <> " << std::to_string(CollectionSchema::TO_DELETE) << ";"; - ENGINE_LOG_DEBUG << "DescribeCollectionIndex: " << statement.str(); + LOG_ENGINE_DEBUG_ << "DescribeCollectionIndex: " << statement.str(); mysqlpp::StoreQueryResult res = statement.store(); @@ -1321,7 +1321,7 @@ MySQLMetaImpl::DropCollectionIndex(const std::string& collection_id) { << " ,updated_time = " << utils::GetMicroSecTimeStamp() << " WHERE table_id = " << mysqlpp::quote << collection_id << " AND file_type = " << std::to_string(SegmentSchema::INDEX) << ";"; - ENGINE_LOG_DEBUG << "DropCollectionIndex: " << statement.str(); + LOG_ENGINE_DEBUG_ << "DropCollectionIndex: " << statement.str(); if (!statement.exec()) { return HandleException("Failed to drop collection index", statement.error()); @@ -1332,7 +1332,7 @@ MySQLMetaImpl::DropCollectionIndex(const std::string& collection_id) { << " ,updated_time = " << utils::GetMicroSecTimeStamp() << " WHERE table_id = " << mysqlpp::quote << collection_id << " AND file_type = " << std::to_string(SegmentSchema::BACKUP) << ";"; - ENGINE_LOG_DEBUG << "DropCollectionIndex: " << statement.str(); + LOG_ENGINE_DEBUG_ << "DropCollectionIndex: " << statement.str(); if (!statement.exec()) { return HandleException("Failed to drop collection index", statement.error()); @@ -1348,14 +1348,14 @@ MySQLMetaImpl::DropCollectionIndex(const std::string& collection_id) { << " , index_params = '{}'" << " WHERE table_id = " << mysqlpp::quote << collection_id << ";"; - ENGINE_LOG_DEBUG << "DropCollectionIndex: " << statement.str(); + LOG_ENGINE_DEBUG_ << "DropCollectionIndex: " << statement.str(); if (!statement.exec()) { return HandleException("Failed to drop collection index", statement.error()); } } // Scoped Connection - ENGINE_LOG_DEBUG << "Successfully drop collection index for " << collection_id; + LOG_ENGINE_DEBUG_ << "Successfully drop collection index for " << collection_id; } catch (std::exception& e) { return HandleException("Failed to drop collection index", e.what()); } @@ -1442,7 +1442,7 @@ MySQLMetaImpl::ShowPartitions(const std::string& collection_id, << " WHERE owner_table = " << mysqlpp::quote << collection_id << " AND state <> " << std::to_string(CollectionSchema::TO_DELETE) << ";"; - ENGINE_LOG_DEBUG << "AllCollections: " << statement.str(); + LOG_ENGINE_DEBUG_ << "AllCollections: " << statement.str(); res = statement.store(); } // Scoped Connection @@ -1498,7 +1498,7 @@ MySQLMetaImpl::GetPartitionName(const std::string& collection_id, const std::str << collection_id << " AND partition_tag = " << mysqlpp::quote << valid_tag << " AND state <> " << std::to_string(CollectionSchema::TO_DELETE) << ";"; - ENGINE_LOG_DEBUG << "AllCollections: " << statement.str(); + LOG_ENGINE_DEBUG_ << "AllCollections: " << statement.str(); res = statement.store(); } // Scoped Connection @@ -1543,7 +1543,7 @@ MySQLMetaImpl::FilesToSearch(const std::string& collection_id, SegmentsSchema& f << " OR file_type = " << std::to_string(SegmentSchema::TO_INDEX) << " OR file_type = " << std::to_string(SegmentSchema::INDEX) << ");"; - ENGINE_LOG_DEBUG << "FilesToSearch: " << statement.str(); + LOG_ENGINE_DEBUG_ << "FilesToSearch: " << statement.str(); res = statement.store(); } // Scoped Connection @@ -1581,7 +1581,7 @@ MySQLMetaImpl::FilesToSearch(const std::string& collection_id, SegmentsSchema& f } if (res.size() > 0) { - ENGINE_LOG_DEBUG << "Collect " << res.size() << " to-search files"; + LOG_ENGINE_DEBUG_ << "Collect " << res.size() << " to-search files"; } return ret; } catch (std::exception& e) { @@ -1621,7 +1621,7 @@ MySQLMetaImpl::FilesToMerge(const std::string& collection_id, SegmentsSchema& fi << " FROM " << META_TABLEFILES << " WHERE table_id = " << mysqlpp::quote << collection_id << " AND file_type = " << std::to_string(SegmentSchema::RAW) << " ORDER BY row_count DESC;"; - ENGINE_LOG_DEBUG << "FilesToMerge: " << statement.str(); + LOG_ENGINE_DEBUG_ << "FilesToMerge: " << statement.str(); res = statement.store(); } // Scoped Connection @@ -1659,7 +1659,7 @@ MySQLMetaImpl::FilesToMerge(const std::string& collection_id, SegmentsSchema& fi } if (to_merge_files > 0) { - ENGINE_LOG_TRACE << "Collect " << to_merge_files << " to-merge files"; + LOG_ENGINE_TRACE_ << "Collect " << to_merge_files << " to-merge files"; } return ret; } catch (std::exception& e) { @@ -1690,7 +1690,7 @@ MySQLMetaImpl::FilesToIndex(SegmentsSchema& files) { << " FROM " << META_TABLEFILES << " WHERE file_type = " << std::to_string(SegmentSchema::TO_INDEX) << ";"; - ENGINE_LOG_DEBUG << "FilesToIndex: " << statement.str(); + LOG_ENGINE_DEBUG_ << "FilesToIndex: " << statement.str(); res = statement.store(); } // Scoped Connection @@ -1734,7 +1734,7 @@ MySQLMetaImpl::FilesToIndex(SegmentsSchema& files) { } if (res.size() > 0) { - ENGINE_LOG_DEBUG << "Collect " << res.size() << " to-index files"; + LOG_ENGINE_DEBUG_ << "Collect " << res.size() << " to-index files"; } return ret; } catch (std::exception& e) { @@ -1780,7 +1780,7 @@ MySQLMetaImpl::FilesByType(const std::string& collection_id, const std::vector& ids, SegmentsSchema& files) << " OR file_type = " << std::to_string(SegmentSchema::TO_INDEX) << " OR file_type = " << std::to_string(SegmentSchema::INDEX) << ");"; - ENGINE_LOG_DEBUG << "FilesToSearch: " << statement.str(); + LOG_ENGINE_DEBUG_ << "FilesToSearch: " << statement.str(); res = statement.store(); } // Scoped Connection @@ -1971,9 +1971,9 @@ MySQLMetaImpl::FilesByID(const std::vector& ids, SegmentsSchema& files) } if (files.empty()) { - ENGINE_LOG_ERROR << "No file to search in file id list"; + LOG_ENGINE_ERROR_ << "No file to search in file id list"; } else { - ENGINE_LOG_DEBUG << "Collect " << files.size() << " files by id"; + LOG_ENGINE_DEBUG_ << "Collect " << files.size() << " files by id"; } return ret; @@ -2013,13 +2013,13 @@ MySQLMetaImpl::Archive() { << std::to_string(now - usecs) << " AND file_type <> " << std::to_string(SegmentSchema::TO_DELETE) << ";"; - ENGINE_LOG_DEBUG << "Archive: " << statement.str(); + LOG_ENGINE_DEBUG_ << "Archive: " << statement.str(); if (!statement.exec()) { return HandleException("Failed to archive", statement.error()); } - ENGINE_LOG_DEBUG << "Archive old files"; + LOG_ENGINE_DEBUG_ << "Archive old files"; } catch (std::exception& e) { return HandleException("Failed to archive", e.what()); } @@ -2031,7 +2031,7 @@ MySQLMetaImpl::Archive() { auto to_delete = (sum - limit * G); DiscardFiles(to_delete); - ENGINE_LOG_DEBUG << "Archive files to free disk"; + LOG_ENGINE_DEBUG_ << "Archive files to free disk"; } } @@ -2059,7 +2059,7 @@ MySQLMetaImpl::Size(uint64_t& result) { << " FROM " << META_TABLEFILES << " WHERE file_type <> " << std::to_string(SegmentSchema::TO_DELETE) << ";"; - ENGINE_LOG_DEBUG << "Size: " << statement.str(); + LOG_ENGINE_DEBUG_ << "Size: " << statement.str(); res = statement.store(); } // Scoped Connection @@ -2094,17 +2094,17 @@ MySQLMetaImpl::CleanUpShadowFiles() { << " WHERE table_schema = " << mysqlpp::quote << mysql_connection_pool_->getDB() << " AND table_name = " << mysqlpp::quote << META_TABLEFILES << ";"; - ENGINE_LOG_DEBUG << "CleanUp: " << statement.str(); + LOG_ENGINE_DEBUG_ << "CleanUp: " << statement.str(); mysqlpp::StoreQueryResult res = statement.store(); if (!res.empty()) { - ENGINE_LOG_DEBUG << "Remove collection file type as NEW"; + LOG_ENGINE_DEBUG_ << "Remove collection file type as NEW"; statement << "DELETE FROM " << META_TABLEFILES << " WHERE file_type IN (" << std::to_string(SegmentSchema::NEW) << "," << std::to_string(SegmentSchema::NEW_MERGE) << "," << std::to_string(SegmentSchema::NEW_INDEX) << ");"; - ENGINE_LOG_DEBUG << "CleanUp: " << statement.str(); + LOG_ENGINE_DEBUG_ << "CleanUp: " << statement.str(); if (!statement.exec()) { return HandleException("Failed to clean shadow files", statement.error()); @@ -2112,7 +2112,7 @@ MySQLMetaImpl::CleanUpShadowFiles() { } if (res.size() > 0) { - ENGINE_LOG_DEBUG << "Clean " << res.size() << " files"; + LOG_ENGINE_DEBUG_ << "Clean " << res.size() << " files"; } } catch (std::exception& e) { return HandleException("Failed to clean shadow files", e.what()); @@ -2148,7 +2148,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/) << std::to_string(SegmentSchema::TO_DELETE) << "," << std::to_string(SegmentSchema::BACKUP) << ")" << " AND updated_time < " << std::to_string(now - seconds * US_PS) << ";"; - ENGINE_LOG_DEBUG << "CleanUpFilesWithTTL: " << statement.str(); + LOG_ENGINE_DEBUG_ << "CleanUpFilesWithTTL: " << statement.str(); mysqlpp::StoreQueryResult res = statement.store(); @@ -2167,8 +2167,8 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/) // check if the file can be deleted if (OngoingFileChecker::GetInstance().IsIgnored(collection_file)) { - ENGINE_LOG_DEBUG << "File:" << collection_file.file_id_ - << " currently is in use, not able to delete now"; + LOG_ENGINE_DEBUG_ << "File:" << collection_file.file_id_ + << " currently is in use, not able to delete now"; continue; // ignore this file, don't delete it } @@ -2180,8 +2180,8 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/) if (collection_file.file_type_ == (int)SegmentSchema::TO_DELETE) { // delete file from disk storage utils::DeleteCollectionFilePath(options_, collection_file); - ENGINE_LOG_DEBUG << "Remove file id:" << collection_file.id_ - << " location:" << collection_file.location_; + LOG_ENGINE_DEBUG_ << "Remove file id:" << collection_file.id_ + << " location:" << collection_file.location_; delete_ids.emplace_back(std::to_string(collection_file.id_)); collection_ids.insert(collection_file.collection_id_); @@ -2202,7 +2202,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/) idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); // remove the last " OR " statement << "DELETE FROM " << META_TABLEFILES << " WHERE " << idsToDeleteStr << ";"; - ENGINE_LOG_DEBUG << "CleanUpFilesWithTTL: " << statement.str(); + LOG_ENGINE_DEBUG_ << "CleanUpFilesWithTTL: " << statement.str(); if (!statement.exec()) { return HandleException("Failed to clean up with ttl", statement.error()); @@ -2210,7 +2210,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/) } if (clean_files > 0) { - ENGINE_LOG_DEBUG << "Clean " << clean_files << " files expired in " << seconds << " seconds"; + LOG_ENGINE_DEBUG_ << "Clean " << clean_files << " files expired in " << seconds << " seconds"; } } // Scoped Connection } catch (std::exception& e) { @@ -2238,7 +2238,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/) << " FROM " << META_TABLES << " WHERE state = " << std::to_string(CollectionSchema::TO_DELETE) << ";"; - ENGINE_LOG_DEBUG << "CleanUpFilesWithTTL: " << statement.str(); + LOG_ENGINE_DEBUG_ << "CleanUpFilesWithTTL: " << statement.str(); mysqlpp::StoreQueryResult res = statement.store(); @@ -2258,7 +2258,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/) idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); // remove the last " OR " statement << "DELETE FROM " << META_TABLES << " WHERE " << idsToDeleteStr << ";"; - ENGINE_LOG_DEBUG << "CleanUpFilesWithTTL: " << statement.str(); + LOG_ENGINE_DEBUG_ << "CleanUpFilesWithTTL: " << statement.str(); if (!statement.exec()) { return HandleException("Failed to clean up with ttl", statement.error()); @@ -2266,7 +2266,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/) } if (remove_collections > 0) { - ENGINE_LOG_DEBUG << "Remove " << remove_collections << " collections from meta"; + LOG_ENGINE_DEBUG_ << "Remove " << remove_collections << " collections from meta"; } } // Scoped Connection } catch (std::exception& e) { @@ -2296,7 +2296,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/) << " FROM " << META_TABLEFILES << " WHERE table_id = " << mysqlpp::quote << collection_id << ";"; - ENGINE_LOG_DEBUG << "CleanUpFilesWithTTL: " << statement.str(); + LOG_ENGINE_DEBUG_ << "CleanUpFilesWithTTL: " << statement.str(); mysqlpp::StoreQueryResult res = statement.store(); @@ -2306,7 +2306,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/) } if (collection_ids.size() > 0) { - ENGINE_LOG_DEBUG << "Remove " << collection_ids.size() << " collections folder"; + LOG_ENGINE_DEBUG_ << "Remove " << collection_ids.size() << " collections folder"; } } } catch (std::exception& e) { @@ -2337,7 +2337,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/) << " FROM " << META_TABLEFILES << " WHERE segment_id = " << mysqlpp::quote << segment_id.first << ";"; - ENGINE_LOG_DEBUG << "CleanUpFilesWithTTL: " << statement.str(); + LOG_ENGINE_DEBUG_ << "CleanUpFilesWithTTL: " << statement.str(); mysqlpp::StoreQueryResult res = statement.store(); @@ -2345,13 +2345,13 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/) utils::DeleteSegment(options_, segment_id.second); std::string segment_dir; utils::GetParentPath(segment_id.second.location_, segment_dir); - ENGINE_LOG_DEBUG << "Remove segment directory: " << segment_dir; + LOG_ENGINE_DEBUG_ << "Remove segment directory: " << segment_dir; ++remove_segments; } } if (remove_segments > 0) { - ENGINE_LOG_DEBUG << "Remove " << remove_segments << " segments folder"; + LOG_ENGINE_DEBUG_ << "Remove " << remove_segments << " segments folder"; } } } catch (std::exception& e) { @@ -2392,7 +2392,7 @@ MySQLMetaImpl::Count(const std::string& collection_id, uint64_t& result) { << " OR file_type = " << std::to_string(SegmentSchema::TO_INDEX) << " OR file_type = " << std::to_string(SegmentSchema::INDEX) << ");"; - ENGINE_LOG_DEBUG << "Count: " << statement.str(); + LOG_ENGINE_DEBUG_ << "Count: " << statement.str(); res = statement.store(); } // Scoped Connection @@ -2412,7 +2412,7 @@ MySQLMetaImpl::Count(const std::string& collection_id, uint64_t& result) { Status MySQLMetaImpl::DropAll() { try { - ENGINE_LOG_DEBUG << "Drop all mysql meta"; + LOG_ENGINE_DEBUG_ << "Drop all mysql meta"; mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); bool is_null_connection = (connectionPtr == nullptr); @@ -2425,7 +2425,7 @@ MySQLMetaImpl::DropAll() { mysqlpp::Query statement = connectionPtr->query(); statement << "DROP TABLE IF EXISTS " << TABLES_SCHEMA.name() << ", " << TABLEFILES_SCHEMA.name() << ";"; - ENGINE_LOG_DEBUG << "DropAll: " << statement.str(); + LOG_ENGINE_DEBUG_ << "DropAll: " << statement.str(); if (statement.exec()) { return Status::OK(); @@ -2441,7 +2441,7 @@ MySQLMetaImpl::DiscardFiles(int64_t to_discard_size) { if (to_discard_size <= 0) { return Status::OK(); } - ENGINE_LOG_DEBUG << "About to discard size=" << to_discard_size; + LOG_ENGINE_DEBUG_ << "About to discard size=" << to_discard_size; try { server::MetricCollector metric; @@ -2462,7 +2462,7 @@ MySQLMetaImpl::DiscardFiles(int64_t to_discard_size) { << std::to_string(SegmentSchema::TO_DELETE) << " ORDER BY id ASC " << " LIMIT 10;"; - ENGINE_LOG_DEBUG << "DiscardFiles: " << statement.str(); + LOG_ENGINE_DEBUG_ << "DiscardFiles: " << statement.str(); mysqlpp::StoreQueryResult res = statement.store(); if (res.num_rows() == 0) { @@ -2478,8 +2478,8 @@ MySQLMetaImpl::DiscardFiles(int64_t to_discard_size) { collection_file.id_ = resRow["id"]; collection_file.file_size_ = resRow["file_size"]; idsToDiscardSS << "id = " << std::to_string(collection_file.id_) << " OR "; - ENGINE_LOG_DEBUG << "Discard file id=" << collection_file.file_id_ - << " file size=" << collection_file.file_size_; + LOG_ENGINE_DEBUG_ << "Discard file id=" << collection_file.file_id_ + << " file size=" << collection_file.file_size_; to_discard_size -= collection_file.file_size_; } @@ -2490,7 +2490,7 @@ MySQLMetaImpl::DiscardFiles(int64_t to_discard_size) { << " ,updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " WHERE " << idsToDiscardStr << ";"; - ENGINE_LOG_DEBUG << "DiscardFiles: " << statement.str(); + LOG_ENGINE_DEBUG_ << "DiscardFiles: " << statement.str(); status = statement.exec(); if (!status) { @@ -2528,7 +2528,7 @@ MySQLMetaImpl::SetGlobalLastLSN(uint64_t lsn) { if (first_create) { // first time to get global lsn mysqlpp::Query statement = connectionPtr->query(); statement << "INSERT INTO " << META_ENVIRONMENT << " VALUES(" << lsn << ");"; - ENGINE_LOG_DEBUG << "SetGlobalLastLSN: " << statement.str(); + LOG_ENGINE_DEBUG_ << "SetGlobalLastLSN: " << statement.str(); if (!statement.exec()) { return HandleException("QUERY ERROR WHEN SET GLOBAL LSN", statement.error()); @@ -2536,7 +2536,7 @@ MySQLMetaImpl::SetGlobalLastLSN(uint64_t lsn) { } else { mysqlpp::Query statement = connectionPtr->query(); statement << "UPDATE " << META_ENVIRONMENT << " SET global_lsn = " << lsn << ";"; - ENGINE_LOG_DEBUG << "SetGlobalLastLSN: " << statement.str(); + LOG_ENGINE_DEBUG_ << "SetGlobalLastLSN: " << statement.str(); if (!statement.exec()) { return HandleException("Failed to set global lsn", statement.error()); @@ -2544,7 +2544,7 @@ MySQLMetaImpl::SetGlobalLastLSN(uint64_t lsn) { } } // Scoped Connection - ENGINE_LOG_DEBUG << "Successfully update global_lsn: " << lsn; + LOG_ENGINE_DEBUG_ << "Successfully update global_lsn: " << lsn; } catch (std::exception& e) { return HandleException("Failed to set global lsn", e.what()); } @@ -2567,7 +2567,7 @@ MySQLMetaImpl::GetGlobalLastLSN(uint64_t& lsn) { mysqlpp::Query statement = connectionPtr->query(); statement << "SELECT global_lsn FROM " << META_ENVIRONMENT << ";"; - ENGINE_LOG_DEBUG << "GetGlobalLastLSN: " << statement.str(); + LOG_ENGINE_DEBUG_ << "GetGlobalLastLSN: " << statement.str(); res = statement.store(); } // Scoped Connection diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index 06542babbf5ba75a5f830ab25012b014d8d8bd71..708b542e25cc97ab2ec28f469937f68b734c1acf 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -46,11 +46,11 @@ namespace { Status HandleException(const std::string& desc, const char* what = nullptr) { if (what == nullptr) { - ENGINE_LOG_ERROR << desc; + LOG_ENGINE_ERROR_ << desc; return Status(DB_META_TRANSACTION_FAILED, desc); } else { std::string msg = desc + ":" + what; - ENGINE_LOG_ERROR << msg; + LOG_ENGINE_ERROR_ << msg; return Status(DB_META_TRANSACTION_FAILED, msg); } } @@ -145,7 +145,7 @@ SqliteMetaImpl::Initialize() { fiu_do_on("SqliteMetaImpl.Initialize.fail_create_directory", ret = false); if (!ret) { std::string msg = "Failed to create db directory " + options_.path_; - ENGINE_LOG_ERROR << msg; + LOG_ENGINE_ERROR_ << msg; throw Exception(DB_INVALID_PATH, msg); } } @@ -198,7 +198,7 @@ SqliteMetaImpl::CreateCollection(CollectionSchema& collection_schema) { return HandleException("Encounter exception when create collection", e.what()); } - ENGINE_LOG_DEBUG << "Successfully create collection: " << collection_schema.collection_id_; + LOG_ENGINE_DEBUG_ << "Successfully create collection: " << collection_schema.collection_id_; return utils::CreateCollectionPath(options_, collection_schema.collection_id_); } catch (std::exception& e) { @@ -319,7 +319,7 @@ SqliteMetaImpl::DropCollection(const std::string& collection_id) { set(c(&CollectionSchema::state_) = (int)CollectionSchema::TO_DELETE), where(c(&CollectionSchema::collection_id_) == collection_id and c(&CollectionSchema::state_) != (int)CollectionSchema::TO_DELETE)); - ENGINE_LOG_DEBUG << "Successfully delete collection, collection id = " << collection_id; + LOG_ENGINE_DEBUG_ << "Successfully delete collection, collection id = " << collection_id; } catch (std::exception& e) { return HandleException("Encounter exception when delete collection", e.what()); } @@ -343,7 +343,7 @@ SqliteMetaImpl::DeleteCollectionFiles(const std::string& collection_id) { where(c(&SegmentSchema::collection_id_) == collection_id and c(&SegmentSchema::file_type_) != (int)SegmentSchema::TO_DELETE)); - ENGINE_LOG_DEBUG << "Successfully delete collection files, collection id = " << collection_id; + LOG_ENGINE_DEBUG_ << "Successfully delete collection files, collection id = " << collection_id; } catch (std::exception& e) { return HandleException("Encounter exception when delete collection files", e.what()); } @@ -387,7 +387,7 @@ SqliteMetaImpl::CreateCollectionFile(SegmentSchema& file_schema) { auto id = ConnectorPtr->insert(file_schema); file_schema.id_ = id; - ENGINE_LOG_DEBUG << "Successfully create collection file, file id = " << file_schema.file_id_; + LOG_ENGINE_DEBUG_ << "Successfully create collection file, file id = " << file_schema.file_id_; return utils::CreateCollectionFilePath(options_, file_schema); } catch (std::exception& e) { return HandleException("Encounter exception when create collection file", e.what()); @@ -439,7 +439,7 @@ SqliteMetaImpl::GetCollectionFiles(const std::string& collection_id, const std:: collection_files.emplace_back(file_schema); } - ENGINE_LOG_DEBUG << "Get collection files by id"; + LOG_ENGINE_DEBUG_ << "Get collection files by id"; return result; } catch (std::exception& e) { return HandleException("Encounter exception when lookup collection files", e.what()); @@ -489,7 +489,7 @@ SqliteMetaImpl::GetCollectionFilesBySegmentId(const std::string& segment_id, } } - ENGINE_LOG_DEBUG << "Get collection files by segment id"; + LOG_ENGINE_DEBUG_ << "Get collection files by segment id"; return Status::OK(); } catch (std::exception& e) { return HandleException("Encounter exception when lookup collection files by segment id", e.what()); @@ -504,7 +504,7 @@ SqliteMetaImpl::UpdateCollectionFlag(const std::string& collection_id, int64_t f // set all backup file to raw ConnectorPtr->update_all(set(c(&CollectionSchema::flag_) = flag), where(c(&CollectionSchema::collection_id_) == collection_id)); - ENGINE_LOG_DEBUG << "Successfully update collection flag, collection id = " << collection_id; + LOG_ENGINE_DEBUG_ << "Successfully update collection flag, collection id = " << collection_id; } catch (std::exception& e) { std::string msg = "Encounter exception when update collection flag: collection_id = " + collection_id; return HandleException(msg, e.what()); @@ -520,7 +520,7 @@ SqliteMetaImpl::UpdateCollectionFlushLSN(const std::string& collection_id, uint6 ConnectorPtr->update_all(set(c(&CollectionSchema::flush_lsn_) = flush_lsn), where(c(&CollectionSchema::collection_id_) == collection_id)); - ENGINE_LOG_DEBUG << "Successfully update collection flush_lsn, collection id = " << collection_id << " flush_lsn = " << flush_lsn;; + LOG_ENGINE_DEBUG_ << "Successfully update collection flush_lsn, collection id = " << collection_id << " flush_lsn = " << flush_lsn;; } catch (std::exception& e) { std::string msg = "Encounter exception when update collection lsn: collection_id = " + collection_id; return HandleException(msg, e.what()); @@ -571,7 +571,7 @@ SqliteMetaImpl::UpdateCollectionFile(SegmentSchema& file_schema) { ConnectorPtr->update(file_schema); - ENGINE_LOG_DEBUG << "Update single collection file, file id = " << file_schema.file_id_; + LOG_ENGINE_DEBUG_ << "Update single collection file, file id = " << file_schema.file_id_; } catch (std::exception& e) { std::string msg = "Exception update collection file: collection_id = " + file_schema.collection_id_ + " file_id = " + file_schema.file_id_; @@ -621,7 +621,7 @@ SqliteMetaImpl::UpdateCollectionFiles(SegmentsSchema& files) { return HandleException("UpdateCollectionFiles error: sqlite transaction failed"); } - ENGINE_LOG_DEBUG << "Update " << files.size() << " collection files"; + LOG_ENGINE_DEBUG_ << "Update " << files.size() << " collection files"; } catch (std::exception& e) { return HandleException("Encounter exception when update collection files", e.what()); } @@ -640,7 +640,7 @@ SqliteMetaImpl::UpdateCollectionFilesRowCount(SegmentsSchema& files) { ConnectorPtr->update_all(set(c(&SegmentSchema::row_count_) = file.row_count_, c(&SegmentSchema::updated_time_) = utils::GetMicroSecTimeStamp()), where(c(&SegmentSchema::file_id_) == file.file_id_)); - ENGINE_LOG_DEBUG << "Update file " << file.file_id_ << " row count to " << file.row_count_; + LOG_ENGINE_DEBUG_ << "Update file " << file.file_id_ << " row count to " << file.row_count_; } } catch (std::exception& e) { return HandleException("Encounter exception when update collection files row count", e.what()); @@ -692,7 +692,7 @@ SqliteMetaImpl::UpdateCollectionIndex(const std::string& collection_id, const Co where(c(&SegmentSchema::collection_id_) == collection_id and c(&SegmentSchema::file_type_) == (int)SegmentSchema::BACKUP)); - ENGINE_LOG_DEBUG << "Successfully update collection index, collection id = " << collection_id; + LOG_ENGINE_DEBUG_ << "Successfully update collection index, collection id = " << collection_id; } catch (std::exception& e) { std::string msg = "Encounter exception when update collection index: collection_id = " + collection_id; return HandleException(msg, e.what()); @@ -715,7 +715,7 @@ SqliteMetaImpl::UpdateCollectionFilesToIndex(const std::string& collection_id) { c(&SegmentSchema::row_count_) >= meta::BUILD_INDEX_THRESHOLD and c(&SegmentSchema::file_type_) == (int)SegmentSchema::RAW)); - ENGINE_LOG_DEBUG << "Update files to to_index, collection id = " << collection_id; + LOG_ENGINE_DEBUG_ << "Update files to to_index, collection id = " << collection_id; } catch (std::exception& e) { return HandleException("Encounter exception when update collection files to to_index", e.what()); } @@ -783,7 +783,7 @@ SqliteMetaImpl::DropCollectionIndex(const std::string& collection_id) { set(c(&CollectionSchema::engine_type_) = raw_engine_type, c(&CollectionSchema::index_params_) = "{}"), where(c(&CollectionSchema::collection_id_) == collection_id)); - ENGINE_LOG_DEBUG << "Successfully drop collection index, collection id = " << collection_id; + LOG_ENGINE_DEBUG_ << "Successfully drop collection index, collection id = " << collection_id; } catch (std::exception& e) { return HandleException("Encounter exception when delete collection index files", e.what()); } @@ -968,11 +968,11 @@ SqliteMetaImpl::FilesToSearch(const std::string& collection_id, SegmentsSchema& files.emplace_back(collection_file); } if (files.empty()) { - ENGINE_LOG_ERROR << "No file to search for collection: " << collection_id; + LOG_ENGINE_ERROR_ << "No file to search for collection: " << collection_id; } if (selected.size() > 0) { - ENGINE_LOG_DEBUG << "Collect " << selected.size() << " to-search files"; + LOG_ENGINE_DEBUG_ << "Collect " << selected.size() << " to-search files"; } return ret; } catch (std::exception& e) { @@ -1038,7 +1038,7 @@ SqliteMetaImpl::FilesToMerge(const std::string& collection_id, SegmentsSchema& f } if (to_merge_files > 0) { - ENGINE_LOG_TRACE << "Collect " << to_merge_files << " to-merge files"; + LOG_ENGINE_TRACE_ << "Collect " << to_merge_files << " to-merge files"; } return result; } catch (std::exception& e) { @@ -1102,7 +1102,7 @@ SqliteMetaImpl::FilesToIndex(SegmentsSchema& files) { } if (selected.size() > 0) { - ENGINE_LOG_DEBUG << "Collect " << selected.size() << " to-index files"; + LOG_ENGINE_DEBUG_ << "Collect " << selected.size() << " to-index files"; } return ret; } catch (std::exception& e) { @@ -1206,7 +1206,7 @@ SqliteMetaImpl::FilesByType(const std::string& collection_id, const std::vector< default:break; } } - ENGINE_LOG_DEBUG << msg; + LOG_ENGINE_DEBUG_ << msg; } } catch (std::exception& e) { return HandleException("Encounter exception when check non index files", e.what()); @@ -1284,9 +1284,9 @@ SqliteMetaImpl::FilesByID(const std::vector& ids, SegmentsSchema& files) } if (files.empty()) { - ENGINE_LOG_ERROR << "No file to search in file id list"; + LOG_ENGINE_ERROR_ << "No file to search in file id list"; } else { - ENGINE_LOG_DEBUG << "Collect " << selected.size() << " files by id"; + LOG_ENGINE_DEBUG_ << "Collect " << selected.size() << " files by id"; } return ret; @@ -1323,7 +1323,7 @@ SqliteMetaImpl::Archive() { return HandleException("Encounter exception when update collection files", e.what()); } - ENGINE_LOG_DEBUG << "Archive old files"; + LOG_ENGINE_DEBUG_ << "Archive old files"; } if (criteria == engine::ARCHIVE_CONF_DISK) { uint64_t sum = 0; @@ -1332,7 +1332,7 @@ SqliteMetaImpl::Archive() { int64_t to_delete = (int64_t)sum - limit * G; DiscardFiles(to_delete); - ENGINE_LOG_DEBUG << "Archive files to free disk"; + LOG_ENGINE_DEBUG_ << "Archive files to free disk"; } } @@ -1375,7 +1375,7 @@ SqliteMetaImpl::CleanUpShadowFiles() { auto commited = ConnectorPtr->transaction([&]() mutable { for (auto& file : files) { - ENGINE_LOG_DEBUG << "Remove collection file type as NEW"; + LOG_ENGINE_DEBUG_ << "Remove collection file type as NEW"; ConnectorPtr->remove(std::get<0>(file)); } return true; @@ -1388,7 +1388,7 @@ SqliteMetaImpl::CleanUpShadowFiles() { } if (files.size() > 0) { - ENGINE_LOG_DEBUG << "Clean " << files.size() << " files"; + LOG_ENGINE_DEBUG_ << "Clean " << files.size() << " files"; } } catch (std::exception& e) { return HandleException("Encounter exception when clean collection file", e.what()); @@ -1439,7 +1439,7 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/ // check if the file can be deleted if (OngoingFileChecker::GetInstance().IsIgnored(collection_file)) { - ENGINE_LOG_DEBUG << "File:" << collection_file.file_id_ + LOG_ENGINE_DEBUG_ << "File:" << collection_file.file_id_ << " currently is in use, not able to delete now"; continue; // ignore this file, don't delete it } @@ -1457,7 +1457,7 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/ // delete file from disk storage utils::DeleteCollectionFilePath(options_, collection_file); - ENGINE_LOG_DEBUG << "Remove file id:" << collection_file.file_id_ << " location:" + LOG_ENGINE_DEBUG_ << "Remove file id:" << collection_file.file_id_ << " location:" << collection_file.location_; collection_ids.insert(collection_file.collection_id_); segment_ids.insert(std::make_pair(collection_file.segment_id_, collection_file)); @@ -1474,7 +1474,7 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/ } if (clean_files > 0) { - ENGINE_LOG_DEBUG << "Clean " << clean_files << " files expired in " << seconds << " seconds"; + LOG_ENGINE_DEBUG_ << "Clean " << clean_files << " files expired in " << seconds << " seconds"; } } catch (std::exception& e) { return HandleException("Encounter exception when clean collection files", e.what()); @@ -1506,7 +1506,7 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/ } if (collections.size() > 0) { - ENGINE_LOG_DEBUG << "Remove " << collections.size() << " collections from meta"; + LOG_ENGINE_DEBUG_ << "Remove " << collections.size() << " collections from meta"; } } catch (std::exception& e) { return HandleException("Encounter exception when clean collection files", e.what()); @@ -1529,7 +1529,7 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/ } if (remove_collections) { - ENGINE_LOG_DEBUG << "Remove " << remove_collections << " collections folder"; + LOG_ENGINE_DEBUG_ << "Remove " << remove_collections << " collections folder"; } } catch (std::exception& e) { return HandleException("Encounter exception when delete collection folder", e.what()); @@ -1549,13 +1549,13 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/ utils::DeleteSegment(options_, segment_id.second); std::string segment_dir; utils::GetParentPath(segment_id.second.location_, segment_dir); - ENGINE_LOG_DEBUG << "Remove segment directory: " << segment_dir; + LOG_ENGINE_DEBUG_ << "Remove segment directory: " << segment_dir; ++remove_segments; } } if (remove_segments > 0) { - ENGINE_LOG_DEBUG << "Remove " << remove_segments << " segments folder"; + LOG_ENGINE_DEBUG_ << "Remove " << remove_segments << " segments folder"; } } catch (std::exception& e) { return HandleException("Encounter exception when delete collection folder", e.what()); @@ -1597,7 +1597,7 @@ SqliteMetaImpl::Count(const std::string& collection_id, uint64_t& result) { Status SqliteMetaImpl::DropAll() { - ENGINE_LOG_DEBUG << "Drop all sqlite meta"; + LOG_ENGINE_DEBUG_ << "Drop all sqlite meta"; try { ConnectorPtr->drop_table(META_TABLES); @@ -1615,7 +1615,7 @@ SqliteMetaImpl::DiscardFiles(int64_t to_discard_size) { return Status::OK(); } - ENGINE_LOG_DEBUG << "About to discard size=" << to_discard_size; + LOG_ENGINE_DEBUG_ << "About to discard size=" << to_discard_size; try { fiu_do_on("SqliteMetaImpl.DiscardFiles.throw_exception", throw std::exception()); @@ -1640,7 +1640,7 @@ SqliteMetaImpl::DiscardFiles(int64_t to_discard_size) { collection_file.id_ = std::get<0>(file); collection_file.file_size_ = std::get<1>(file); ids.push_back(collection_file.id_); - ENGINE_LOG_DEBUG << "Discard file id=" << collection_file.file_id_ + LOG_ENGINE_DEBUG_ << "Discard file id=" << collection_file.file_id_ << " file size=" << collection_file.file_size_; to_discard_size -= collection_file.file_size_; } @@ -1685,7 +1685,7 @@ SqliteMetaImpl::SetGlobalLastLSN(uint64_t lsn) { ConnectorPtr->update_all(set(c(&EnvironmentSchema::global_lsn_) = lsn)); } - ENGINE_LOG_DEBUG << "Update global lsn = " << lsn; + LOG_ENGINE_DEBUG_ << "Update global lsn = " << lsn; } catch (std::exception& e) { std::string msg = "Exception update global lsn = " + lsn; return HandleException(msg, e.what()); diff --git a/core/src/db/wal/WalBuffer.cpp b/core/src/db/wal/WalBuffer.cpp index c0a092a9cfc594af1fa8848f1f42eebd0f907a87..258848b25cb6b1de035d9538afa407f5cc0b6be8 100644 --- a/core/src/db/wal/WalBuffer.cpp +++ b/core/src/db/wal/WalBuffer.cpp @@ -50,7 +50,7 @@ MXLogBuffer::~MXLogBuffer() { */ bool MXLogBuffer::Init(uint64_t start_lsn, uint64_t end_lsn) { - WAL_LOG_DEBUG << "start_lsn " << start_lsn << " end_lsn " << end_lsn; + LOG_WAL_DEBUG_ << "start_lsn " << start_lsn << " end_lsn " << end_lsn; ParserLsn(start_lsn, mxlog_buffer_reader_.file_no, mxlog_buffer_reader_.buf_offset); ParserLsn(end_lsn, mxlog_buffer_writer_.file_no, mxlog_buffer_writer_.buf_offset); @@ -72,7 +72,7 @@ MXLogBuffer::Init(uint64_t start_lsn, uint64_t end_lsn) { file_handler.SetFileName(ToFileName(i)); auto file_size = file_handler.GetFileSize(); if (file_size == 0) { - WAL_LOG_ERROR << "bad wal file " << i; + LOG_WAL_ERROR_ << "bad wal file " << i; return false; } if (file_size > buffer_size_need) { @@ -85,7 +85,7 @@ MXLogBuffer::Init(uint64_t start_lsn, uint64_t end_lsn) { if (buffer_size_need > mxlog_buffer_size_) { mxlog_buffer_size_ = buffer_size_need; - WAL_LOG_INFO << "recovery will need more buffer, buffer size changed " << mxlog_buffer_size_; + LOG_WAL_INFO_ << "recovery will need more buffer, buffer size changed " << mxlog_buffer_size_; } } @@ -104,14 +104,14 @@ MXLogBuffer::Init(uint64_t start_lsn, uint64_t end_lsn) { } else { mxlog_writer_.SetFileOpenMode("r+"); if (!mxlog_writer_.FileExists()) { - WAL_LOG_ERROR << "wal file not exist " << mxlog_buffer_writer_.file_no; + LOG_WAL_ERROR_ << "wal file not exist " << mxlog_buffer_writer_.file_no; return false; } auto read_offset = mxlog_buffer_reader_.buf_offset; auto read_size = mxlog_buffer_writer_.buf_offset - mxlog_buffer_reader_.buf_offset; if (!mxlog_writer_.Load(buf_[0].get() + read_offset, read_offset, read_size)) { - WAL_LOG_ERROR << "load wal file error " << read_offset << " " << read_size; + LOG_WAL_ERROR_ << "load wal file error " << read_offset << " " << read_size; return false; } } @@ -135,11 +135,11 @@ MXLogBuffer::Init(uint64_t start_lsn, uint64_t end_lsn) { mxlog_writer_.SetFileName(ToFileName(mxlog_buffer_writer_.file_no)); mxlog_writer_.SetFileOpenMode("r+"); if (!mxlog_writer_.FileExists()) { - WAL_LOG_ERROR << "wal file not exist " << mxlog_buffer_writer_.file_no; + LOG_WAL_ERROR_ << "wal file not exist " << mxlog_buffer_writer_.file_no; return false; } if (!mxlog_writer_.Load(buf_[1].get(), 0, mxlog_buffer_writer_.buf_offset)) { - WAL_LOG_ERROR << "load wal file error " << mxlog_buffer_writer_.file_no; + LOG_WAL_ERROR_ << "load wal file error " << mxlog_buffer_writer_.file_no; return false; } } @@ -151,7 +151,7 @@ MXLogBuffer::Init(uint64_t start_lsn, uint64_t end_lsn) { void MXLogBuffer::Reset(uint64_t lsn) { - WAL_LOG_DEBUG << "reset lsn " << lsn; + LOG_WAL_DEBUG_ << "reset lsn " << lsn; buf_[0] = BufferPtr(new char[mxlog_buffer_size_]); buf_[1] = BufferPtr(new char[mxlog_buffer_size_]); @@ -206,7 +206,7 @@ MXLogBuffer::Append(MXLogRecord& record) { // Reborn means close old wal file and open new wal file if (!mxlog_writer_.ReBorn(ToFileName(mxlog_buffer_writer_.file_no), "w")) { - WAL_LOG_ERROR << "ReBorn wal file error " << mxlog_buffer_writer_.file_no; + LOG_WAL_ERROR_ << "ReBorn wal file error " << mxlog_buffer_writer_.file_no; return WAL_FILE_ERROR; } } @@ -247,7 +247,7 @@ MXLogBuffer::Append(MXLogRecord& record) { bool write_rst = mxlog_writer_.Write(current_write_buf + mxlog_buffer_writer_.buf_offset, record_size); if (!write_rst) { - WAL_LOG_ERROR << "write wal file error"; + LOG_WAL_ERROR_ << "write wal file error"; return WAL_FILE_ERROR; } @@ -289,7 +289,7 @@ MXLogBuffer::Next(const uint64_t last_applied_lsn, MXLogRecord& record) { mxlog_reader.SetFileOpenMode("r"); uint32_t file_size = mxlog_reader.Load(buf_[mxlog_buffer_reader_.buf_idx].get(), 0); if (file_size == 0) { - WAL_LOG_ERROR << "load wal file error " << mxlog_buffer_reader_.file_no; + LOG_WAL_ERROR_ << "load wal file error " << mxlog_buffer_reader_.file_no; return WAL_FILE_ERROR; } mxlog_buffer_reader_.max_offset = file_size; @@ -346,29 +346,29 @@ MXLogBuffer::GetReadLsn() { bool MXLogBuffer::ResetWriteLsn(uint64_t lsn) { - WAL_LOG_INFO << "reset write lsn " << lsn; + LOG_WAL_INFO_ << "reset write lsn " << lsn; int32_t old_file_no = mxlog_buffer_writer_.file_no; ParserLsn(lsn, mxlog_buffer_writer_.file_no, mxlog_buffer_writer_.buf_offset); if (old_file_no == mxlog_buffer_writer_.file_no) { - WAL_LOG_DEBUG << "file No. is not changed"; + LOG_WAL_DEBUG_ << "file No. is not changed"; return true; } std::unique_lock lck(mutex_); if (mxlog_buffer_writer_.file_no == mxlog_buffer_reader_.file_no) { mxlog_buffer_writer_.buf_idx = mxlog_buffer_reader_.buf_idx; - WAL_LOG_DEBUG << "file No. is the same as reader"; + LOG_WAL_DEBUG_ << "file No. is the same as reader"; return true; } lck.unlock(); if (!mxlog_writer_.ReBorn(ToFileName(mxlog_buffer_writer_.file_no), "r+")) { - WAL_LOG_ERROR << "reborn file error " << mxlog_buffer_writer_.file_no; + LOG_WAL_ERROR_ << "reborn file error " << mxlog_buffer_writer_.file_no; return false; } if (!mxlog_writer_.Load(buf_[mxlog_buffer_writer_.buf_idx].get(), 0, mxlog_buffer_writer_.buf_offset)) { - WAL_LOG_ERROR << "load file error"; + LOG_WAL_ERROR_ << "load file error"; return false; } @@ -387,7 +387,7 @@ MXLogBuffer::SetFileNoFrom(uint32_t file_no) { if (!file_handler.FileExists()) { break; } - WAL_LOG_INFO << "Delete wal file " << file_no; + LOG_WAL_INFO_ << "Delete wal file " << file_no; file_handler.DeleteFile(); } while (file_no > 0); } @@ -402,7 +402,7 @@ MXLogBuffer::RemoveOldFiles(uint64_t flushed_lsn) { MXLogFileHandler file_handler(mxlog_writer_.GetFilePath()); do { file_handler.SetFileName(ToFileName(file_no_from_)); - WAL_LOG_INFO << "Delete wal file " << file_no_from_; + LOG_WAL_INFO_ << "Delete wal file " << file_no_from_; file_handler.DeleteFile(); } while (++file_no_from_ < file_no); } diff --git a/core/src/db/wal/WalManager.cpp b/core/src/db/wal/WalManager.cpp index b4beee671bfe9dfc0f6182a156d7743be0bdb0f6..70c78504a60761421241cef07d82441b6edacb01 100644 --- a/core/src/db/wal/WalManager.cpp +++ b/core/src/db/wal/WalManager.cpp @@ -41,7 +41,7 @@ WalManager::WalManager(const MXLogConfiguration& config) { auto status = server::CommonUtil::CreateDirectory(mxlog_config_.mxlog_path); if (!status.ok()) { std::string msg = "failed to create wal directory " + mxlog_config_.mxlog_path; - ENGINE_LOG_ERROR << msg; + LOG_ENGINE_ERROR_ << msg; throw Exception(WAL_PATH_ERROR, msg); } } @@ -148,8 +148,8 @@ WalManager::GetNextRecovery(MXLogRecord& record) { } } - WAL_LOG_INFO << "record type " << (int32_t)record.type << " record lsn " << record.lsn << " error code " - << error_code; + LOG_WAL_INFO_ << "record type " << (int32_t)record.type << " record lsn " << record.lsn << " error code " + << error_code; return error_code; } @@ -166,7 +166,7 @@ WalManager::GetNextRecord(MXLogRecord& record) { record.lsn = flush_info_.lsn_; flush_info_.Clear(); - WAL_LOG_INFO << "record flush collection " << record.collection_id << " lsn " << record.lsn; + LOG_WAL_INFO_ << "record flush collection " << record.collection_id << " lsn " << record.lsn; return true; } } @@ -195,14 +195,14 @@ WalManager::GetNextRecord(MXLogRecord& record) { } } - WAL_LOG_INFO << "record type " << (int32_t)record.type << " collection " << record.collection_id << " lsn " - << record.lsn; + LOG_WAL_INFO_ << "record type " << (int32_t)record.type << " collection " << record.collection_id << " lsn " + << record.lsn; return error_code; } uint64_t WalManager::CreateCollection(const std::string& collection_id) { - WAL_LOG_INFO << "create collection " << collection_id << " " << last_applied_lsn_; + LOG_WAL_INFO_ << "create collection " << collection_id << " " << last_applied_lsn_; std::lock_guard lck(mutex_); uint64_t applied_lsn = last_applied_lsn_; tables_[collection_id] = {applied_lsn, applied_lsn}; @@ -211,7 +211,7 @@ WalManager::CreateCollection(const std::string& collection_id) { void WalManager::DropCollection(const std::string& collection_id) { - WAL_LOG_INFO << "drop collection " << collection_id; + LOG_WAL_INFO_ << "drop collection " << collection_id; std::lock_guard lck(mutex_); tables_.erase(collection_id); } @@ -225,7 +225,7 @@ WalManager::CollectionFlushed(const std::string& collection_id, uint64_t lsn) { } lck.unlock(); - WAL_LOG_INFO << collection_id << " is flushed by lsn " << lsn; + LOG_WAL_INFO_ << collection_id << " is flushed by lsn " << lsn; } template @@ -243,7 +243,7 @@ WalManager::Insert(const std::string& collection_id, const std::string& partitio size_t vector_num = vector_ids.size(); if (vector_num == 0) { - WAL_LOG_ERROR << LogOut("[%s][%ld] The ids is empty.", "insert", 0); + LOG_WAL_ERROR_ << LogOut("[%s][%ld] The ids is empty.", "insert", 0); return false; } size_t dim = vectors.size() / vector_num; @@ -265,8 +265,8 @@ WalManager::Insert(const std::string& collection_id, const std::string& partitio max_rcd_num = (mxlog_config_.buffer_size - head_size) / unit_size; } if (max_rcd_num == 0) { - WAL_LOG_ERROR << LogOut("[%s][%ld]", "insert", 0) << "Wal buffer size is too small " - << mxlog_config_.buffer_size << " unit " << unit_size; + LOG_WAL_ERROR_ << LogOut("[%s][%ld]", "insert", 0) << "Wal buffer size is too small " + << mxlog_config_.buffer_size << " unit " << unit_size; return false; } @@ -291,8 +291,8 @@ WalManager::Insert(const std::string& collection_id, const std::string& partitio } lck.unlock(); - WAL_LOG_INFO << LogOut("[%s][%ld]", "insert", 0) << collection_id << " insert in part " << partition_tag - << " with lsn " << new_lsn; + LOG_WAL_INFO_ << LogOut("[%s][%ld]", "insert", 0) << collection_id << " insert in part " << partition_tag + << " with lsn " << new_lsn; return p_meta_handler_->SetMXLogInternalMeta(new_lsn); } @@ -301,7 +301,7 @@ bool WalManager::DeleteById(const std::string& collection_id, const IDNumbers& vector_ids) { size_t vector_num = vector_ids.size(); if (vector_num == 0) { - WAL_LOG_ERROR << "The ids is empty."; + LOG_WAL_ERROR_ << "The ids is empty."; return false; } @@ -344,7 +344,7 @@ WalManager::DeleteById(const std::string& collection_id, const IDNumbers& vector } lck.unlock(); - WAL_LOG_INFO << collection_id << " delete rows by id, lsn " << new_lsn; + LOG_WAL_INFO_ << collection_id << " delete rows by id, lsn " << new_lsn; return p_meta_handler_->SetMXLogInternalMeta(new_lsn); } @@ -381,7 +381,7 @@ WalManager::Flush(const std::string& collection_id) { flush_info_.lsn_ = lsn; } - WAL_LOG_INFO << collection_id << " want to be flush, lsn " << lsn; + LOG_WAL_INFO_ << collection_id << " want to be flush, lsn " << lsn; return lsn; } diff --git a/core/src/index/archive/KnowhereResource.cpp b/core/src/index/archive/KnowhereResource.cpp index 905d40069ff1a9f3f7f3a758eab9382cd0067397..07fb6d784c2a95448d49ba2bc983f6a3ddda91a1 100644 --- a/core/src/index/archive/KnowhereResource.cpp +++ b/core/src/index/archive/KnowhereResource.cpp @@ -40,7 +40,7 @@ KnowhereResource::Initialize() { faiss::faiss_use_avx512 = use_avx512; std::string cpu_flag; if (faiss::hook_init(cpu_flag)) { - ENGINE_LOG_DEBUG << "FAISS hook " << cpu_flag; + LOG_ENGINE_DEBUG_ << "FAISS hook " << cpu_flag; } else { return Status(KNOWHERE_UNEXPECTED_ERROR, "FAISS hook fail, CPU not supported!"); } diff --git a/core/src/metrics/SystemInfo.cpp b/core/src/metrics/SystemInfo.cpp index b47d88cbb8b6f87e7e84f6539476aa61e9323a78..cab394ac9e7fe9c6e32e5c4bbd4c7528c7ad88b1 100644 --- a/core/src/metrics/SystemInfo.cpp +++ b/core/src/metrics/SystemInfo.cpp @@ -63,13 +63,13 @@ SystemInfo::Init() { nvmlresult = nvmlInit(); fiu_do_on("SystemInfo.Init.nvmInit_fail", nvmlresult = NVML_ERROR_NOT_FOUND); if (NVML_SUCCESS != nvmlresult) { - SERVER_LOG_ERROR << "System information initilization failed"; + LOG_SERVER_ERROR_ << "System information initilization failed"; return; } nvmlresult = nvmlDeviceGetCount(&num_device_); fiu_do_on("SystemInfo.Init.nvm_getDevice_fail", nvmlresult = NVML_ERROR_NOT_FOUND); if (NVML_SUCCESS != nvmlresult) { - SERVER_LOG_ERROR << "Unable to get devidce number"; + LOG_SERVER_ERROR_ << "Unable to get devidce number"; return; } #endif @@ -158,7 +158,7 @@ SystemInfo::getTotalCpuTime(std::vector& work_time_array) { FILE* file = fopen("/proc/stat", "r"); fiu_do_on("SystemInfo.getTotalCpuTime.open_proc", file = NULL); if (file == NULL) { - SERVER_LOG_ERROR << "Could not open stat file"; + LOG_SERVER_ERROR_ << "Could not open stat file"; return total_time_array; } @@ -170,7 +170,7 @@ SystemInfo::getTotalCpuTime(std::vector& work_time_array) { char* ret = fgets(buffer, sizeof(buffer) - 1, file); fiu_do_on("SystemInfo.getTotalCpuTime.read_proc", ret = NULL); if (ret == NULL) { - SERVER_LOG_ERROR << "Could not read stat file"; + LOG_SERVER_ERROR_ << "Could not read stat file"; fclose(file); return total_time_array; } @@ -265,7 +265,7 @@ SystemInfo::CPUTemperature() { dir = opendir(path.c_str()); fiu_do_on("SystemInfo.CPUTemperature.opendir", dir = NULL); if (!dir) { - SERVER_LOG_ERROR << "Could not open hwmon directory"; + LOG_SERVER_ERROR_ << "Could not open hwmon directory"; return result; } @@ -283,7 +283,7 @@ SystemInfo::CPUTemperature() { FILE* file = fopen(object.c_str(), "r"); fiu_do_on("SystemInfo.CPUTemperature.openfile", file = NULL); if (file == nullptr) { - SERVER_LOG_ERROR << "Could not open temperature file"; + LOG_SERVER_ERROR_ << "Could not open temperature file"; return result; } float temp; diff --git a/core/src/metrics/prometheus/PrometheusMetrics.cpp b/core/src/metrics/prometheus/PrometheusMetrics.cpp index bfdbcb90c423f2f637a5fc9284658c3c5f5fb42c..3bdb2d1b9acbfa9d9ce307f935e4d56399b3bf0a 100644 --- a/core/src/metrics/prometheus/PrometheusMetrics.cpp +++ b/core/src/metrics/prometheus/PrometheusMetrics.cpp @@ -49,7 +49,7 @@ PrometheusMetrics::Init() { // Pushgateway Registry gateway_->RegisterCollectable(registry_); } catch (std::exception& ex) { - SERVER_LOG_ERROR << "Failed to connect prometheus server: " << std::string(ex.what()); + LOG_SERVER_ERROR_ << "Failed to connect prometheus server: " << std::string(ex.what()); return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } diff --git a/core/src/metrics/prometheus/PrometheusMetrics.h b/core/src/metrics/prometheus/PrometheusMetrics.h index e44c7a54313f8018ecb06bf3e1bd14df30f448b4..b452eb2224d3940e74079a1480be07d21808c129 100644 --- a/core/src/metrics/prometheus/PrometheusMetrics.h +++ b/core/src/metrics/prometheus/PrometheusMetrics.h @@ -293,7 +293,7 @@ class PrometheusMetrics : public MetricsBase { PushToGateway() override { if (startup_) { if (gateway_->Push() != 200) { - ENGINE_LOG_WARNING << "Metrics pushgateway failed"; + LOG_ENGINE_WARNING_ << "Metrics pushgateway failed"; } } } diff --git a/core/src/scheduler/JobMgr.cpp b/core/src/scheduler/JobMgr.cpp index 4a4147ab03932497f220ba1d8972d8c4e56fea52..78e00ad6c896a6103109752f31748821fa15ec14 100644 --- a/core/src/scheduler/JobMgr.cpp +++ b/core/src/scheduler/JobMgr.cpp @@ -68,6 +68,7 @@ JobMgr::Put(const JobPtr& job) { void JobMgr::worker_function() { + SetThreadName("jobmgr_thread"); while (running_) { std::unique_lock lock(mutex_); cv_.wait(lock, [this] { return !queue_.empty(); }); diff --git a/core/src/scheduler/ResourceMgr.cpp b/core/src/scheduler/ResourceMgr.cpp index dfbf0b600199da227ee725067c1031ae2e0e2fea..ea5e3171d2e74b1fcb1eb8b39802f33ab368f056 100644 --- a/core/src/scheduler/ResourceMgr.cpp +++ b/core/src/scheduler/ResourceMgr.cpp @@ -19,8 +19,8 @@ namespace scheduler { void ResourceMgr::Start() { if (not check_resource_valid()) { - ENGINE_LOG_ERROR << "Resources invalid, cannot start ResourceMgr."; - ENGINE_LOG_ERROR << Dump(); + LOG_ENGINE_ERROR_ << "Resources invalid, cannot start ResourceMgr."; + LOG_ENGINE_ERROR_ << Dump(); return; } @@ -54,7 +54,7 @@ ResourceMgr::Add(ResourcePtr&& resource) { std::lock_guard lck(resources_mutex_); if (running_) { - ENGINE_LOG_ERROR << "ResourceMgr is running, not allow to add resource"; + LOG_ENGINE_ERROR_ << "ResourceMgr is running, not allow to add resource"; return ret; } @@ -97,7 +97,7 @@ void ResourceMgr::Clear() { std::lock_guard lck(resources_mutex_); if (running_) { - ENGINE_LOG_ERROR << "ResourceMgr is running, cannot clear."; + LOG_ENGINE_ERROR_ << "ResourceMgr is running, cannot clear."; return; } disk_resources_.clear(); @@ -237,6 +237,7 @@ ResourceMgr::post_event(const EventPtr& event) { void ResourceMgr::event_process() { + SetThreadName("resevt_thread"); while (running_) { std::unique_lock lock(event_mutex_); event_cv_.wait(lock, [this] { return !queue_.empty(); }); diff --git a/core/src/scheduler/SchedInst.h b/core/src/scheduler/SchedInst.h index 3e89cfabbece2203205788eb57ce85589be1a9d2..2ef7e5dd4334b4a80c692e2240e5fb29d67e1375 100644 --- a/core/src/scheduler/SchedInst.h +++ b/core/src/scheduler/SchedInst.h @@ -110,14 +110,14 @@ class OptimizerInst { for (auto build_id : build_gpus) { build_msg.append(" gpu" + std::to_string(build_id)); } - SERVER_LOG_DEBUG << LogOut("[%s][%d] %s", "search", 0, build_msg.c_str()); + LOG_SERVER_DEBUG_ << LogOut("[%s][%d] %s", "search", 0, build_msg.c_str()); std::string search_msg = "Search gpu:"; for (auto search_id : search_gpus) { search_msg.append(" gpu" + std::to_string(search_id)); } search_msg.append(". gpu_search_threshold:" + std::to_string(gpu_search_threshold)); - SERVER_LOG_DEBUG << LogOut("[%s][%d] %s", "search", 0, build_msg.c_str()); + LOG_SERVER_DEBUG_ << LogOut("[%s][%d] %s", "search", 0, build_msg.c_str()); pass_list.push_back(std::make_shared()); pass_list.push_back(std::make_shared()); diff --git a/core/src/scheduler/Scheduler.cpp b/core/src/scheduler/Scheduler.cpp index 65493e25bb1dbf4c58ea15739f7b33bac399c1d5..bc7c788490b794d9831fb52a4aa7d02054590b81 100644 --- a/core/src/scheduler/Scheduler.cpp +++ b/core/src/scheduler/Scheduler.cpp @@ -79,6 +79,7 @@ Scheduler::process(const EventPtr& event) { void Scheduler::worker_function() { + SetThreadName("schedevt_thread"); while (running_) { std::unique_lock lock(event_mutex_); event_cv_.wait(lock, [this] { return !event_queue_.empty(); }); diff --git a/core/src/scheduler/TaskTable.cpp b/core/src/scheduler/TaskTable.cpp index 3f2a6fedabd4e1e18e5ec6b284685336753395c5..90996dc549407825fbb5e11bbf6166dd6c189c90 100644 --- a/core/src/scheduler/TaskTable.cpp +++ b/core/src/scheduler/TaskTable.cpp @@ -173,7 +173,7 @@ TaskTable::PickToLoad(uint64_t limit) { // if task is a build index task, limit it if (task->Type() == TaskType::BuildIndexTask && task->path().Current() == "cpu") { if (BuildMgrInst::GetInstance()->NumOfAvailable() < 1) { - SERVER_LOG_WARNING << "BuildMgr doesnot have available place for building index"; + LOG_SERVER_WARNING_ << "BuildMgr doesnot have available place for building index"; continue; } } @@ -188,7 +188,7 @@ TaskTable::PickToLoad(uint64_t limit) { size_t count = 0; for (uint64_t j = last_finish_ + 1; j < table_.size(); ++j) { if (not table_[j]) { - SERVER_LOG_WARNING << "collection[" << j << "] is nullptr"; + LOG_SERVER_WARNING_ << "collection[" << j << "] is nullptr"; } if (table_[j]->task->path().Current() == "cpu") { diff --git a/core/src/scheduler/job/BuildIndexJob.cpp b/core/src/scheduler/job/BuildIndexJob.cpp index c2a88f9ea2f2b72a00b4ed3949dcfe7259e876cd..80a7ff8d1625e6c186e491c56bea2e0c5f8d3a38 100644 --- a/core/src/scheduler/job/BuildIndexJob.cpp +++ b/core/src/scheduler/job/BuildIndexJob.cpp @@ -31,8 +31,8 @@ BuildIndexJob::AddToIndexFiles(const engine::meta::SegmentSchemaPtr& to_index_fi return false; } - SERVER_LOG_DEBUG << "BuildIndexJob " << id() << " add to_index file: " << to_index_file->id_ - << ", location: " << to_index_file->location_; + LOG_SERVER_DEBUG_ << "BuildIndexJob " << id() << " add to_index file: " << to_index_file->id_ + << ", location: " << to_index_file->location_; to_index_files_[to_index_file->id_] = to_index_file; return true; @@ -42,7 +42,7 @@ void BuildIndexJob::WaitBuildIndexFinish() { std::unique_lock lock(mutex_); cv_.wait(lock, [this] { return to_index_files_.empty(); }); - SERVER_LOG_DEBUG << "BuildIndexJob " << id() << " all done"; + LOG_SERVER_DEBUG_ << "BuildIndexJob " << id() << " all done"; } void @@ -50,7 +50,7 @@ BuildIndexJob::BuildIndexDone(size_t to_index_id) { std::unique_lock lock(mutex_); to_index_files_.erase(to_index_id); cv_.notify_all(); - SERVER_LOG_DEBUG << "BuildIndexJob " << id() << " finish index file: " << to_index_id; + LOG_SERVER_DEBUG_ << "BuildIndexJob " << id() << " finish index file: " << to_index_id; } json diff --git a/core/src/scheduler/job/SearchJob.cpp b/core/src/scheduler/job/SearchJob.cpp index b19256ce4e2ec8db203c561ae5317952ef064c79..977ea3a31fd03248aaf139a60b5d8eab93787d59 100644 --- a/core/src/scheduler/job/SearchJob.cpp +++ b/core/src/scheduler/job/SearchJob.cpp @@ -28,7 +28,7 @@ SearchJob::AddIndexFile(const SegmentSchemaPtr& index_file) { return false; } - SERVER_LOG_DEBUG << LogOut("[%s][%ld] SearchJob %ld add index file: %ld", "search", 0, id(), index_file->id_); + LOG_SERVER_DEBUG_ << LogOut("[%s][%ld] SearchJob %ld add index file: %ld", "search", 0, id(), index_file->id_); index_files_[index_file->id_] = index_file; return true; @@ -38,7 +38,7 @@ void SearchJob::WaitResult() { std::unique_lock lock(mutex_); cv_.wait(lock, [this] { return index_files_.empty(); }); - SERVER_LOG_DEBUG << LogOut("[%s][%ld] SearchJob %ld all done", "search", 0, id()); + LOG_SERVER_DEBUG_ << LogOut("[%s][%ld] SearchJob %ld all done", "search", 0, id()); } void @@ -49,7 +49,7 @@ SearchJob::SearchDone(size_t index_id) { cv_.notify_all(); } - SERVER_LOG_DEBUG << LogOut("[%s][%ld] SearchJob %ld finish index file: %ld", "search", 0, id(), index_id); + LOG_SERVER_DEBUG_ << LogOut("[%s][%ld] SearchJob %ld finish index file: %ld", "search", 0, id(), index_id); } ResultIds& diff --git a/core/src/scheduler/optimizer/BuildIndexPass.cpp b/core/src/scheduler/optimizer/BuildIndexPass.cpp index ec83e107fedac38d0edec55dd253989337e9b784..b0702177f33d0ff65e027bacb771581d2ce7ee92 100644 --- a/core/src/scheduler/optimizer/BuildIndexPass.cpp +++ b/core/src/scheduler/optimizer/BuildIndexPass.cpp @@ -39,19 +39,19 @@ BuildIndexPass::Run(const TaskPtr& task) { ResourcePtr res_ptr; if (!gpu_enable_) { - SERVER_LOG_DEBUG << "Gpu disabled, specify cpu to build index!"; + LOG_SERVER_DEBUG_ << "Gpu disabled, specify cpu to build index!"; res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); } else { fiu_do_on("BuildIndexPass.Run.empty_gpu_ids", build_gpus_.clear()); if (build_gpus_.empty()) { - SERVER_LOG_WARNING << "BuildIndexPass cannot get build index gpu!"; + LOG_SERVER_WARNING_ << "BuildIndexPass cannot get build index gpu!"; return false; } if (specified_gpu_id_ >= build_gpus_.size()) { specified_gpu_id_ = specified_gpu_id_ % build_gpus_.size(); } - SERVER_LOG_DEBUG << "Specify gpu" << specified_gpu_id_ << " to build index!"; + LOG_SERVER_DEBUG_ << "Specify gpu" << specified_gpu_id_ << " to build index!"; res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, build_gpus_[specified_gpu_id_]); specified_gpu_id_ = (specified_gpu_id_ + 1) % build_gpus_.size(); } diff --git a/core/src/scheduler/optimizer/FaissFlatPass.cpp b/core/src/scheduler/optimizer/FaissFlatPass.cpp index c0fb4780ad00db193883dd43160190b033f88574..871a833eacf6f97b4794feab0cb2b66bd6af7808 100644 --- a/core/src/scheduler/optimizer/FaissFlatPass.cpp +++ b/core/src/scheduler/optimizer/FaissFlatPass.cpp @@ -54,16 +54,16 @@ FaissFlatPass::Run(const TaskPtr& task) { auto search_job = std::static_pointer_cast(search_task->job_.lock()); ResourcePtr res_ptr; if (!gpu_enable_) { - SERVER_LOG_DEBUG << LogOut("[%s][%d] FaissFlatPass: gpu disable, specify cpu to search!", "search", 0); + LOG_SERVER_DEBUG_ << LogOut("[%s][%d] FaissFlatPass: gpu disable, specify cpu to search!", "search", 0); res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); } else if (search_job->nq() < threshold_) { - SERVER_LOG_DEBUG << LogOut("[%s][%d] FaissFlatPass: nq < gpu_search_threshold, specify cpu to search!", - "search", 0); + LOG_SERVER_DEBUG_ << LogOut("[%s][%d] FaissFlatPass: nq < gpu_search_threshold, specify cpu to search!", + "search", 0); res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); } else { auto best_device_id = count_ % search_gpus_.size(); - SERVER_LOG_DEBUG << LogOut("[%s][%d] FaissFlatPass: nq > gpu_search_threshold, specify gpu %d to search!", - best_device_id, "search", 0); + LOG_SERVER_DEBUG_ << LogOut("[%s][%d] FaissFlatPass: nq > gpu_search_threshold, specify gpu %d to search!", + best_device_id, "search", 0); ++count_; res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, search_gpus_[best_device_id]); } diff --git a/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp b/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp index 61c1f44e4f37d773ffa69337bf15b10354218dc2..cc45eb9e382e704f94529ffd08cad339a62c26ae 100644 --- a/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp @@ -55,16 +55,16 @@ FaissIVFFlatPass::Run(const TaskPtr& task) { auto search_job = std::static_pointer_cast(search_task->job_.lock()); ResourcePtr res_ptr; if (!gpu_enable_) { - SERVER_LOG_DEBUG << LogOut("[%s][%d] FaissIVFFlatPass: gpu disable, specify cpu to search!", "search", 0); + LOG_SERVER_DEBUG_ << LogOut("[%s][%d] FaissIVFFlatPass: gpu disable, specify cpu to search!", "search", 0); res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); } else if (search_job->nq() < threshold_) { - SERVER_LOG_DEBUG << LogOut("[%s][%d] FaissIVFFlatPass: nq < gpu_search_threshold, specify cpu to search!", - "search", 0); + LOG_SERVER_DEBUG_ << LogOut("[%s][%d] FaissIVFFlatPass: nq < gpu_search_threshold, specify cpu to search!", + "search", 0); res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); } else { auto best_device_id = count_ % search_gpus_.size(); - SERVER_LOG_DEBUG << LogOut("[%s][%d] FaissIVFFlatPass: nq > gpu_search_threshold, specify gpu %d to search!", - "search", 0, best_device_id); + LOG_SERVER_DEBUG_ << LogOut("[%s][%d] FaissIVFFlatPass: nq > gpu_search_threshold, specify gpu %d to search!", + "search", 0, best_device_id); count_++; res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, search_gpus_[best_device_id]); } diff --git a/core/src/scheduler/optimizer/FaissIVFPQPass.cpp b/core/src/scheduler/optimizer/FaissIVFPQPass.cpp index 6a83d398974e2c84b14595f72d38b61a5a3a59c7..53f16dd08817c4df3bc00e696bf9d71f53abe464 100644 --- a/core/src/scheduler/optimizer/FaissIVFPQPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFPQPass.cpp @@ -57,16 +57,16 @@ FaissIVFPQPass::Run(const TaskPtr& task) { auto search_job = std::static_pointer_cast(search_task->job_.lock()); ResourcePtr res_ptr; if (!gpu_enable_) { - SERVER_LOG_DEBUG << LogOut("[%s][%d] FaissIVFPQPass: gpu disable, specify cpu to search!", "search", 0); + LOG_SERVER_DEBUG_ << LogOut("[%s][%d] FaissIVFPQPass: gpu disable, specify cpu to search!", "search", 0); res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); } else if (search_job->nq() < threshold_) { - SERVER_LOG_DEBUG << LogOut("[%s][%d] FaissIVFPQPass: nq < gpu_search_threshold, specify cpu to search!", - "search", 0); + LOG_SERVER_DEBUG_ << LogOut("[%s][%d] FaissIVFPQPass: nq < gpu_search_threshold, specify cpu to search!", + "search", 0); res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); } else { auto best_device_id = count_ % search_gpus_.size(); - SERVER_LOG_DEBUG << LogOut("[%s][%d] FaissIVFPQPass: nq > gpu_search_threshold, specify gpu %d to search!", - "search", 0, best_device_id); + LOG_SERVER_DEBUG_ << LogOut("[%s][%d] FaissIVFPQPass: nq > gpu_search_threshold, specify gpu %d to search!", + "search", 0, best_device_id); ++count_; res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, search_gpus_[best_device_id]); } diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp index 5a78089ab31e85afe1825592bb98e754e4574a28..8e99be828ac2d4ce8d3a754db53bb7b5b796a53b 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp @@ -54,17 +54,17 @@ FaissIVFSQ8HPass::Run(const TaskPtr& task) { auto search_job = std::static_pointer_cast(search_task->job_.lock()); ResourcePtr res_ptr; if (!gpu_enable_) { - SERVER_LOG_DEBUG << LogOut("[%s][%d] FaissIVFSQ8HPass: gpu disable, specify cpu to search!", "search", 0); + LOG_SERVER_DEBUG_ << LogOut("[%s][%d] FaissIVFSQ8HPass: gpu disable, specify cpu to search!", "search", 0); res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); } if (search_job->nq() < threshold_) { - SERVER_LOG_DEBUG << LogOut("[%s][%d] FaissIVFSQ8HPass: nq < gpu_search_threshold, specify cpu to search!", - "search", 0); + LOG_SERVER_DEBUG_ << LogOut("[%s][%d] FaissIVFSQ8HPass: nq < gpu_search_threshold, specify cpu to search!", + "search", 0); res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); } else { auto best_device_id = count_ % search_gpus_.size(); - SERVER_LOG_DEBUG << LogOut("[%s][%d] FaissIVFSQ8HPass: nq > gpu_search_threshold, specify gpu %d to search!", - "search", 0, best_device_id); + LOG_SERVER_DEBUG_ << LogOut("[%s][%d] FaissIVFSQ8HPass: nq > gpu_search_threshold, specify gpu %d to search!", + "search", 0, best_device_id); ++count_; res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, search_gpus_[best_device_id]); } diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp index e89fab0fed24d6bb20a1cf56862251ffd184c035..0f19f282b8ac341f389dcd3fb34aff54e1022e7e 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp @@ -55,16 +55,16 @@ FaissIVFSQ8Pass::Run(const TaskPtr& task) { auto search_job = std::static_pointer_cast(search_task->job_.lock()); ResourcePtr res_ptr; if (!gpu_enable_) { - SERVER_LOG_DEBUG << LogOut("[%s][%d] FaissIVFSQ8Pass: gpu disable, specify cpu to search!", "search", 0); + LOG_SERVER_DEBUG_ << LogOut("[%s][%d] FaissIVFSQ8Pass: gpu disable, specify cpu to search!", "search", 0); res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); } else if (search_job->nq() < threshold_) { - SERVER_LOG_DEBUG << LogOut("[%s][%d] FaissIVFSQ8Pass: nq < gpu_search_threshold, specify cpu to search!", - "search", 0); + LOG_SERVER_DEBUG_ << LogOut("[%s][%d] FaissIVFSQ8Pass: nq < gpu_search_threshold, specify cpu to search!", + "search", 0); res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); } else { auto best_device_id = count_ % search_gpus_.size(); - SERVER_LOG_DEBUG << LogOut("[%s][%d] FaissIVFSQ8Pass: nq > gpu_search_threshold, specify gpu %d to search!", - "search", 0, best_device_id); + LOG_SERVER_DEBUG_ << LogOut("[%s][%d] FaissIVFSQ8Pass: nq > gpu_search_threshold, specify gpu %d to search!", + "search", 0, best_device_id); count_++; res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, search_gpus_[best_device_id]); } diff --git a/core/src/scheduler/optimizer/FallbackPass.cpp b/core/src/scheduler/optimizer/FallbackPass.cpp index ff0c2439258ae4aa14e98ac4f301b51f80a1f0e4..281f57cfeb3c308e286a4a1937a4aad1c25acdf3 100644 --- a/core/src/scheduler/optimizer/FallbackPass.cpp +++ b/core/src/scheduler/optimizer/FallbackPass.cpp @@ -27,7 +27,7 @@ FallbackPass::Run(const TaskPtr& task) { return false; } // NEVER be empty - SERVER_LOG_DEBUG << "FallbackPass!"; + LOG_SERVER_DEBUG_ << "FallbackPass!"; auto cpu = ResMgrInst::GetInstance()->GetCpuResources()[0]; auto label = std::make_shared(cpu); task->label() = label; diff --git a/core/src/scheduler/resource/Resource.cpp b/core/src/scheduler/resource/Resource.cpp index d4172362b455fed59db2c4b76a87d3e6127c7887..d74a6ed5b0ecc0b3e31aa89ccf8926384ee69def 100644 --- a/core/src/scheduler/resource/Resource.cpp +++ b/core/src/scheduler/resource/Resource.cpp @@ -153,6 +153,7 @@ Resource::pick_task_execute() { void Resource::loader_function() { + SetThreadName("taskloader_th"); while (running_) { std::unique_lock lock(load_mutex_); load_cv_.wait(lock, [&] { return load_flag_; }); @@ -165,7 +166,7 @@ Resource::loader_function() { } if (task_item->task->Type() == TaskType::BuildIndexTask && name() == "cpu") { BuildMgrInst::GetInstance()->Take(); - SERVER_LOG_DEBUG << name() << " load BuildIndexTask"; + LOG_SERVER_DEBUG_ << name() << " load BuildIndexTask"; } LoadFile(task_item->task); task_item->Loaded(); @@ -183,6 +184,7 @@ Resource::loader_function() { void Resource::executor_function() { + SetThreadName("taskexector_th"); if (subscriber_) { auto event = std::make_shared(shared_from_this()); subscriber_(std::static_pointer_cast(event)); diff --git a/core/src/scheduler/task/BuildIndexTask.cpp b/core/src/scheduler/task/BuildIndexTask.cpp index f258a6b936bb371a587c17444d0b1eb742390c09..fc8dca9d3ea57d50816cf59b21e00fa8d2856ae6 100644 --- a/core/src/scheduler/task/BuildIndexTask.cpp +++ b/core/src/scheduler/task/BuildIndexTask.cpp @@ -136,7 +136,7 @@ XBuildIndexTask::Execute() { fiu_do_on("XBuildIndexTask.Execute.create_table_success", status = Status::OK()); if (!status.ok()) { - ENGINE_LOG_ERROR << "Failed to create collection file: " << status.ToString(); + LOG_ENGINE_ERROR_ << "Failed to create collection file: " << status.ToString(); build_index_job->BuildIndexDone(to_index_id_); build_index_job->GetStatus() = status; to_index_engine_ = nullptr; @@ -146,7 +146,7 @@ XBuildIndexTask::Execute() { auto failed_build_index = [&](std::string log_msg, std::string err_msg) { table_file.file_type_ = engine::meta::SegmentSchema::TO_DELETE; status = meta_ptr->UpdateCollectionFile(table_file); - ENGINE_LOG_ERROR << log_msg; + LOG_ENGINE_ERROR_ << log_msg; build_index_job->BuildIndexDone(to_index_id_); build_index_job->GetStatus() = Status(DB_ERROR, err_msg); @@ -155,7 +155,7 @@ XBuildIndexTask::Execute() { // step 2: build index try { - ENGINE_LOG_DEBUG << "Begin build index for file:" + table_file.location_; + LOG_ENGINE_DEBUG_ << "Begin build index for file:" + table_file.location_; index = to_index_engine_->BuildIndex(table_file.location_, (EngineType)table_file.engine_type_); fiu_do_on("XBuildIndexTask.Execute.build_index_fail", index = nullptr); if (index == nullptr) { @@ -215,9 +215,9 @@ XBuildIndexTask::Execute() { fiu_do_on("XBuildIndexTask.Execute.update_table_file_fail", status = Status(SERVER_UNEXPECTED_ERROR, "")); if (status.ok()) { - ENGINE_LOG_DEBUG << "New index file " << table_file.file_id_ << " of size " << table_file.file_size_ - << " bytes" - << " from file " << origin_file.file_id_; + LOG_ENGINE_DEBUG_ << "New index file " << table_file.file_id_ << " of size " << table_file.file_size_ + << " bytes" + << " from file " << origin_file.file_id_; if (build_index_job->options().insert_cache_immediately_) { index->Cache(); } @@ -225,12 +225,13 @@ XBuildIndexTask::Execute() { // failed to update meta, mark the new file as to_delete, don't delete old file origin_file.file_type_ = engine::meta::SegmentSchema::TO_INDEX; status = meta_ptr->UpdateCollectionFile(origin_file); - ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << origin_file.file_id_ << " to to_index"; + LOG_ENGINE_DEBUG_ << "Failed to update file to index, mark file: " << origin_file.file_id_ + << " to to_index"; table_file.file_type_ = engine::meta::SegmentSchema::TO_DELETE; status = meta_ptr->UpdateCollectionFile(table_file); - ENGINE_LOG_DEBUG << "Failed to up date file to index, mark file: " << table_file.file_id_ - << " to to_delete"; + LOG_ENGINE_DEBUG_ << "Failed to up date file to index, mark file: " << table_file.file_id_ + << " to to_delete"; } build_index_job->BuildIndexDone(to_index_id_); diff --git a/core/src/scheduler/task/SearchTask.cpp b/core/src/scheduler/task/SearchTask.cpp index a60fae68ec3e7a814c711f13895d765fb08a9e70..fdb7f9f7658d2f1e5fe9c26785c0fa4f550145c6 100644 --- a/core/src/scheduler/task/SearchTask.cpp +++ b/core/src/scheduler/task/SearchTask.cpp @@ -57,7 +57,7 @@ static constexpr size_t PARALLEL_REDUCE_BATCH = 1000; // if (thread_count > 0) { // reduce_batch = max_index / thread_count + 1; // } -// ENGINE_LOG_DEBUG << "use " << thread_count << +// LOG_ENGINE_DEBUG_ << "use " << thread_count << // " thread parallelly do reduce, each thread process " << reduce_batch << " vectors"; // // std::vector > thread_array; @@ -159,7 +159,7 @@ XSearchTask::Load(LoadType type, uint8_t device_id) { } catch (std::exception& ex) { // typical error: out of disk space or permition denied error_msg = "Failed to load index file: " + std::string(ex.what()); - ENGINE_LOG_ERROR << LogOut("[%s][%ld] Encounter execption: %s", "search", 0, error_msg.c_str()); + LOG_ENGINE_ERROR_ << LogOut("[%s][%ld] Encounter execption: %s", "search", 0, error_msg.c_str()); stat = Status(SERVER_UNEXPECTED_ERROR, error_msg); } fiu_do_on("XSearchTask.Load.out_of_memory", stat = Status(SERVER_UNEXPECTED_ERROR, "out of memory")); @@ -202,7 +202,7 @@ void XSearchTask::Execute() { milvus::server::ContextFollower tracer(context_, "XSearchTask::Execute " + std::to_string(index_id_)); - // ENGINE_LOG_DEBUG << "Searching in file id:" << index_id_ << " with " + // LOG_ENGINE_DEBUG_ << "Searching in file id:" << index_id_ << " with " // << search_contexts_.size() << " tasks"; // TimeRecorder rc("DoSearch file id:" + std::to_string(index_id_)); @@ -266,8 +266,8 @@ XSearchTask::Execute() { // step 3: pick up topk result auto spec_k = file_->row_count_ < topk ? file_->row_count_ : topk; if (spec_k == 0) { - ENGINE_LOG_WARNING << LogOut("[%s][%ld] Searching in an empty file. file location = %s", "search", 0, - file_->location_.c_str()); + LOG_ENGINE_WARNING_ << LogOut("[%s][%ld] Searching in an empty file. file location = %s", "search", 0, + file_->location_.c_str()); } { @@ -288,7 +288,7 @@ XSearchTask::Execute() { span = rc.RecordSection(hdr + ", reduce topk"); // search_job->AccumReduceCost(span); } catch (std::exception& ex) { - ENGINE_LOG_ERROR << LogOut("[%s][%ld] SearchTask encounter exception: %s", "search", 0, ex.what()); + LOG_ENGINE_ERROR_ << LogOut("[%s][%ld] SearchTask encounter exception: %s", "search", 0, ex.what()); // search_job->IndexSearchDone(index_id_);//mark as done avoid dead lock, even search failed } @@ -307,7 +307,7 @@ XSearchTask::MergeTopkToResultSet(const scheduler::ResultIds& src_ids, const sch size_t src_k, size_t nq, size_t topk, bool ascending, scheduler::ResultIds& tar_ids, scheduler::ResultDistances& tar_distances) { if (src_ids.empty()) { - ENGINE_LOG_DEBUG << LogOut("[%s][%d] Search result is empty.", "search", 0); + LOG_ENGINE_DEBUG_ << LogOut("[%s][%d] Search result is empty.", "search", 0); return; } diff --git a/core/src/segment/IdBloomFilter.cpp b/core/src/segment/IdBloomFilter.cpp index ee9835608a0c15f46b3e16cf7fa1db408af173e0..890faa00549b2fb6383c6b2938b09e861f6d8c5c 100644 --- a/core/src/segment/IdBloomFilter.cpp +++ b/core/src/segment/IdBloomFilter.cpp @@ -53,7 +53,7 @@ IdBloomFilter::Add(doc_id_t uid) { const std::lock_guard lock(mutex_); if (scaling_bloom_add(bloom_filter_, s.c_str(), s.size(), uid) == -1) { // Counter overflow does not affect bloom filter's normal functionality - ENGINE_LOG_WARNING << "Warning adding id=" << s << " to bloom filter: 4 bit counter Overflow"; + LOG_ENGINE_WARNING_ << "Warning adding id=" << s << " to bloom filter: 4 bit counter Overflow"; // return Status(DB_BLOOM_FILTER_ERROR, "Bloom filter error: 4 bit counter Overflow"); } return Status::OK(); @@ -65,7 +65,7 @@ IdBloomFilter::Remove(doc_id_t uid) { const std::lock_guard lock(mutex_); if (scaling_bloom_remove(bloom_filter_, s.c_str(), s.size(), uid) == -1) { // Should never go in here, but just to be safe - ENGINE_LOG_WARNING << "Warning removing id=" << s << " in bloom filter: Decrementing zero in counter"; + LOG_ENGINE_WARNING_ << "Warning removing id=" << s << " in bloom filter: Decrementing zero in counter"; // return Status(DB_BLOOM_FILTER_ERROR, "Error removing in bloom filter: Decrementing zero in counter"); } return Status::OK(); diff --git a/core/src/segment/SegmentReader.cpp b/core/src/segment/SegmentReader.cpp index 384be3af91d468735768b7895f7379e5c2c225f7..fe0bb62e7f14737473648d2d4d3ee0e03bbd6fab 100644 --- a/core/src/segment/SegmentReader.cpp +++ b/core/src/segment/SegmentReader.cpp @@ -66,7 +66,7 @@ SegmentReader::LoadVectors(off_t offset, size_t num_bytes, std::vector& default_codec.GetVectorsFormat()->read_vectors(fs_ptr_, offset, num_bytes, raw_vectors); } catch (std::exception& e) { std::string err_msg = "Failed to load raw vectors: " + std::string(e.what()); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; return Status(DB_ERROR, err_msg); } return Status::OK(); @@ -80,7 +80,7 @@ SegmentReader::LoadUids(std::vector& uids) { default_codec.GetVectorsFormat()->read_uids(fs_ptr_, uids); } catch (std::exception& e) { std::string err_msg = "Failed to load uids: " + std::string(e.what()); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; return Status(DB_ERROR, err_msg); } return Status::OK(); @@ -100,7 +100,7 @@ SegmentReader::LoadVectorIndex(const std::string& location, segment::VectorIndex default_codec.GetVectorIndexFormat()->read(fs_ptr_, location, vector_index_ptr); } catch (std::exception& e) { std::string err_msg = "Failed to load vector index: " + std::string(e.what()); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; return Status(DB_ERROR, err_msg); } return Status::OK(); @@ -114,7 +114,7 @@ SegmentReader::LoadBloomFilter(segment::IdBloomFilterPtr& id_bloom_filter_ptr) { default_codec.GetIdBloomFilterFormat()->read(fs_ptr_, id_bloom_filter_ptr); } catch (std::exception& e) { std::string err_msg = "Failed to load bloom filter: " + std::string(e.what()); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; return Status(DB_ERROR, err_msg); } return Status::OK(); @@ -128,7 +128,7 @@ SegmentReader::LoadDeletedDocs(segment::DeletedDocsPtr& deleted_docs_ptr) { default_codec.GetDeletedDocsFormat()->read(fs_ptr_, deleted_docs_ptr); } catch (std::exception& e) { std::string err_msg = "Failed to load deleted docs: " + std::string(e.what()); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; return Status(DB_ERROR, err_msg); } return Status::OK(); @@ -142,7 +142,7 @@ SegmentReader::ReadDeletedDocsSize(size_t& size) { default_codec.GetDeletedDocsFormat()->readSize(fs_ptr_, size); } catch (std::exception& e) { std::string err_msg = "Failed to read deleted docs size: " + std::string(e.what()); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; return Status(DB_ERROR, err_msg); } return Status::OK(); diff --git a/core/src/segment/SegmentWriter.cpp b/core/src/segment/SegmentWriter.cpp index 2c650d4a1953b0678438f926f77b3eb313a1c5d0..e774f67347cad61a1200112b67bbf78edeef9ccd 100644 --- a/core/src/segment/SegmentWriter.cpp +++ b/core/src/segment/SegmentWriter.cpp @@ -62,7 +62,7 @@ SegmentWriter::Serialize() { auto status = WriteBloomFilter(); if (!status.ok()) { - ENGINE_LOG_ERROR << status.message(); + LOG_ENGINE_ERROR_ << status.message(); return status; } @@ -70,7 +70,7 @@ SegmentWriter::Serialize() { status = WriteVectors(); if (!status.ok()) { - ENGINE_LOG_ERROR << "Write vectors fail: " << status.message(); + LOG_ENGINE_ERROR_ << "Write vectors fail: " << status.message(); return status; } @@ -92,7 +92,7 @@ SegmentWriter::WriteVectors() { default_codec.GetVectorsFormat()->write(fs_ptr_, segment_ptr_->vectors_ptr_); } catch (std::exception& e) { std::string err_msg = "Failed to write vectors: " + std::string(e.what()); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; return Status(SERVER_WRITE_ERROR, err_msg); } return Status::OK(); @@ -106,7 +106,7 @@ SegmentWriter::WriteVectorIndex(const std::string& location) { default_codec.GetVectorIndexFormat()->write(fs_ptr_, location, segment_ptr_->vector_index_ptr_); } catch (std::exception& e) { std::string err_msg = "Failed to write vector index: " + std::string(e.what()); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; return Status(SERVER_WRITE_ERROR, err_msg); } return Status::OK(); @@ -136,7 +136,7 @@ SegmentWriter::WriteBloomFilter() { recorder.RecordSection("Writing bloom filter"); } catch (std::exception& e) { std::string err_msg = "Failed to write vectors: " + std::string(e.what()); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; return Status(SERVER_WRITE_ERROR, err_msg); } return Status::OK(); @@ -151,7 +151,7 @@ SegmentWriter::WriteDeletedDocs() { default_codec.GetDeletedDocsFormat()->write(fs_ptr_, deleted_docs_ptr); } catch (std::exception& e) { std::string err_msg = "Failed to write deleted docs: " + std::string(e.what()); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; return Status(SERVER_WRITE_ERROR, err_msg); } return Status::OK(); @@ -165,7 +165,7 @@ SegmentWriter::WriteDeletedDocs(const DeletedDocsPtr& deleted_docs) { default_codec.GetDeletedDocsFormat()->write(fs_ptr_, deleted_docs); } catch (std::exception& e) { std::string err_msg = "Failed to write deleted docs: " + std::string(e.what()); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; return Status(SERVER_WRITE_ERROR, err_msg); } return Status::OK(); @@ -179,7 +179,7 @@ SegmentWriter::WriteBloomFilter(const IdBloomFilterPtr& id_bloom_filter_ptr) { default_codec.GetIdBloomFilterFormat()->write(fs_ptr_, id_bloom_filter_ptr); } catch (std::exception& e) { std::string err_msg = "Failed to write bloom filter: " + std::string(e.what()); - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; return Status(SERVER_WRITE_ERROR, err_msg); } return Status::OK(); @@ -203,7 +203,7 @@ SegmentWriter::Merge(const std::string& dir_to_merge, const std::string& name) { return Status(DB_ERROR, "Cannot Merge Self"); } - ENGINE_LOG_DEBUG << "Merging from " << dir_to_merge << " to " << fs_ptr_->operation_ptr_->GetDirectory(); + LOG_ENGINE_DEBUG_ << "Merging from " << dir_to_merge << " to " << fs_ptr_->operation_ptr_->GetDirectory(); TimeRecorder recorder("SegmentWriter::Merge"); @@ -214,7 +214,7 @@ SegmentWriter::Merge(const std::string& dir_to_merge, const std::string& name) { status = segment_reader_to_merge.Load(); if (!status.ok()) { std::string msg = "Failed to load segment from " + dir_to_merge; - ENGINE_LOG_ERROR << msg; + LOG_ENGINE_ERROR_ << msg; return Status(DB_ERROR, msg); } } @@ -238,7 +238,7 @@ SegmentWriter::Merge(const std::string& dir_to_merge, const std::string& name) { auto rows = segment_to_merge->vectors_ptr_->GetCount(); recorder.RecordSection("Adding " + std::to_string(rows) + " vectors and uids"); - ENGINE_LOG_DEBUG << "Merging completed from " << dir_to_merge << " to " << fs_ptr_->operation_ptr_->GetDirectory(); + LOG_ENGINE_DEBUG_ << "Merging completed from " << dir_to_merge << " to " << fs_ptr_->operation_ptr_->GetDirectory(); return Status::OK(); } diff --git a/core/src/segment/Vectors.cpp b/core/src/segment/Vectors.cpp index 67b6ec4c31d5f53c2363818b8ea5c9550c5da83d..a8555089aff0e6cbec46fab6644d1fd20a878ebc 100644 --- a/core/src/segment/Vectors.cpp +++ b/core/src/segment/Vectors.cpp @@ -72,7 +72,7 @@ Vectors::Erase(std::vector& offsets) { recorder.RecordSection("Deduplicating " + std::to_string(offsets.size()) + " offsets to delete"); // Reconstruct raw vectors and uids - ENGINE_LOG_DEBUG << "Begin erasing..."; + LOG_ENGINE_DEBUG_ << "Begin erasing..."; size_t new_size = uids_.size() - offsets.size(); std::vector new_uids(new_size); diff --git a/core/src/server/DBWrapper.cpp b/core/src/server/DBWrapper.cpp index 0b7dafa594abcebb3a587b3d4445a2efec7e78e0..26672f4d0b49f02144b37b579aeb5e6bc6c57bb9 100644 --- a/core/src/server/DBWrapper.cpp +++ b/core/src/server/DBWrapper.cpp @@ -131,7 +131,7 @@ DBWrapper::StartService() { if (omp_thread > 0) { omp_set_num_threads(omp_thread); - SERVER_LOG_DEBUG << "Specify openmp thread number: " << omp_thread; + LOG_SERVER_DEBUG_ << "Specify openmp thread number: " << omp_thread; } else { int64_t sys_thread_cnt = 8; if (CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) { diff --git a/core/src/server/Server.cpp b/core/src/server/Server.cpp index 63eb25d5d9f3baf058badd707e1a59285d129b5c..aac78f84e28ebd15fbc2ed618e11aa1a5285d4db 100644 --- a/core/src/server/Server.cpp +++ b/core/src/server/Server.cpp @@ -58,7 +58,7 @@ Server::Daemonize() { // std::string log_path(GetLogDirFullPath()); // log_path += "zdb_server.(INFO/WARNNING/ERROR/CRITICAL)"; - // SERVER_LOG_INFO << "Log will be exported to: " + log_path); + // LOG_SERVER_INFO_ << "Log will be exported to: " + log_path); pid_t pid = 0; @@ -187,11 +187,11 @@ Server::Start() { InitLog(log_config_file_); // print version information - SERVER_LOG_INFO << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << ", built at " << BUILD_TIME; + LOG_SERVER_INFO_ << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << ", built at " << BUILD_TIME; #ifdef MILVUS_GPU_VERSION - SERVER_LOG_INFO << "GPU edition"; + LOG_SERVER_INFO_ << "GPU edition"; #else - SERVER_LOG_INFO << "CPU edition"; + LOG_SERVER_INFO_ << "CPU edition"; #endif /* record config and hardware information into log */ LogConfigInFile(config_filename_); @@ -262,7 +262,7 @@ Server::StartService() { Status stat; stat = engine::KnowhereResource::Initialize(); if (!stat.ok()) { - SERVER_LOG_ERROR << "KnowhereResource initialize fail: " << stat.message(); + LOG_SERVER_ERROR_ << "KnowhereResource initialize fail: " << stat.message(); goto FAIL; } @@ -270,7 +270,7 @@ Server::StartService() { stat = DBWrapper::GetInstance().StartService(); if (!stat.ok()) { - SERVER_LOG_ERROR << "DBWrapper start service fail: " << stat.message(); + LOG_SERVER_ERROR_ << "DBWrapper start service fail: " << stat.message(); goto FAIL; } @@ -279,7 +279,7 @@ Server::StartService() { // stat = storage::S3ClientWrapper::GetInstance().StartService(); // if (!stat.ok()) { - // SERVER_LOG_ERROR << "S3Client start service fail: " << stat.message(); + // LOG_SERVER_ERROR_ << "S3Client start service fail: " << stat.message(); // goto FAIL; // } diff --git a/core/src/server/delivery/RequestScheduler.cpp b/core/src/server/delivery/RequestScheduler.cpp index 491293edd63427bf399e4576b821410c4df178a7..264c7f141931d0ce59bc467bc3e0b5961b964e4f 100644 --- a/core/src/server/delivery/RequestScheduler.cpp +++ b/core/src/server/delivery/RequestScheduler.cpp @@ -53,7 +53,7 @@ RequestScheduler::Stop() { return; } - SERVER_LOG_INFO << "Scheduler gonna stop..."; + LOG_SERVER_INFO_ << "Scheduler gonna stop..."; { std::lock_guard lock(queue_mtx_); for (auto& iter : request_groups_) { @@ -71,7 +71,7 @@ RequestScheduler::Stop() { request_groups_.clear(); execute_threads_.clear(); stopped_ = true; - SERVER_LOG_INFO << "Scheduler stopped"; + LOG_SERVER_INFO_ << "Scheduler stopped"; } Status @@ -90,7 +90,7 @@ RequestScheduler::ExecuteRequest(const BaseRequestPtr& request_ptr) { fiu_do_on("RequestScheduler.ExecuteRequest.push_queue_fail", status = Status(SERVER_INVALID_ARGUMENT, "")); if (!status.ok()) { - SERVER_LOG_ERROR << "Put request to queue failed with code: " << status.ToString(); + LOG_SERVER_ERROR_ << "Put request to queue failed with code: " << status.ToString(); request_ptr->Done(); return status; } @@ -109,6 +109,7 @@ RequestScheduler::ExecuteRequest(const BaseRequestPtr& request_ptr) { void RequestScheduler::TakeToExecute(RequestQueuePtr request_queue) { + SetThreadName("reqsched_thread"); if (request_queue == nullptr) { return; } @@ -116,7 +117,7 @@ RequestScheduler::TakeToExecute(RequestQueuePtr request_queue) { while (true) { BaseRequestPtr request = request_queue->TakeRequest(); if (request == nullptr) { - SERVER_LOG_ERROR << "Take null from request queue, stop thread"; + LOG_SERVER_ERROR_ << "Take null from request queue, stop thread"; break; // stop the thread } @@ -126,10 +127,10 @@ RequestScheduler::TakeToExecute(RequestQueuePtr request_queue) { fiu_do_on("RequestScheduler.TakeToExecute.throw_std_exception", throw std::exception()); fiu_do_on("RequestScheduler.TakeToExecute.execute_fail", status = Status(SERVER_INVALID_ARGUMENT, "")); if (!status.ok()) { - SERVER_LOG_ERROR << "Request failed with code: " << status.ToString(); + LOG_SERVER_ERROR_ << "Request failed with code: " << status.ToString(); } } catch (std::exception& ex) { - SERVER_LOG_ERROR << "Request failed to execute: " << ex.what(); + LOG_SERVER_ERROR_ << "Request failed to execute: " << ex.what(); } } } @@ -152,7 +153,7 @@ RequestScheduler::PutToQueue(const BaseRequestPtr& request_ptr) { fiu_do_on("RequestScheduler.PutToQueue.push_null_thread", execute_threads_.push_back(nullptr)); execute_threads_.push_back(thread); - SERVER_LOG_INFO << "Create new thread for request group: " << group_name; + LOG_SERVER_INFO_ << "Create new thread for request group: " << group_name; } return Status::OK(); diff --git a/core/src/server/delivery/request/BaseRequest.cpp b/core/src/server/delivery/request/BaseRequest.cpp index ade8e6488c14d0ce95cbbd3428e3fa39c7a724ca..16d99dad911fc7a06f7a1def0d24c3258de9b62c 100644 --- a/core/src/server/delivery/request/BaseRequest.cpp +++ b/core/src/server/delivery/request/BaseRequest.cpp @@ -66,7 +66,7 @@ RequestGroup(BaseRequest::RequestType type) { auto iter = s_map_type_group.find(type); if (iter == s_map_type_group.end()) { - SERVER_LOG_ERROR << "Unsupported request type: " << type; + LOG_SERVER_ERROR_ << "Unsupported request type: " << type; throw Exception(SERVER_NOT_IMPLEMENT, "request group undefined"); } return iter->second; @@ -125,7 +125,7 @@ void BaseRequest::set_status(const Status& status) { status_ = status; if (!status_.ok()) { - SERVER_LOG_ERROR << status_.message(); + LOG_SERVER_ERROR_ << status_.message(); } } diff --git a/core/src/server/delivery/request/DeleteByIDRequest.cpp b/core/src/server/delivery/request/DeleteByIDRequest.cpp index 9bc3399a563284aea5262a083abf8666654d11af..5d4603d876bffb47cfb747c3266ec0409169c7e5 100644 --- a/core/src/server/delivery/request/DeleteByIDRequest.cpp +++ b/core/src/server/delivery/request/DeleteByIDRequest.cpp @@ -72,7 +72,7 @@ DeleteByIDRequest::OnExecute() { collection_schema.engine_type_ == (int32_t)engine::EngineType::SPTAG_KDT) { std::string err_msg = "Index type " + std::to_string(collection_schema.engine_type_) + " does not support delete operation"; - SERVER_LOG_ERROR << err_msg; + LOG_SERVER_ERROR_ << err_msg; return Status(SERVER_UNSUPPORTED_ERROR, err_msg); } diff --git a/core/src/server/delivery/request/DropPartitionRequest.cpp b/core/src/server/delivery/request/DropPartitionRequest.cpp index c85f7ca7beaa2af6cf6f289a5cb1efbdc282741a..1eda9bd3859ec97dd7ec0525d004781619fdd40f 100644 --- a/core/src/server/delivery/request/DropPartitionRequest.cpp +++ b/core/src/server/delivery/request/DropPartitionRequest.cpp @@ -52,7 +52,7 @@ DropPartitionRequest::OnExecute() { // step 2: check partition tag if (partition_tag == milvus::engine::DEFAULT_PARTITON_TAG) { std::string msg = "Default partition cannot be dropped."; - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_COLLECTION_NAME, msg); } diff --git a/core/src/server/delivery/request/FlushRequest.cpp b/core/src/server/delivery/request/FlushRequest.cpp index c9d0a0f8c85831685c7ab4317f3c286bc45ab570..2f0fd7c6ef771f5caa8e744300b0d552d6aac630 100644 --- a/core/src/server/delivery/request/FlushRequest.cpp +++ b/core/src/server/delivery/request/FlushRequest.cpp @@ -47,7 +47,7 @@ FlushRequest::OnExecute() { TimeRecorderAuto rc(hdr); Status status = Status::OK(); - SERVER_LOG_DEBUG << hdr; + LOG_SERVER_DEBUG_ << hdr; for (auto& name : collection_names_) { // only process root collection, ignore partition collection diff --git a/core/src/server/delivery/request/InsertRequest.cpp b/core/src/server/delivery/request/InsertRequest.cpp index 7dc84bf74fa4c48e6ae1fb0c9230e6a867d4e66f..cc19e1e69eaca4be4534db0dececc5fb270cec0d 100644 --- a/core/src/server/delivery/request/InsertRequest.cpp +++ b/core/src/server/delivery/request/InsertRequest.cpp @@ -46,7 +46,7 @@ InsertRequest::Create(const std::shared_ptr& context, c Status InsertRequest::OnExecute() { - SERVER_LOG_INFO << LogOut("[%s][%ld] ", "insert", 0) << "Execute insert request."; + LOG_SERVER_INFO_ << LogOut("[%s][%ld] ", "insert", 0) << "Execute insert request."; try { int64_t vector_count = vectors_data_.vector_count_; fiu_do_on("InsertRequest.OnExecute.throw_std_exception", throw std::exception()); @@ -57,12 +57,12 @@ InsertRequest::OnExecute() { // step 1: check arguments auto status = ValidationUtil::ValidateCollectionName(collection_name_); if (!status.ok()) { - SERVER_LOG_ERROR << LogOut("[%s][%ld] Invalid collection name: %s", "insert", 0, status.message().c_str()); + LOG_SERVER_ERROR_ << LogOut("[%s][%ld] Invalid collection name: %s", "insert", 0, status.message().c_str()); return status; } if (vectors_data_.float_data_.empty() && vectors_data_.binary_data_.empty()) { std::string msg = "The vector array is empty. Make sure you have entered vector records."; - SERVER_LOG_ERROR << LogOut("[%s][%ld] Invalid records: %s", "insert", 0, msg.c_str()); + LOG_SERVER_ERROR_ << LogOut("[%s][%ld] Invalid records: %s", "insert", 0, msg.c_str()); return Status(SERVER_INVALID_ROWRECORD_ARRAY, msg); } @@ -70,7 +70,7 @@ InsertRequest::OnExecute() { if (!vectors_data_.id_array_.empty()) { if (vectors_data_.id_array_.size() != vector_count) { std::string msg = "The size of vector ID array must be equal to the size of the vector."; - SERVER_LOG_ERROR << LogOut("[%s][%ld] Invalid id array: %s", "insert", 0, msg.c_str()); + LOG_SERVER_ERROR_ << LogOut("[%s][%ld] Invalid id array: %s", "insert", 0, msg.c_str()); return Status(SERVER_ILLEGAL_VECTOR_ID, msg); } } @@ -85,17 +85,17 @@ InsertRequest::OnExecute() { status = Status(milvus::SERVER_UNEXPECTED_ERROR, "")); if (!status.ok()) { if (status.code() == DB_NOT_FOUND) { - SERVER_LOG_ERROR << LogOut("[%s][%ld] Collection %s not found", "insert", 0, collection_name_.c_str()); + LOG_SERVER_ERROR_ << LogOut("[%s][%ld] Collection %s not found", "insert", 0, collection_name_.c_str()); return Status(SERVER_COLLECTION_NOT_EXIST, CollectionNotExistMsg(collection_name_)); } else { - SERVER_LOG_ERROR << LogOut("[%s][%ld] Describe collection %s fail: %s", "insert", 0, - collection_name_.c_str(), status.message().c_str()); + LOG_SERVER_ERROR_ << LogOut("[%s][%ld] Describe collection %s fail: %s", "insert", 0, + collection_name_.c_str(), status.message().c_str()); return status; } } else { if (!collection_schema.owner_collection_.empty()) { - SERVER_LOG_ERROR << LogOut("[%s][%ld] owner collection of %s is empty", "insert", 0, - collection_name_.c_str()); + LOG_SERVER_ERROR_ << LogOut("[%s][%ld] owner collection of %s is empty", "insert", 0, + collection_name_.c_str()); return Status(SERVER_INVALID_COLLECTION_NAME, CollectionNotExistMsg(collection_name_)); } } @@ -108,7 +108,7 @@ InsertRequest::OnExecute() { // user already provided id before, all insert action require user id if ((collection_schema.flag_ & engine::meta::FLAG_MASK_HAS_USERID) != 0 && !user_provide_ids) { std::string msg = "Entities IDs are user-defined. Please provide IDs for all entities of the collection."; - SERVER_LOG_ERROR << LogOut("[%s][%ld] %s", "insert", 0, msg.c_str()); + LOG_SERVER_ERROR_ << LogOut("[%s][%ld] %s", "insert", 0, msg.c_str()); return Status(SERVER_ILLEGAL_VECTOR_ID, msg); } @@ -131,40 +131,40 @@ InsertRequest::OnExecute() { if (!vectors_data_.float_data_.empty()) { // insert float vectors if (engine::utils::IsBinaryMetricType(collection_schema.metric_type_)) { std::string msg = "Collection metric type doesn't support float vectors."; - SERVER_LOG_ERROR << LogOut("[%s][%ld] %s", "insert", 0, msg.c_str()); + LOG_SERVER_ERROR_ << LogOut("[%s][%ld] %s", "insert", 0, msg.c_str()); return Status(SERVER_INVALID_ROWRECORD_ARRAY, msg); } // check prepared float data if (vectors_data_.float_data_.size() % vector_count != 0) { std::string msg = "The vector dimension must be equal to the collection dimension."; - SERVER_LOG_ERROR << LogOut("[%s][%ld] %s", "insert", 0, msg.c_str()); + LOG_SERVER_ERROR_ << LogOut("[%s][%ld] %s", "insert", 0, msg.c_str()); return Status(SERVER_INVALID_ROWRECORD_ARRAY, msg); } fiu_do_on("InsertRequest.OnExecute.invalid_dim", collection_schema.dimension_ = -1); if (vectors_data_.float_data_.size() / vector_count != collection_schema.dimension_) { std::string msg = "The vector dimension must be equal to the collection dimension."; - SERVER_LOG_ERROR << LogOut("[%s][%ld] %s", "insert", 0, msg.c_str()); + LOG_SERVER_ERROR_ << LogOut("[%s][%ld] %s", "insert", 0, msg.c_str()); return Status(SERVER_INVALID_VECTOR_DIMENSION, msg); } } else if (!vectors_data_.binary_data_.empty()) { // insert binary vectors if (!engine::utils::IsBinaryMetricType(collection_schema.metric_type_)) { std::string msg = "Collection metric type doesn't support binary vectors."; - SERVER_LOG_ERROR << LogOut("[%s][%ld] %s", "insert", 0, msg.c_str()); + LOG_SERVER_ERROR_ << LogOut("[%s][%ld] %s", "insert", 0, msg.c_str()); return Status(SERVER_INVALID_ROWRECORD_ARRAY, msg); } // check prepared binary data if (vectors_data_.binary_data_.size() % vector_count != 0) { std::string msg = "The vector dimension must be equal to the collection dimension."; - SERVER_LOG_ERROR << LogOut("[%s][%ld] %s", "insert", 0, msg.c_str()); + LOG_SERVER_ERROR_ << LogOut("[%s][%ld] %s", "insert", 0, msg.c_str()); return Status(SERVER_INVALID_ROWRECORD_ARRAY, msg); } if (vectors_data_.binary_data_.size() * 8 / vector_count != collection_schema.dimension_) { std::string msg = "The vector dimension must be equal to the collection dimension."; - SERVER_LOG_ERROR << LogOut("[%s][%ld] %s", "insert", 0, msg.c_str()); + LOG_SERVER_ERROR_ << LogOut("[%s][%ld] %s", "insert", 0, msg.c_str()); return Status(SERVER_INVALID_VECTOR_DIMENSION, msg); } } @@ -176,7 +176,7 @@ InsertRequest::OnExecute() { status = DBWrapper::DB()->InsertVectors(collection_name_, partition_tag_, vectors_data_); fiu_do_on("InsertRequest.OnExecute.insert_fail", status = Status(milvus::SERVER_UNEXPECTED_ERROR, "")); if (!status.ok()) { - SERVER_LOG_ERROR << LogOut("[%s][%ld] Insert fail: %s", "insert", 0, status.message().c_str()); + LOG_SERVER_ERROR_ << LogOut("[%s][%ld] Insert fail: %s", "insert", 0, status.message().c_str()); return status; } @@ -185,7 +185,7 @@ InsertRequest::OnExecute() { if (ids_size != vec_count) { std::string msg = "Add " + std::to_string(vec_count) + " vectors but only return " + std::to_string(ids_size) + " id"; - SERVER_LOG_ERROR << LogOut("[%s][%ld] Insert fail: %s", "insert", 0, msg.c_str()); + LOG_SERVER_ERROR_ << LogOut("[%s][%ld] Insert fail: %s", "insert", 0, msg.c_str()); return Status(SERVER_ILLEGAL_VECTOR_ID, msg); } @@ -201,7 +201,7 @@ InsertRequest::OnExecute() { rc.RecordSection("add vectors to engine"); rc.ElapseFromBegin("total cost"); } catch (std::exception& ex) { - SERVER_LOG_ERROR << LogOut("[%s][%ld] Encounter exception: %s", "insert", 0, ex.what()); + LOG_SERVER_ERROR_ << LogOut("[%s][%ld] Encounter exception: %s", "insert", 0, ex.what()); return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } diff --git a/core/src/server/delivery/request/SearchByIDRequest.cpp b/core/src/server/delivery/request/SearchByIDRequest.cpp index cf1e82001690faeb63c099a7a47563f83c7db4f4..b5d4360324a9ef82d2afae749ae5982afb460e5c 100644 --- a/core/src/server/delivery/request/SearchByIDRequest.cpp +++ b/core/src/server/delivery/request/SearchByIDRequest.cpp @@ -111,7 +111,7 @@ SearchByIDRequest::OnExecute() { config.GetGpuResourceConfigSearchResources(search_resources); if (!search_resources.empty()) { std::string err_msg = "SearchByID cannot be executed on GPU"; - SERVER_LOG_ERROR << err_msg; + LOG_SERVER_ERROR_ << err_msg; return Status(SERVER_UNSUPPORTED_ERROR, err_msg); } } @@ -125,7 +125,7 @@ SearchByIDRequest::OnExecute() { collection_schema.engine_type_ != (int32_t)engine::EngineType::FAISS_IVFSQ8) { std::string err_msg = "Index type " + std::to_string(collection_schema.engine_type_) + " does not support SearchByID operation"; - SERVER_LOG_ERROR << err_msg; + LOG_SERVER_ERROR_ << err_msg; return Status(SERVER_UNSUPPORTED_ERROR, err_msg); } diff --git a/core/src/server/delivery/request/SearchCombineRequest.cpp b/core/src/server/delivery/request/SearchCombineRequest.cpp index dc8578827c4651644043f3967a0ab7d9becbe981..996cfaecc35184285e21e597ce27f1874ad23f56 100644 --- a/core/src/server/delivery/request/SearchCombineRequest.cpp +++ b/core/src/server/delivery/request/SearchCombineRequest.cpp @@ -233,8 +233,8 @@ Status SearchCombineRequest::OnExecute() { try { size_t combined_request = request_list_.size(); - SERVER_LOG_DEBUG << "SearchCombineRequest execute, request count=" << combined_request - << ", extra_params=" << extra_params_.dump(); + LOG_SERVER_DEBUG_ << "SearchCombineRequest execute, request count=" << combined_request + << ", extra_params=" << extra_params_.dump(); std::string hdr = "SearchCombineRequest(collection=" + collection_name_ + ")"; TimeRecorderAuto rc(hdr); @@ -309,12 +309,12 @@ SearchCombineRequest::OnExecute() { // all requests are skipped if (request_list_.empty()) { - SERVER_LOG_DEBUG << "all combined requests were skipped"; + LOG_SERVER_DEBUG_ << "all combined requests were skipped"; return Status::OK(); } - SERVER_LOG_DEBUG << (combined_request - run_request) << " requests were skipped"; - SERVER_LOG_DEBUG << "reset topk to " << search_topk_; + LOG_SERVER_DEBUG_ << (combined_request - run_request) << " requests were skipped"; + LOG_SERVER_DEBUG_ << "reset topk to " << search_topk_; rc.RecordSection("check validation"); // step 3: construct vectors_data @@ -348,7 +348,7 @@ SearchCombineRequest::OnExecute() { } } - SERVER_LOG_DEBUG << total_count << " query vectors combined"; + LOG_SERVER_DEBUG_ << total_count << " query vectors combined"; rc.RecordSection("combined query vectors"); // step 4: search vectors diff --git a/core/src/server/delivery/request/SearchRequest.cpp b/core/src/server/delivery/request/SearchRequest.cpp index bf3f4c3462829afcada4560b111fa09da0a5edf3..5836d90b06446b7751389f383fe517cf917cadf5 100644 --- a/core/src/server/delivery/request/SearchRequest.cpp +++ b/core/src/server/delivery/request/SearchRequest.cpp @@ -54,7 +54,7 @@ SearchRequest::Create(const std::shared_ptr& context, c Status SearchRequest::OnPreExecute() { - SERVER_LOG_INFO << LogOut("[%s][%ld] ", "search", 0) << "Search pre-execute. Check search parameters"; + LOG_SERVER_INFO_ << LogOut("[%s][%ld] ", "search", 0) << "Search pre-execute. Check search parameters"; std::string hdr = "SearchRequest pre-execute(collection=" + collection_name_ + ")"; TimeRecorderAuto rc(LogOut("[%s][%ld] %s", "search", 0, hdr.c_str())); @@ -62,14 +62,14 @@ SearchRequest::OnPreExecute() { // step 1: check collection name auto status = ValidationUtil::ValidateCollectionName(collection_name_); if (!status.ok()) { - SERVER_LOG_ERROR << LogOut("[%s][%d] %s", "search", 0, status.message().c_str()); + LOG_SERVER_ERROR_ << LogOut("[%s][%d] %s", "search", 0, status.message().c_str()); return status; } // step 2: check search topk status = ValidationUtil::ValidateSearchTopk(topk_); if (!status.ok()) { - SERVER_LOG_ERROR << LogOut("[%s][%d] %s", "search", 0, status.message().c_str()); + LOG_SERVER_ERROR_ << LogOut("[%s][%d] %s", "search", 0, status.message().c_str()); return status; } @@ -77,7 +77,7 @@ SearchRequest::OnPreExecute() { status = ValidationUtil::ValidatePartitionTags(partition_list_); fiu_do_on("SearchRequest.OnExecute.invalid_partition_tags", status = Status(milvus::SERVER_UNEXPECTED_ERROR, "")); if (!status.ok()) { - SERVER_LOG_ERROR << LogOut("[%s][%d] %s", "search", 0, status.message().c_str()); + LOG_SERVER_ERROR_ << LogOut("[%s][%d] %s", "search", 0, status.message().c_str()); return status; } @@ -86,7 +86,7 @@ SearchRequest::OnPreExecute() { Status SearchRequest::OnExecute() { - SERVER_LOG_INFO << LogOut("[%s][%ld] ", "search", 0) << "Search execute."; + LOG_SERVER_INFO_ << LogOut("[%s][%ld] ", "search", 0) << "Search execute."; try { uint64_t vector_count = vectors_data_.vector_count_; fiu_do_on("SearchRequest.OnExecute.throw_std_exception", throw std::exception()); @@ -103,17 +103,18 @@ SearchRequest::OnExecute() { status = Status(milvus::SERVER_UNEXPECTED_ERROR, "")); if (!status.ok()) { if (status.code() == DB_NOT_FOUND) { - SERVER_LOG_ERROR << LogOut("[%s][%d] Collection %s not found: %s", "search", 0, - collection_name_.c_str(), status.message().c_str()); + LOG_SERVER_ERROR_ << LogOut("[%s][%d] Collection %s not found: %s", "search", 0, + collection_name_.c_str(), status.message().c_str()); return Status(SERVER_COLLECTION_NOT_EXIST, CollectionNotExistMsg(collection_name_)); } else { - SERVER_LOG_ERROR << LogOut("[%s][%d] Error occurred when describing collection %s: %s", "search", 0, - collection_name_.c_str(), status.message().c_str()); + LOG_SERVER_ERROR_ << LogOut("[%s][%d] Error occurred when describing collection %s: %s", "search", 0, + collection_name_.c_str(), status.message().c_str()); return status; } } else { if (!collection_schema_.owner_collection_.empty()) { - SERVER_LOG_ERROR << LogOut("[%s][%d] %s", "search", 0, CollectionNotExistMsg(collection_name_).c_str()); + LOG_SERVER_ERROR_ << LogOut("[%s][%d] %s", "search", 0, + CollectionNotExistMsg(collection_name_).c_str()); return Status(SERVER_INVALID_COLLECTION_NAME, CollectionNotExistMsg(collection_name_)); } } @@ -121,14 +122,14 @@ SearchRequest::OnExecute() { // step 5: check search parameters status = ValidationUtil::ValidateSearchParams(extra_params_, collection_schema_, topk_); if (!status.ok()) { - SERVER_LOG_ERROR << LogOut("[%s][%d] Invalid search params: %s", "search", 0, status.message().c_str()); + LOG_SERVER_ERROR_ << LogOut("[%s][%d] Invalid search params: %s", "search", 0, status.message().c_str()); return status; } // step 6: check vector data according to metric type status = ValidationUtil::ValidateVectorData(vectors_data_, collection_schema_); if (!status.ok()) { - SERVER_LOG_ERROR << LogOut("[%s][%d] Invalid vector data: %s", "search", 0, status.message().c_str()); + LOG_SERVER_ERROR_ << LogOut("[%s][%d] Invalid vector data: %s", "search", 0, status.message().c_str()); return status; } @@ -158,7 +159,7 @@ SearchRequest::OnExecute() { #endif fiu_do_on("SearchRequest.OnExecute.query_fail", status = Status(milvus::SERVER_UNEXPECTED_ERROR, "")); if (!status.ok()) { - SERVER_LOG_ERROR << LogOut("[%s][%d] Query fail: %s", "search", 0, status.message().c_str()); + LOG_SERVER_ERROR_ << LogOut("[%s][%d] Query fail: %s", "search", 0, status.message().c_str()); return status; } fiu_do_on("SearchRequest.OnExecute.empty_result_ids", result_ids.clear()); @@ -173,7 +174,7 @@ SearchRequest::OnExecute() { result_.distance_list_.swap(result_distances); rc.RecordSection("construct result"); } catch (std::exception& ex) { - SERVER_LOG_ERROR << LogOut("[%s][%d] Encounter exception: %s", "search", 0, ex.what()); + LOG_SERVER_ERROR_ << LogOut("[%s][%d] Encounter exception: %s", "search", 0, ex.what()); return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } diff --git a/core/src/server/delivery/strategy/SearchReqStrategy.cpp b/core/src/server/delivery/strategy/SearchReqStrategy.cpp index 8cdb306966714ccb5301e94bc410494d00dc4d0c..2ab3f6ff1f9ab61b6fde3649162b077cbde19a37 100644 --- a/core/src/server/delivery/strategy/SearchReqStrategy.cpp +++ b/core/src/server/delivery/strategy/SearchReqStrategy.cpp @@ -30,7 +30,7 @@ Status SearchReqStrategy::ReScheduleQueue(const BaseRequestPtr& request, std::queue& queue) { if (request->GetRequestType() != BaseRequest::kSearch) { std::string msg = "search strategy can only handle search request"; - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_UNSUPPORTED_ERROR, msg); } @@ -49,7 +49,7 @@ SearchReqStrategy::ReScheduleQueue(const BaseRequestPtr& request, std::queueCombine(last_search_req); combine_request->Combine(new_search_req); queue.push(combine_request); - SERVER_LOG_DEBUG << "Combine 2 search request"; + LOG_SERVER_DEBUG_ << "Combine 2 search request"; } else { // directly put to queue queue.push(request); @@ -59,14 +59,14 @@ SearchReqStrategy::ReScheduleQueue(const BaseRequestPtr& request, std::queueCanCombine(new_search_req)) { // combine request combine_req->Combine(new_search_req); - SERVER_LOG_DEBUG << "Combine more search request"; + LOG_SERVER_DEBUG_ << "Combine more search request"; } else { // directly put to queue queue.push(request); } } else { std::string msg = "unsupported request type for search strategy"; - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_UNSUPPORTED_ERROR, msg); } diff --git a/core/src/server/grpc_impl/GrpcRequestHandler.cpp b/core/src/server/grpc_impl/GrpcRequestHandler.cpp index 78afd0341533e6dbe6bc2b1baeee47ba7f15514a..a9a3265761abb6d9750d9684490029160cc437fa 100644 --- a/core/src/server/grpc_impl/GrpcRequestHandler.cpp +++ b/core/src/server/grpc_impl/GrpcRequestHandler.cpp @@ -174,7 +174,7 @@ void set_request_id(::grpc::ServerContext* context, const std::string& request_id) { if (not context) { // error - SERVER_LOG_ERROR << "set_request_id: grpc::ServerContext is nullptr" << std::endl; + LOG_SERVER_ERROR_ << "set_request_id: grpc::ServerContext is nullptr" << std::endl; return; } @@ -185,7 +185,7 @@ std::string get_request_id(::grpc::ServerContext* context) { if (not context) { // error - SERVER_LOG_ERROR << "get_request_id: grpc::ServerContext is nullptr" << std::endl; + LOG_SERVER_ERROR_ << "get_request_id: grpc::ServerContext is nullptr" << std::endl; return "INVALID_ID"; } @@ -194,7 +194,7 @@ get_request_id(::grpc::ServerContext* context) { auto request_id_kv = server_metadata.find(REQ_ID); if (request_id_kv == server_metadata.end()) { // error - SERVER_LOG_ERROR << std::string(REQ_ID) << " not found in grpc.server_metadata" << std::endl; + LOG_SERVER_ERROR_ << std::string(REQ_ID) << " not found in grpc.server_metadata" << std::endl; return "INVALID_ID"; } @@ -242,7 +242,7 @@ GrpcRequestHandler::OnPostRecvInitialMetaData( auto request_id_kv = client_metadata.find("request_id"); if (request_id_kv != client_metadata.end()) { request_id = request_id_kv->second.data(); - SERVER_LOG_DEBUG << "client provide request_id: " << request_id; + LOG_SERVER_DEBUG_ << "client provide request_id: " << request_id; // if request_id is being used by another request, // convert it to request_id_n. @@ -265,7 +265,7 @@ GrpcRequestHandler::OnPostRecvInitialMetaData( } else { request_id = std::to_string(get_sequential_id()); set_request_id(server_context, request_id); - SERVER_LOG_DEBUG << "milvus generate request_id: " << request_id; + LOG_SERVER_DEBUG_ << "milvus generate request_id: " << request_id; } auto trace_context = std::make_shared(span); @@ -282,7 +282,7 @@ GrpcRequestHandler::OnPreSendMessage(::grpc::experimental::ServerRpcInfo* server if (context_map_.find(request_id) == context_map_.end()) { // error - SERVER_LOG_ERROR << "request_id " << request_id << " not found in context_map_"; + LOG_SERVER_ERROR_ << "request_id " << request_id << " not found in context_map_"; return; } context_map_[request_id]->GetTraceContext()->GetSpan()->Finish(); @@ -294,7 +294,7 @@ GrpcRequestHandler::GetContext(::grpc::ServerContext* server_context) { std::lock_guard lock(context_map_mutex_); auto request_id = get_request_id(server_context); if (context_map_.find(request_id) == context_map_.end()) { - SERVER_LOG_ERROR << "GetContext: request_id " << request_id << " not found in context_map_"; + LOG_SERVER_ERROR_ << "GetContext: request_id " << request_id << " not found in context_map_"; return nullptr; } return context_map_[request_id]; @@ -382,7 +382,7 @@ GrpcRequestHandler::Insert(::grpc::ServerContext* context, const ::milvus::grpc: ::milvus::grpc::VectorIds* response) { CHECK_NULLPTR_RETURN(request); - SERVER_LOG_INFO << LogOut("[%s][%d] Start insert.", "insert", 0); + LOG_SERVER_INFO_ << LogOut("[%s][%d] Start insert.", "insert", 0); // step 1: copy vector data engine::VectorsData vectors; @@ -397,7 +397,7 @@ GrpcRequestHandler::Insert(::grpc::ServerContext* context, const ::milvus::grpc: memcpy(response->mutable_vector_id_array()->mutable_data(), vectors.id_array_.data(), vectors.id_array_.size() * sizeof(int64_t)); - SERVER_LOG_INFO << LogOut("[%s][%d] Insert done.", "insert", 0); + LOG_SERVER_INFO_ << LogOut("[%s][%d] Insert done.", "insert", 0); SET_RESPONSE(response->mutable_status(), status, context); return ::grpc::Status::OK; } @@ -450,7 +450,7 @@ GrpcRequestHandler::Search(::grpc::ServerContext* context, const ::milvus::grpc: ::milvus::grpc::TopKQueryResult* response) { CHECK_NULLPTR_RETURN(request); - SERVER_LOG_INFO << LogOut("[%s][%d] Search start in gRPC server", "search", 0); + LOG_SERVER_INFO_ << LogOut("[%s][%d] Search start in gRPC server", "search", 0); // step 1: copy vector data engine::VectorsData vectors; CopyRowRecords(request->query_record_array(), google::protobuf::RepeatedField(), vectors); @@ -481,7 +481,7 @@ GrpcRequestHandler::Search(::grpc::ServerContext* context, const ::milvus::grpc: // step 5: construct and return result ConstructResults(result, response); - SERVER_LOG_INFO << LogOut("[%s][%d] Search done.", "search", 0); + LOG_SERVER_INFO_ << LogOut("[%s][%d] Search done.", "search", 0); SET_RESPONSE(response->mutable_status(), status, context); diff --git a/core/src/server/grpc_impl/GrpcServer.cpp b/core/src/server/grpc_impl/GrpcServer.cpp index de4549c4d63b3f98e461df6399a15a21d9139f91..4598bfb5a8ac936c9dc622ddeac78fba75af27a9 100644 --- a/core/src/server/grpc_impl/GrpcServer.cpp +++ b/core/src/server/grpc_impl/GrpcServer.cpp @@ -73,6 +73,7 @@ GrpcServer::Stop() { Status GrpcServer::StartService() { + SetThreadName("grpcserv_thread"); Config& config = Config::GetInstance(); std::string address, port; diff --git a/core/src/server/web_impl/WebServer.cpp b/core/src/server/web_impl/WebServer.cpp index e029ca0031f29a2b336b13d9052e2c7e5c351042..c2500eda3a05b5319361c2f93852ad73790882bf 100644 --- a/core/src/server/web_impl/WebServer.cpp +++ b/core/src/server/web_impl/WebServer.cpp @@ -39,6 +39,7 @@ WebServer::Stop() { Status WebServer::StartService() { + SetThreadName("webserv_thread"); oatpp::base::Environment::init(); Config& config = Config::GetInstance(); diff --git a/core/src/storage/disk/DiskOperation.cpp b/core/src/storage/disk/DiskOperation.cpp index 90a36c2bb2c3d273bcd19f38734c25113a3595dc..5c6b1d08e41dae7ea9666042b2dd8d3dd4107c79 100644 --- a/core/src/storage/disk/DiskOperation.cpp +++ b/core/src/storage/disk/DiskOperation.cpp @@ -33,7 +33,7 @@ DiskOperation::CreateDirectory() { auto ret = boost::filesystem::create_directory(dir_path_); if (!ret) { std::string err_msg = "Failed to create directory: " + dir_path_; - ENGINE_LOG_ERROR << err_msg; + LOG_ENGINE_ERROR_ << err_msg; throw Exception(SERVER_CANNOT_CREATE_FOLDER, err_msg); } } diff --git a/core/src/storage/s3/S3ClientWrapper.cpp b/core/src/storage/s3/S3ClientWrapper.cpp index 3f4c0ba4ce9fcc39e26e3d05103e40063a2be901..12b991fa872afb3e89a4132413467851a283a240 100644 --- a/core/src/storage/s3/S3ClientWrapper.cpp +++ b/core/src/storage/s3/S3ClientWrapper.cpp @@ -38,7 +38,7 @@ S3ClientWrapper::StartService() { CONFIG_CHECK(config.GetStorageConfigS3Enable(s3_enable)); fiu_do_on("S3ClientWrapper.StartService.s3_disable", s3_enable = false); if (!s3_enable) { - STORAGE_LOG_INFO << "S3 not enabled!"; + LOG_STORAGE_INFO_ << "S3 not enabled!"; return Status::OK(); } @@ -89,12 +89,12 @@ S3ClientWrapper::CreateBucket() { if (!outcome.IsSuccess()) { auto err = outcome.GetError(); if (err.GetErrorType() != Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU) { - STORAGE_LOG_ERROR << "ERROR: CreateBucket: " << err.GetExceptionName() << ": " << err.GetMessage(); + LOG_STORAGE_ERROR_ << "ERROR: CreateBucket: " << err.GetExceptionName() << ": " << err.GetMessage(); return Status(SERVER_UNEXPECTED_ERROR, err.GetMessage()); } } - STORAGE_LOG_DEBUG << "CreateBucket '" << s3_bucket_ << "' successfully!"; + LOG_STORAGE_DEBUG_ << "CreateBucket '" << s3_bucket_ << "' successfully!"; return Status::OK(); } @@ -108,11 +108,11 @@ S3ClientWrapper::DeleteBucket() { fiu_do_on("S3ClientWrapper.DeleteBucket.outcome.fail", outcome = Aws::S3::Model::DeleteBucketOutcome()); if (!outcome.IsSuccess()) { auto err = outcome.GetError(); - STORAGE_LOG_ERROR << "ERROR: DeleteBucket: " << err.GetExceptionName() << ": " << err.GetMessage(); + LOG_STORAGE_ERROR_ << "ERROR: DeleteBucket: " << err.GetExceptionName() << ": " << err.GetMessage(); return Status(SERVER_UNEXPECTED_ERROR, err.GetMessage()); } - STORAGE_LOG_DEBUG << "DeleteBucket '" << s3_bucket_ << "' successfully!"; + LOG_STORAGE_DEBUG_ << "DeleteBucket '" << s3_bucket_ << "' successfully!"; return Status::OK(); } @@ -121,7 +121,7 @@ S3ClientWrapper::PutObjectFile(const std::string& object_name, const std::string struct stat buffer; if (stat(file_path.c_str(), &buffer) != 0) { std::string str = "File '" + file_path + "' not exist!"; - STORAGE_LOG_ERROR << "ERROR: " << str; + LOG_STORAGE_ERROR_ << "ERROR: " << str; return Status(SERVER_UNEXPECTED_ERROR, str); } @@ -137,11 +137,11 @@ S3ClientWrapper::PutObjectFile(const std::string& object_name, const std::string fiu_do_on("S3ClientWrapper.PutObjectFile.outcome.fail", outcome = Aws::S3::Model::PutObjectOutcome()); if (!outcome.IsSuccess()) { auto err = outcome.GetError(); - STORAGE_LOG_ERROR << "ERROR: PutObject: " << err.GetExceptionName() << ": " << err.GetMessage(); + LOG_STORAGE_ERROR_ << "ERROR: PutObject: " << err.GetExceptionName() << ": " << err.GetMessage(); return Status(SERVER_UNEXPECTED_ERROR, err.GetMessage()); } - STORAGE_LOG_DEBUG << "PutObjectFile '" << file_path << "' successfully!"; + LOG_STORAGE_DEBUG_ << "PutObjectFile '" << file_path << "' successfully!"; return Status::OK(); } @@ -159,11 +159,11 @@ S3ClientWrapper::PutObjectStr(const std::string& object_name, const std::string& fiu_do_on("S3ClientWrapper.PutObjectStr.outcome.fail", outcome = Aws::S3::Model::PutObjectOutcome()); if (!outcome.IsSuccess()) { auto err = outcome.GetError(); - STORAGE_LOG_ERROR << "ERROR: PutObject: " << err.GetExceptionName() << ": " << err.GetMessage(); + LOG_STORAGE_ERROR_ << "ERROR: PutObject: " << err.GetExceptionName() << ": " << err.GetMessage(); return Status(SERVER_UNEXPECTED_ERROR, err.GetMessage()); } - STORAGE_LOG_DEBUG << "PutObjectStr successfully!"; + LOG_STORAGE_DEBUG_ << "PutObjectStr successfully!"; return Status::OK(); } @@ -177,7 +177,7 @@ S3ClientWrapper::GetObjectFile(const std::string& object_name, const std::string fiu_do_on("S3ClientWrapper.GetObjectFile.outcome.fail", outcome = Aws::S3::Model::GetObjectOutcome()); if (!outcome.IsSuccess()) { auto err = outcome.GetError(); - STORAGE_LOG_ERROR << "ERROR: GetObject: " << err.GetExceptionName() << ": " << err.GetMessage(); + LOG_STORAGE_ERROR_ << "ERROR: GetObject: " << err.GetExceptionName() << ": " << err.GetMessage(); return Status(SERVER_UNEXPECTED_ERROR, err.GetMessage()); } @@ -186,7 +186,7 @@ S3ClientWrapper::GetObjectFile(const std::string& object_name, const std::string output_file << retrieved_file.rdbuf(); output_file.close(); - STORAGE_LOG_DEBUG << "GetObjectFile '" << file_path << "' successfully!"; + LOG_STORAGE_DEBUG_ << "GetObjectFile '" << file_path << "' successfully!"; return Status::OK(); } @@ -200,7 +200,7 @@ S3ClientWrapper::GetObjectStr(const std::string& object_name, std::string& conte fiu_do_on("S3ClientWrapper.GetObjectStr.outcome.fail", outcome = Aws::S3::Model::GetObjectOutcome()); if (!outcome.IsSuccess()) { auto err = outcome.GetError(); - STORAGE_LOG_ERROR << "ERROR: GetObject: " << err.GetExceptionName() << ": " << err.GetMessage(); + LOG_STORAGE_ERROR_ << "ERROR: GetObject: " << err.GetExceptionName() << ": " << err.GetMessage(); return Status(SERVER_UNEXPECTED_ERROR, err.GetMessage()); } @@ -209,7 +209,7 @@ S3ClientWrapper::GetObjectStr(const std::string& object_name, std::string& conte ss << retrieved_file.rdbuf(); content = std::move(ss.str()); - STORAGE_LOG_DEBUG << "GetObjectStr successfully!"; + LOG_STORAGE_DEBUG_ << "GetObjectStr successfully!"; return Status::OK(); } @@ -227,7 +227,7 @@ S3ClientWrapper::ListObjects(std::vector& object_list, const std::s fiu_do_on("S3ClientWrapper.ListObjects.outcome.fail", outcome = Aws::S3::Model::ListObjectsOutcome()); if (!outcome.IsSuccess()) { auto err = outcome.GetError(); - STORAGE_LOG_ERROR << "ERROR: ListObjects: " << err.GetExceptionName() << ": " << err.GetMessage(); + LOG_STORAGE_ERROR_ << "ERROR: ListObjects: " << err.GetExceptionName() << ": " << err.GetMessage(); return Status(SERVER_UNEXPECTED_ERROR, err.GetMessage()); } @@ -238,9 +238,9 @@ S3ClientWrapper::ListObjects(std::vector& object_list, const std::s } if (marker.empty()) { - STORAGE_LOG_DEBUG << "ListObjects '" << s3_bucket_ << "' successfully!"; + LOG_STORAGE_DEBUG_ << "ListObjects '" << s3_bucket_ << "' successfully!"; } else { - STORAGE_LOG_DEBUG << "ListObjects '" << s3_bucket_ << ":" << marker << "' successfully!"; + LOG_STORAGE_DEBUG_ << "ListObjects '" << s3_bucket_ << ":" << marker << "' successfully!"; } return Status::OK(); } @@ -255,11 +255,11 @@ S3ClientWrapper::DeleteObject(const std::string& object_name) { fiu_do_on("S3ClientWrapper.DeleteObject.outcome.fail", outcome = Aws::S3::Model::DeleteObjectOutcome()); if (!outcome.IsSuccess()) { auto err = outcome.GetError(); - STORAGE_LOG_ERROR << "ERROR: DeleteObject: " << err.GetExceptionName() << ": " << err.GetMessage(); + LOG_STORAGE_ERROR_ << "ERROR: DeleteObject: " << err.GetExceptionName() << ": " << err.GetMessage(); return Status(SERVER_UNEXPECTED_ERROR, err.GetMessage()); } - STORAGE_LOG_DEBUG << "DeleteObject '" << object_name << "' successfully!"; + LOG_STORAGE_DEBUG_ << "DeleteObject '" << object_name << "' successfully!"; return Status::OK(); } diff --git a/core/src/utils/CommonUtil.cpp b/core/src/utils/CommonUtil.cpp index 1f353a3bba5956029e346052d65bc83d1b7dd68a..92882cb4801b3549803674ae5b49087d9cd1512f 100644 --- a/core/src/utils/CommonUtil.cpp +++ b/core/src/utils/CommonUtil.cpp @@ -247,7 +247,7 @@ CommonUtil::GetCurrentTimeStr() { void CommonUtil::EraseFromCache(const std::string& item_key) { if (item_key.empty()) { - SERVER_LOG_ERROR << "Empty key cannot be erased from cache"; + LOG_SERVER_ERROR_ << "Empty key cannot be erased from cache"; return; } diff --git a/core/src/utils/Log.cpp b/core/src/utils/Log.cpp index 6af45a2e1668fdad949fdce26bfbbc3ade34d868..500a997d57e6494cca063e02b9fc8a432329a2e0 100644 --- a/core/src/utils/Log.cpp +++ b/core/src/utils/Log.cpp @@ -31,4 +31,22 @@ LogOut(const char* pattern, ...) { return std::string(str_p.get()); } +void +SetThreadName(const std::string& name) { + pthread_setname_np(pthread_self(), name.c_str()); +} + +std::string +GetThreadName() { + std::string thread_name = "unamed"; + char name[16]; + size_t len = 16; + auto err = pthread_getname_np(pthread_self(), name, len); + if (not err) { + thread_name = name; + } + + return thread_name; +} + } // namespace milvus diff --git a/core/src/utils/Log.h b/core/src/utils/Log.h index 6cc64c387ca9d8b58b15b2e103501d05c5e5efe1..68e7d6f7b79399ea099aba5affd59fc2b3e1dcb1 100644 --- a/core/src/utils/Log.h +++ b/core/src/utils/Log.h @@ -17,57 +17,172 @@ namespace milvus { +/* + * Please use LOG_MODULE_LEVEL_C macro in member function of class + * and LOG_MODULE_LEVEL_ macro in other functions. + */ + +///////////////////////////////////////////////////////////////////////////////////////////////// +#define SERVER_MODULE_NAME "SERVER" +#define SERVER_MODULE_CLASS_FUNCTION \ + LogOut("[%s][%s::%s][%s] ", SERVER_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, GetThreadName().c_str()) +#define SERVER_MODULE_FUNCTION LogOut("[%s][%s][%s] ", SERVER_MODULE_NAME, __FUNCTION__, GetThreadName().c_str()) + +#define LOG_SERVER_TRACE_C LOG(TRACE) << SERVER_MODULE_CLASS_FUNCTION +#define LOG_SERVER_DEBUG_C LOG(DEBUG) << SERVER_MODULE_CLASS_FUNCTION +#define LOG_SERVER_INFO_C LOG(INFO) << SERVER_MODULE_CLASS_FUNCTION +#define LOG_SERVER_WARNING_C LOG(WARNING) << SERVER_MODULE_CLASS_FUNCTION +#define LOG_SERVER_ERROR_C LOG(ERROR) << SERVER_MODULE_CLASS_FUNCTION +#define LOG_SERVER_FATAL_C LOG(FATAL) << SERVER_MODULE_CLASS_FUNCTION + +#define LOG_SERVER_TRACE_ LOG(TRACE) << SERVER_MODULE_FUNCTION +#define LOG_SERVER_DEBUG_ LOG(DEBUG) << SERVER_MODULE_FUNCTION +#define LOG_SERVER_INFO_ LOG(INFO) << SERVER_MODULE_FUNCTION +#define LOG_SERVER_WARNING_ LOG(WARNING) << SERVER_MODULE_FUNCTION +#define LOG_SERVER_ERROR_ LOG(ERROR) << SERVER_MODULE_FUNCTION +#define LOG_SERVER_FATAL_ LOG(FATAL) << SERVER_MODULE_FUNCTION + +///////////////////////////////////////////////////////////////////////////////////////////////// +#define ENGINE_MODULE_NAME "ENGINE" +#define ENGINE_MODULE_CLASS_FUNCTION \ + LogOut("[%s][%s::%s][%s] ", ENGINE_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, GetThreadName().c_str()) +#define ENGINE_MODULE_FUNCTION LogOut("[%s][%s][%s] ", ENGINE_MODULE_NAME, __FUNCTION__, GetThreadName().c_str()) + +#define LOG_ENGINE_TRACE_C LOG(TRACE) << ENGINE_MODULE_CLASS_FUNCTION +#define LOG_ENGINE_DEBUG_C LOG(DEBUG) << ENGINE_MODULE_CLASS_FUNCTION +#define LOG_ENGINE_INFO_C LOG(INFO) << ENGINE_MODULE_CLASS_FUNCTION +#define LOG_ENGINE_WARNING_C LOG(WARNING) << ENGINE_MODULE_CLASS_FUNCTION +#define LOG_ENGINE_ERROR_C LOG(ERROR) << ENGINE_MODULE_CLASS_FUNCTION +#define LOG_ENGINE_FATAL_C LOG(FATAL) << ENGINE_MODULE_CLASS_FUNCTION + +#define LOG_ENGINE_TRACE_ LOG(TRACE) << ENGINE_MODULE_FUNCTION +#define LOG_ENGINE_DEBUG_ LOG(DEBUG) << ENGINE_MODULE_FUNCTION +#define LOG_ENGINE_INFO_ LOG(INFO) << ENGINE_MODULE_FUNCTION +#define LOG_ENGINE_WARNING_ LOG(WARNING) << ENGINE_MODULE_FUNCTION +#define LOG_ENGINE_ERROR_ LOG(ERROR) << ENGINE_MODULE_FUNCTION +#define LOG_ENGINE_FATAL_ LOG(FATAL) << ENGINE_MODULE_FUNCTION + +///////////////////////////////////////////////////////////////////////////////////////////////// +#define WRAPPER_MODULE_NAME "WRAPPER" +#define WRAPPER_MODULE_CLASS_FUNCTION \ + LogOut("[%s][%s::%s][%s] ", WRAPPER_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, GetThreadName().c_str()) +#define WRAPPER_MODULE_FUNCTION LogOut("[%s][%s][%s] ", WRAPPER_MODULE_NAME, __FUNCTION__, GetThreadName().c_str()) + +#define LOG_WRAPPER_TRACE_C LOG(TRACE) << WRAPPER_MODULE_CLASS_FUNCTION +#define LOG_WRAPPER_DEBUG_C LOG(DEBUG) << WRAPPER_MODULE_CLASS_FUNCTION +#define LOG_WRAPPER_INFO_C LOG(INFO) << WRAPPER_MODULE_CLASS_FUNCTION +#define LOG_WRAPPER_WARNING_C LOG(WARNING) << WRAPPER_MODULE_CLASS_FUNCTION +#define LOG_WRAPPER_ERROR_C LOG(ERROR) << WRAPPER_MODULE_CLASS_FUNCTION +#define LOG_WRAPPER_FATAL_C LOG(FATAL) << WRAPPER_MODULE_CLASS_FUNCTION + +#define LOG_WRAPPER_TRACE_ LOG(TRACE) << WRAPPER_MODULE_FUNCTION +#define LOG_WRAPPER_DEBUG_ LOG(DEBUG) << WRAPPER_MODULE_FUNCTION +#define LOG_WRAPPER_INFO_ LOG(INFO) << WRAPPER_MODULE_FUNCTION +#define LOG_WRAPPER_WARNING_ LOG(WARNING) << WRAPPER_MODULE_FUNCTION +#define LOG_WRAPPER_ERROR_ LOG(ERROR) << WRAPPER_MODULE_FUNCTION +#define LOG_WRAPPER_FATAL_ LOG(FATAL) << WRAPPER_MODULE_FUNCTION + ///////////////////////////////////////////////////////////////////////////////////////////////// -#define SERVER_DOMAIN_NAME "[SERVER] " +#define STORAGE_MODULE_NAME "STORAGE" +#define STORAGE_MODULE_CLASS_FUNCTION \ + LogOut("[%s][%s::%s][%s] ", STORAGE_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, GetThreadName().c_str()) +#define STORAGE_MODULE_FUNCTION LogOut("[%s][%s][%s] ", STORAGE_MODULE_NAME, __FUNCTION__, GetThreadName().c_str()) + +#define LOG_STORAGE_TRACE_C LOG(TRACE) << STORAGE_MODULE_CLASS_FUNCTION +#define LOG_STORAGE_DEBUG_C LOG(DEBUG) << STORAGE_MODULE_CLASS_FUNCTION +#define LOG_STORAGE_INFO_C LOG(INFO) << STORAGE_MODULE_CLASS_FUNCTION +#define LOG_STORAGE_WARNING_C LOG(WARNING) << STORAGE_MODULE_CLASS_FUNCTION +#define LOG_STORAGE_ERROR_C LOG(ERROR) << STORAGE_MODULE_CLASS_FUNCTION +#define LOG_STORAGE_FATAL_C LOG(FATAL) << STORAGE_MODULE_CLASS_FUNCTION + +#define LOG_STORAGE_TRACE_ LOG(TRACE) << STORAGE_MODULE_FUNCTION +#define LOG_STORAGE_DEBUG_ LOG(DEBUG) << STORAGE_MODULE_FUNCTION +#define LOG_STORAGE_INFO_ LOG(INFO) << STORAGE_MODULE_FUNCTION +#define LOG_STORAGE_WARNING_ LOG(WARNING) << STORAGE_MODULE_FUNCTION +#define LOG_STORAGE_ERROR_ LOG(ERROR) << STORAGE_MODULE_FUNCTION +#define LOG_STORAGE_FATAL_ LOG(FATAL) << STORAGE_MODULE_FUNCTION -#define SERVER_LOG_TRACE LOG(TRACE) << SERVER_DOMAIN_NAME -#define SERVER_LOG_DEBUG LOG(DEBUG) << SERVER_DOMAIN_NAME -#define SERVER_LOG_INFO LOG(INFO) << SERVER_DOMAIN_NAME -#define SERVER_LOG_WARNING LOG(WARNING) << SERVER_DOMAIN_NAME -#define SERVER_LOG_ERROR LOG(ERROR) << SERVER_DOMAIN_NAME -#define SERVER_LOG_FATAL LOG(FATAL) << SERVER_DOMAIN_NAME +///////////////////////////////////////////////////////////////////////////////////////////////// +#define WAL_MODULE_NAME "WAL" +#define WAL_MODULE_CLASS_FUNCTION \ + LogOut("[%s][%s::%s][%s] ", WAL_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, GetThreadName().c_str()) +#define WAL_MODULE_FUNCTION LogOut("[%s][%s][%s] ", WAL_MODULE_NAME, __FUNCTION__, GetThreadName().c_str()) + +#define LOG_WAL_TRACE_C LOG(TRACE) << WAL_MODULE_CLASS_FUNCTION +#define LOG_WAL_DEBUG_C LOG(DEBUG) << WAL_MODULE_CLASS_FUNCTION +#define LOG_WAL_INFO_C LOG(INFO) << WAL_MODULE_CLASS_FUNCTION +#define LOG_WAL_WARNING_C LOG(WARNING) << WAL_MODULE_CLASS_FUNCTION +#define LOG_WAL_ERROR_C LOG(ERROR) << WAL_MODULE_CLASS_FUNCTION +#define LOG_WAL_FATAL_C LOG(FATAL) << WAL_MODULE_CLASS_FUNCTION + +#define LOG_WAL_TRACE_ LOG(TRACE) << WAL_MODULE_FUNCTION +#define LOG_WAL_DEBUG_ LOG(DEBUG) << WAL_MODULE_FUNCTION +#define LOG_WAL_INFO_ LOG(INFO) << WAL_MODULE_FUNCTION +#define LOG_WAL_WARNING_ LOG(WARNING) << WAL_MODULE_FUNCTION +#define LOG_WAL_ERROR_ LOG(ERROR) << WAL_MODULE_FUNCTION +#define LOG_WAL_FATAL_ LOG(FATAL) << WAL_MODULE_FUNCTION + +/* + * Deprecated + */ + +///////////////////////////////////////////////////////////////////////////////////////////////// +// #define SERVER_DOMAIN_NAME "[SERVER]" + +// #define LOG_SERVER_TRACE_ LOG(TRACE) << SERVER_DOMAIN_NAME +// #define LOG_SERVER_DEBUG_ LOG(DEBUG) << SERVER_DOMAIN_NAME +// #define LOG_SERVER_INFO_ LOG(INFO) << SERVER_DOMAIN_NAME +// #define LOG_SERVER_WARNING_ LOG(WARNING) << SERVER_DOMAIN_NAME +// #define LOG_SERVER_ERROR_ LOG(ERROR) << SERVER_DOMAIN_NAME +// #define LOG_SERVER_FATAL_ LOG(FATAL) << SERVER_DOMAIN_NAME ///////////////////////////////////////////////////////////////////////////////////////////////// -#define ENGINE_DOMAIN_NAME "[ENGINE] " +// #define ENGINE_DOMAIN_NAME "[ENGINE]" -#define ENGINE_LOG_TRACE LOG(TRACE) << ENGINE_DOMAIN_NAME -#define ENGINE_LOG_DEBUG LOG(DEBUG) << ENGINE_DOMAIN_NAME -#define ENGINE_LOG_INFO LOG(INFO) << ENGINE_DOMAIN_NAME -#define ENGINE_LOG_WARNING LOG(WARNING) << ENGINE_DOMAIN_NAME -#define ENGINE_LOG_ERROR LOG(ERROR) << ENGINE_DOMAIN_NAME -#define ENGINE_LOG_FATAL LOG(FATAL) << ENGINE_DOMAIN_NAME +// #define LOG_ENGINE_TRACE_ LOG(TRACE) << ENGINE_DOMAIN_NAME +// #define LOG_ENGINE_DEBUG_ LOG(DEBUG) << ENGINE_DOMAIN_NAME +// #define LOG_ENGINE_INFO_ LOG(INFO) << ENGINE_DOMAIN_NAME +// #define LOG_ENGINE_WARNING_ LOG(WARNING) << ENGINE_DOMAIN_NAME +// #define LOG_ENGINE_ERROR_ LOG(ERROR) << ENGINE_DOMAIN_NAME +// #define LOG_ENGINE_FATAL_ LOG(FATAL) << ENGINE_DOMAIN_NAME ///////////////////////////////////////////////////////////////////////////////////////////////// -#define WRAPPER_DOMAIN_NAME "[WRAPPER] " +// #define WRAPPER_DOMAIN_NAME "[WRAPPER]" -#define WRAPPER_LOG_TRACE LOG(TRACE) << WRAPPER_DOMAIN_NAME -#define WRAPPER_LOG_DEBUG LOG(DEBUG) << WRAPPER_DOMAIN_NAME -#define WRAPPER_LOG_INFO LOG(INFO) << WRAPPER_DOMAIN_NAME -#define WRAPPER_LOG_WARNING LOG(WARNING) << WRAPPER_DOMAIN_NAME -#define WRAPPER_LOG_ERROR LOG(ERROR) << WRAPPER_DOMAIN_NAME -#define WRAPPER_LOG_FATAL LOG(FATAL) << WRAPPER_DOMAIN_NAME +// #define WRAPPER_LOG_TRACE LOG(TRACE) << WRAPPER_DOMAIN_NAME +// #define WRAPPER_LOG_DEBUG LOG(DEBUG) << WRAPPER_DOMAIN_NAME +// #define WRAPPER_LOG_INFO LOG(INFO) << WRAPPER_DOMAIN_NAME +// #define WRAPPER_LOG_WARNING LOG(WARNING) << WRAPPER_DOMAIN_NAME +// #define WRAPPER_LOG_ERROR LOG(ERROR) << WRAPPER_DOMAIN_NAME +// #define WRAPPER_LOG_FATAL LOG(FATAL) << WRAPPER_DOMAIN_NAME ///////////////////////////////////////////////////////////////////////////////////////////////// -#define STORAGE_DOMAIN_NAME "[STORAGE] " +// #define STORAGE_DOMAIN_NAME "[STORAGE]" -#define STORAGE_LOG_TRACE LOG(TRACE) << STORAGE_DOMAIN_NAME -#define STORAGE_LOG_DEBUG LOG(DEBUG) << STORAGE_DOMAIN_NAME -#define STORAGE_LOG_INFO LOG(INFO) << STORAGE_DOMAIN_NAME -#define STORAGE_LOG_WARNING LOG(WARNING) << STORAGE_DOMAIN_NAME -#define STORAGE_LOG_ERROR LOG(ERROR) << STORAGE_DOMAIN_NAME -#define STORAGE_LOG_FATAL LOG(FATAL) << STORAGE_DOMAIN_NAME +// #define LOG_STORAGE_TRACE_ LOG(TRACE) << STORAGE_DOMAIN_NAME +// #define LOG_STORAGE_DEBUG_ LOG(DEBUG) << STORAGE_DOMAIN_NAME +// #define LOG_STORAGE_INFO_ LOG(INFO) << STORAGE_DOMAIN_NAME +// #define LOG_STORAGE_WARNING_ LOG(WARNING) << STORAGE_DOMAIN_NAME +// #define LOG_STORAGE_ERROR_ LOG(ERROR) << STORAGE_DOMAIN_NAME +// #define LOG_STORAGE_FATAL_ LOG(FATAL) << STORAGE_DOMAIN_NAME -#define WAL_DOMAIN_NAME "[WAL] " +// #define WAL_DOMAIN_NAME "[WAL]" -#define WAL_LOG_TRACE LOG(TRACE) << WAL_DOMAIN_NAME -#define WAL_LOG_DEBUG LOG(DEBUG) << WAL_DOMAIN_NAME -#define WAL_LOG_INFO LOG(INFO) << WAL_DOMAIN_NAME -#define WAL_LOG_WARNING LOG(WARNING) << WAL_DOMAIN_NAME -#define WAL_LOG_ERROR LOG(ERROR) << WAL_DOMAIN_NAME -#define WAL_LOG_FATAL LOG(FATAL) << WAL_DOMAIN_NAME +// #define LOG_WAL_TRACE_ LOG(TRACE) << WAL_DOMAIN_NAME +// #define LOG_WAL_DEBUG_ LOG(DEBUG) << WAL_DOMAIN_NAME +// #define LOG_WAL_INFO_ LOG(INFO) << WAL_DOMAIN_NAME +// #define LOG_WAL_WARNING_ LOG(WARNING) << WAL_DOMAIN_NAME +// #define LOG_WAL_ERROR_ LOG(ERROR) << WAL_DOMAIN_NAME +// #define LOG_WAL_FATAL_ LOG(FATAL) << WAL_DOMAIN_NAME ///////////////////////////////////////////////////////////////////////////////////////////////////// std::string LogOut(const char* pattern, ...); +void +SetThreadName(const std::string& name); + +std::string +GetThreadName(); + } // namespace milvus diff --git a/core/src/utils/LogUtil.cpp b/core/src/utils/LogUtil.cpp index 95056d24fae724bae690b05d44b5d2be5b15846d..706f916e2a90b9ee311f55a2bf2856127fed3f16 100644 --- a/core/src/utils/LogUtil.cpp +++ b/core/src/utils/LogUtil.cpp @@ -97,7 +97,7 @@ LogConfigInFile(const std::string& path) { auto node = YAML::LoadFile(path); YAML::Emitter out; out << node; - SERVER_LOG_DEBUG << "\n\n" + LOG_SERVER_INFO_ << "\n\n" << std::string(15, '*') << "Config in file" << std::string(15, '*') << "\n\n" << out.c_str(); } @@ -107,7 +107,7 @@ LogConfigInMem() { auto& config = Config::GetInstance(); std::string config_str; config.GetConfigJsonStr(config_str, 3); - SERVER_LOG_DEBUG << "\n\n" + LOG_SERVER_INFO_ << "\n\n" << std::string(15, '*') << "Config in memory" << std::string(15, '*') << "\n\n" << config_str; } @@ -117,7 +117,7 @@ LogCpuInfo() { /*CPU information*/ std::fstream fcpu("/proc/cpuinfo", std::ios::in); if (!fcpu.is_open()) { - SERVER_LOG_WARNING << "Cannot obtain CPU information. Open file /proc/cpuinfo fail: " << strerror(errno); + LOG_SERVER_WARNING_ << "Cannot obtain CPU information. Open file /proc/cpuinfo fail: " << strerror(errno); return; } std::stringstream cpu_info_ss; @@ -127,12 +127,12 @@ LogCpuInfo() { auto processor_pos = cpu_info.rfind("processor"); if (std::string::npos == processor_pos) { - SERVER_LOG_WARNING << "Cannot obtain CPU information. No sub string \'processor\'"; + LOG_SERVER_WARNING_ << "Cannot obtain CPU information. No sub string \'processor\'"; return; } auto sub_str = cpu_info.substr(processor_pos); - SERVER_LOG_DEBUG << "\n\n" << std::string(15, '*') << "CPU" << std::string(15, '*') << "\n\n" << sub_str; + LOG_SERVER_INFO_ << "\n\n" << std::string(15, '*') << "CPU" << std::string(15, '*') << "\n\n" << sub_str; } } // namespace server diff --git a/core/src/utils/SignalUtil.cpp b/core/src/utils/SignalUtil.cpp index e30a58e90a7c10c2860558d81aac470c0c18234f..003088c5796fca8e37e9f42aa6d3a88ee0304f5c 100644 --- a/core/src/utils/SignalUtil.cpp +++ b/core/src/utils/SignalUtil.cpp @@ -25,7 +25,7 @@ SignalUtil::HandleSignal(int signum) { switch (signum) { case SIGINT: case SIGUSR2: { - SERVER_LOG_INFO << "Server received signal: " << signum; + LOG_SERVER_INFO_ << "Server received signal: " << signum; server::Server& server = server::Server::GetInstance(); server.Stop(); @@ -33,7 +33,7 @@ SignalUtil::HandleSignal(int signum) { exit(0); } default: { - SERVER_LOG_INFO << "Server received critical signal: " << signum; + LOG_SERVER_INFO_ << "Server received critical signal: " << signum; SignalUtil::PrintStacktrace(); server::Server& server = server::Server::GetInstance(); @@ -46,7 +46,7 @@ SignalUtil::HandleSignal(int signum) { void SignalUtil::PrintStacktrace() { - SERVER_LOG_INFO << "Call stack:"; + LOG_SERVER_INFO_ << "Call stack:"; const int size = 32; void* array[size]; @@ -54,7 +54,7 @@ SignalUtil::PrintStacktrace() { char** stacktrace = backtrace_symbols(array, stack_num); for (int i = 0; i < stack_num; ++i) { std::string info = stacktrace[i]; - SERVER_LOG_INFO << info; + LOG_SERVER_INFO_ << info; } free(stacktrace); } diff --git a/core/src/utils/TimeRecorder.cpp b/core/src/utils/TimeRecorder.cpp index cd64aa04cf5340a18cc486fa0fce555cdf68c74c..cb8b674753cdd852b94cdea28771893d12fc77e3 100644 --- a/core/src/utils/TimeRecorder.cpp +++ b/core/src/utils/TimeRecorder.cpp @@ -40,31 +40,31 @@ TimeRecorder::PrintTimeRecord(const std::string& msg, double span) { switch (log_level_) { case 0: { - SERVER_LOG_TRACE << str_log; + LOG_SERVER_TRACE_ << str_log; break; } case 1: { - SERVER_LOG_DEBUG << str_log; + LOG_SERVER_DEBUG_ << str_log; break; } case 2: { - SERVER_LOG_INFO << str_log; + LOG_SERVER_INFO_ << str_log; break; } case 3: { - SERVER_LOG_WARNING << str_log; + LOG_SERVER_WARNING_ << str_log; break; } case 4: { - SERVER_LOG_ERROR << str_log; + LOG_SERVER_ERROR_ << str_log; break; } case 5: { - SERVER_LOG_FATAL << str_log; + LOG_SERVER_FATAL_ << str_log; break; } default: { - SERVER_LOG_INFO << str_log; + LOG_SERVER_INFO_ << str_log; break; } } diff --git a/core/src/utils/TimeRecorder.h b/core/src/utils/TimeRecorder.h index d41796fd0a918c5712b6d76cdc488cd38ac39545..ac11414970b0107af664b41c5aa7cff22d86d262 100644 --- a/core/src/utils/TimeRecorder.h +++ b/core/src/utils/TimeRecorder.h @@ -24,7 +24,7 @@ print_timestamp(const std::string& message) { micros %= 1000000; double millisecond = (double)micros / 1000.0; - SERVER_LOG_DEBUG << std::fixed << " " << millisecond << "(ms) [timestamp]" << message; + LOG_SERVER_DEBUG_ << std::fixed << " " << millisecond << "(ms) [timestamp]" << message; } class TimeRecorder { diff --git a/core/src/utils/ValidationUtil.cpp b/core/src/utils/ValidationUtil.cpp index cba9cd988b20514b06cf6a720861b52a950529ee..3fab7904f3b669c044817a2b469a0d288ce87657 100644 --- a/core/src/utils/ValidationUtil.cpp +++ b/core/src/utils/ValidationUtil.cpp @@ -45,7 +45,7 @@ CheckParameterRange(const milvus::json& json_params, const std::string& param_na if (json_params.find(param_name) == json_params.end()) { std::string msg = "Parameter list must contain: "; msg += param_name; - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_ARGUMENT, msg); } @@ -57,13 +57,13 @@ CheckParameterRange(const milvus::json& json_params, const std::string& param_na std::string msg = "Invalid " + param_name + " value: " + std::to_string(value) + ". Valid range is " + (min_close ? "[" : "(") + std::to_string(min) + ", " + std::to_string(max) + (max_closed ? "]" : ")"); - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_ARGUMENT, msg); } } catch (std::exception& e) { std::string msg = "Invalid " + param_name + ": "; msg += e.what(); - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_ARGUMENT, msg); } @@ -75,7 +75,7 @@ CheckParameterExistence(const milvus::json& json_params, const std::string& para if (json_params.find(param_name) == json_params.end()) { std::string msg = "Parameter list must contain: "; msg += param_name; - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_ARGUMENT, msg); } @@ -83,13 +83,13 @@ CheckParameterExistence(const milvus::json& json_params, const std::string& para int64_t value = json_params[param_name]; if (value < 0) { std::string msg = "Invalid " + param_name + " value: " + std::to_string(value); - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_ARGUMENT, msg); } } catch (std::exception& e) { std::string msg = "Invalid " + param_name + ": "; msg += e.what(); - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_ARGUMENT, msg); } @@ -103,7 +103,7 @@ ValidationUtil::ValidateCollectionName(const std::string& collection_name) { // Collection name shouldn't be empty. if (collection_name.empty()) { std::string msg = "Collection name should not be empty."; - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_COLLECTION_NAME, msg); } @@ -111,7 +111,7 @@ ValidationUtil::ValidateCollectionName(const std::string& collection_name) { // Collection name size shouldn't exceed 16384. if (collection_name.size() > COLLECTION_NAME_SIZE_LIMIT) { std::string msg = invalid_msg + "The length of a collection name must be less than 255 characters."; - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_COLLECTION_NAME, msg); } @@ -119,7 +119,7 @@ ValidationUtil::ValidateCollectionName(const std::string& collection_name) { char first_char = collection_name[0]; if (first_char != '_' && std::isalpha(first_char) == 0) { std::string msg = invalid_msg + "The first character of a collection name must be an underscore or letter."; - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_COLLECTION_NAME, msg); } @@ -128,7 +128,7 @@ ValidationUtil::ValidateCollectionName(const std::string& collection_name) { char name_char = collection_name[i]; if (name_char != '_' && std::isalnum(name_char) == 0) { std::string msg = invalid_msg + "Collection name can only contain numbers, letters, and underscores."; - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_COLLECTION_NAME, msg); } } @@ -142,7 +142,7 @@ ValidationUtil::ValidateTableDimension(int64_t dimension, int64_t metric_type) { std::string msg = "Invalid collection dimension: " + std::to_string(dimension) + ". " + "The collection dimension must be within the range of 1 ~ " + std::to_string(COLLECTION_DIMENSION_LIMIT) + "."; - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_VECTOR_DIMENSION, msg); } @@ -150,7 +150,7 @@ ValidationUtil::ValidateTableDimension(int64_t dimension, int64_t metric_type) { if ((dimension % 8) != 0) { std::string msg = "Invalid collection dimension: " + std::to_string(dimension) + ". " + "The collection dimension must be a multiple of 8"; - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_VECTOR_DIMENSION, msg); } } @@ -164,7 +164,7 @@ ValidationUtil::ValidateCollectionIndexType(int32_t index_type) { if (engine_type <= 0 || engine_type > static_cast(engine::EngineType::MAX_VALUE)) { std::string msg = "Invalid index type: " + std::to_string(index_type) + ". " + "Make sure the index type is in IndexType list."; - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_INDEX_TYPE, msg); } @@ -172,7 +172,7 @@ ValidationUtil::ValidateCollectionIndexType(int32_t index_type) { // special case, hybird index only available in customize faiss library if (engine_type == static_cast(engine::EngineType::FAISS_IVFSQ8H)) { std::string msg = "Unsupported index type: " + std::to_string(index_type); - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_INDEX_TYPE, msg); } #endif @@ -215,7 +215,7 @@ ValidationUtil::ValidateIndexParams(const milvus::json& index_params, int64_t m_value = index_params[index_params, knowhere::IndexParams::m]; if (resset.empty()) { std::string msg = "Invalid collection dimension, unable to get reasonable values for 'm'"; - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_COLLECTION_DIMENSION, msg); } @@ -230,7 +230,7 @@ ValidationUtil::ValidateIndexParams(const milvus::json& index_params, msg += std::to_string(resset[i]); } - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_ARGUMENT, msg); } @@ -364,7 +364,7 @@ ValidationUtil::ValidateCollectionIndexFileSize(int64_t index_file_size) { std::string msg = "Invalid index file size: " + std::to_string(index_file_size) + ". " + "The index file size must be within the range of 1 ~ " + std::to_string(INDEX_FILE_SIZE_LIMIT) + "."; - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_INDEX_FILE_SIZE, msg); } @@ -376,7 +376,7 @@ ValidationUtil::ValidateCollectionIndexMetricType(int32_t metric_type) { if (metric_type <= 0 || metric_type > static_cast(engine::MetricType::MAX_VALUE)) { std::string msg = "Invalid index metric type: " + std::to_string(metric_type) + ". " + "Make sure the metric type is in MetricType list."; - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_INDEX_METRIC_TYPE, msg); } return Status::OK(); @@ -387,7 +387,7 @@ ValidationUtil::ValidateSearchTopk(int64_t top_k) { if (top_k <= 0 || top_k > QUERY_MAX_TOPK) { std::string msg = "Invalid topk: " + std::to_string(top_k) + ". " + "The topk must be within the range of 1 ~ 2048."; - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_TOPK, msg); } @@ -398,7 +398,7 @@ Status ValidationUtil::ValidatePartitionName(const std::string& partition_name) { if (partition_name.empty()) { std::string msg = "Partition name should not be empty."; - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_COLLECTION_NAME, msg); } @@ -406,7 +406,7 @@ ValidationUtil::ValidatePartitionName(const std::string& partition_name) { // Collection name size shouldn't exceed 16384. if (partition_name.size() > COLLECTION_NAME_SIZE_LIMIT) { std::string msg = invalid_msg + "The length of a partition name must be less than 255 characters."; - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_COLLECTION_NAME, msg); } @@ -414,7 +414,7 @@ ValidationUtil::ValidatePartitionName(const std::string& partition_name) { char first_char = partition_name[0]; if (first_char != '_' && std::isalpha(first_char) == 0) { std::string msg = invalid_msg + "The first character of a partition name must be an underscore or letter."; - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_COLLECTION_NAME, msg); } @@ -423,7 +423,7 @@ ValidationUtil::ValidatePartitionName(const std::string& partition_name) { char name_char = partition_name[i]; if (name_char != '_' && std::isalnum(name_char) == 0) { std::string msg = invalid_msg + "Partition name can only contain numbers, letters, and underscores."; - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_COLLECTION_NAME, msg); } } @@ -440,14 +440,14 @@ ValidationUtil::ValidatePartitionTags(const std::vector& partition_ StringHelpFunctions::TrimStringBlank(valid_tag); if (valid_tag.empty()) { std::string msg = "Invalid partition tag: " + valid_tag + ". " + "Partition tag should not be empty."; - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_PARTITION_TAG, msg); } // max length of partition tag if (valid_tag.length() > 255) { std::string msg = "Invalid partition tag: " + valid_tag + ". " + "Partition tag exceed max length(255)."; - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_PARTITION_TAG, msg); } } @@ -464,13 +464,13 @@ ValidationUtil::ValidateGpuIndex(int32_t gpu_index) { if (cuda_err != cudaSuccess) { std::string msg = "Failed to get gpu card number, cuda error:" + std::to_string(cuda_err); - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_UNEXPECTED_ERROR, msg); } if (gpu_index >= num_devices) { std::string msg = "Invalid gpu index: " + std::to_string(gpu_index); - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_ARGUMENT, msg); } #endif @@ -489,7 +489,7 @@ ValidationUtil::GetGpuMemory(int32_t gpu_index, size_t& memory) { if (cuda_err) { std::string msg = "Failed to get gpu properties for gpu" + std::to_string(gpu_index) + " , cuda error:" + std::to_string(cuda_err); - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_UNEXPECTED_ERROR, msg); } @@ -511,12 +511,12 @@ ValidationUtil::ValidateIpAddress(const std::string& ip_address) { return Status::OK(); case 0: { std::string msg = "Invalid IP address: " + ip_address; - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_ARGUMENT, msg); } default: { std::string msg = "IP address conversion error: " + ip_address; - SERVER_LOG_ERROR << msg; + LOG_SERVER_ERROR_ << msg; return Status(SERVER_UNEXPECTED_ERROR, msg); } } @@ -583,7 +583,7 @@ ValidationUtil::ValidateDbURI(const std::string& uri) { std::string dialect = pieces_match[1].str(); std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower); if (dialect.find("mysql") == std::string::npos && dialect.find("sqlite") == std::string::npos) { - SERVER_LOG_ERROR << "Invalid dialect in URI: dialect = " << dialect; + LOG_SERVER_ERROR_ << "Invalid dialect in URI: dialect = " << dialect; okay = false; } @@ -593,7 +593,7 @@ ValidationUtil::ValidateDbURI(const std::string& uri) { std::string host = pieces_match[4].str(); if (!host.empty() && host != "localhost") { if (ValidateIpAddress(host) != SERVER_SUCCESS) { - SERVER_LOG_ERROR << "Invalid host ip address in uri = " << host; + LOG_SERVER_ERROR_ << "Invalid host ip address in uri = " << host; okay = false; } } @@ -603,12 +603,12 @@ ValidationUtil::ValidateDbURI(const std::string& uri) { if (!port.empty()) { auto status = ValidateStringIsNumber(port); if (!status.ok()) { - SERVER_LOG_ERROR << "Invalid port in uri = " << port; + LOG_SERVER_ERROR_ << "Invalid port in uri = " << port; okay = false; } } } else { - SERVER_LOG_ERROR << "Wrong URI format: URI = " << uri; + LOG_SERVER_ERROR_ << "Wrong URI format: URI = " << uri; okay = false; }