From 12b3bf0693dd277626fcef873efa5342155e45ab Mon Sep 17 00:00:00 2001 From: Wang XiangYu Date: Sun, 26 Apr 2020 16:31:18 +0800 Subject: [PATCH] Add instance lock (#2060) * Add instance lock Signed-off-by: wxyu * update message Signed-off-by: wxyu * update unittest CMakeLists.txt Signed-off-by: wxyu * update Signed-off-by: wxyu * update Signed-off-by: wxyu * update Signed-off-by: wxyu * update Signed-off-by: wxyu * fix clang-format Signed-off-by: wxyu * update Signed-off-by: wxyu Co-authored-by: Jin Hai --- CHANGELOG.md | 1 + core/conf/log_config.template | 3 ++ core/src/CMakeLists.txt | 1 + core/src/main.cpp | 5 +- core/src/server/Server.cpp | 37 +++++++++++++ core/src/server/init/InstanceLockCheck.cpp | 62 ++++++++++++++++++++++ core/src/server/init/InstanceLockCheck.h | 27 ++++++++++ core/unittest/server/CMakeLists.txt | 1 + 8 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 core/src/server/init/InstanceLockCheck.cpp create mode 100644 core/src/server/init/InstanceLockCheck.h diff --git a/CHANGELOG.md b/CHANGELOG.md index 0af71617..39ac26c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ Please mark all change in change log and use the issue from GitHub - \#1962 Add api HasPartition - \#1965 FAISS/NSG/HNSW/ANNOY use unified distance calculation algorithm - \#2054 Check if CPU instruction sets are illegal +- \#2059 Add lock file avoid multiple instances modifying data at the same time - \#2064 Warn when use SQLite as metadata management ## Improvement diff --git a/core/conf/log_config.template b/core/conf/log_config.template index 32864fa2..c5a5be4c 100644 --- a/core/conf/log_config.template +++ b/core/conf/log_config.template @@ -7,6 +7,9 @@ SUBSECOND_PRECISION = 3 PERFORMANCE_TRACKING = false MAX_LOG_FILE_SIZE = 209715200 ## Throw log files away after 200MB +* INFO: + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%y-%M-%d-%H:%m}-info.log" + ENABLED = true * DEBUG: FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%y-%M-%d-%H:%m}-debug.log" ENABLED = true diff --git a/core/src/CMakeLists.txt b/core/src/CMakeLists.txt index 9542de83..32d80cbb 100644 --- a/core/src/CMakeLists.txt +++ b/core/src/CMakeLists.txt @@ -85,6 +85,7 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/server/delivery delivery_files) set(server_files ${server_init_files} ${server_service_files} + ${server_init_files} ${delivery_request_files} ${delivery_hybrid_request_files} ${delivery_strategy_files} diff --git a/core/src/main.cpp b/core/src/main.cpp index c64eb516..3a7f5922 100644 --- a/core/src/main.cpp +++ b/core/src/main.cpp @@ -76,6 +76,7 @@ main(int argc, char* argv[]) { std::string config_filename, log_config_file; std::string pid_filename; std::string app_name = argv[0]; + milvus::Status s; milvus::server::Server& server = milvus::server::Server::GetInstance(); @@ -133,9 +134,11 @@ main(int argc, char* argv[]) { server.Init(start_daemonized, pid_filename, config_filename, log_config_file); - if (server.Start().ok()) { + s = server.Start(); + if (s.ok()) { std::cout << "Milvus server started successfully!" << std::endl; } else { + std::cout << s.message() << std::endl; goto FAIL; } diff --git a/core/src/server/Server.cpp b/core/src/server/Server.cpp index 863e81c4..5f945a50 100644 --- a/core/src/server/Server.cpp +++ b/core/src/server/Server.cpp @@ -10,9 +10,11 @@ // or implied. See the License for the specific language governing permissions and limitations under the License. #include "server/Server.h" +#include "server/init/InstanceLockCheck.h" #include #include +#include #include #include "config/Config.h" @@ -189,6 +191,41 @@ Server::Start() { InitLog(log_config_file_); + std::string deploy_mode; + s = config.GetServerConfigDeployMode(deploy_mode); + if (!s.ok()) { + return s; + } + + if (deploy_mode == "single" || deploy_mode == "cluster_writable") { + std::string db_path; + s = config.GetStorageConfigPrimaryPath(db_path); + if (!s.ok()) { + return s; + } + + s = InstanceLockCheck::Check(db_path); + if (!s.ok()) { + std::cerr << "deploy_mode: " << deploy_mode << " instance lock db path failed." << std::endl; + return s; + } + + std::string wal_path; + s = config.GetWalConfigWalPath(wal_path); + if (!s.ok()) { + return s; + } + + if (not boost::filesystem::create_directories(wal_path)) { + return Status(SERVER_UNEXPECTED_ERROR, "Cannot create wal dir"); + } + s = InstanceLockCheck::Check(wal_path); + if (!s.ok()) { + std::cerr << "deploy_mode: " << deploy_mode << " instance lock wal path failed." << std::endl; + return s; + } + } + // print version information LOG_SERVER_INFO_ << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << ", built at " << BUILD_TIME; #ifdef MILVUS_GPU_VERSION diff --git a/core/src/server/init/InstanceLockCheck.cpp b/core/src/server/init/InstanceLockCheck.cpp new file mode 100644 index 00000000..293bb70a --- /dev/null +++ b/core/src/server/init/InstanceLockCheck.cpp @@ -0,0 +1,62 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. + +#include "server/init/InstanceLockCheck.h" +#include "utils/Log.h" + +#include +#include +#include + +namespace milvus { +namespace server { + +Status +InstanceLockCheck::Check(const std::string& path) { + std::string lock_path = path + "/lock"; + auto fd = open(lock_path.c_str(), O_RDWR | O_CREAT | O_NOFOLLOW, 0640); + if (fd < 0) { + std::string msg; + if (errno == EROFS) { + // Not using locking for read-only lock file + msg += "Lock file is read-only."; + } + msg += "Could not open lock file."; + return Status(SERVER_UNEXPECTED_ERROR, msg); + } + + // Acquire a write lock + struct flock fl; + // exclusive lock + fl.l_type = F_WRLCK; + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; + if (fcntl(fd, F_SETLK, &fl) == -1) { + std::string msg; + if (errno == EACCES || errno == EAGAIN) { + msg += "Permission denied. "; + } else if (errno == ENOLCK) { + // Not using locking for nfs mounted lock file + msg += "Using nfs. "; + } + close(fd); + msg += "Could not get lock."; + return Status(SERVER_UNEXPECTED_ERROR, msg); + } + + LOG_SERVER_INFO_ << "InstanceLockCheck passed."; + + return Status::OK(); +} + +} // namespace server +} // namespace milvus diff --git a/core/src/server/init/InstanceLockCheck.h b/core/src/server/init/InstanceLockCheck.h new file mode 100644 index 00000000..14f88343 --- /dev/null +++ b/core/src/server/init/InstanceLockCheck.h @@ -0,0 +1,27 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. + +#pragma once + +#include +#include "utils/Status.h" + +namespace milvus { +namespace server { + +class InstanceLockCheck { + public: + static Status + Check(const std::string& path); +}; // InstanceLockCheck + +} // namespace server +} // namespace milvus diff --git a/core/unittest/server/CMakeLists.txt b/core/unittest/server/CMakeLists.txt index 7b549ac6..fc2b10ed 100644 --- a/core/unittest/server/CMakeLists.txt +++ b/core/unittest/server/CMakeLists.txt @@ -40,6 +40,7 @@ set(grpc_service_files set(server_test_files ${common_files} ${server_files} + ${server_init_files} ${grpc_server_files} ${grpc_service_files} ${server_delivery_files} -- GitLab