Skip to content

milvus
milvus

qq_34331598 / milvus
与 Fork 源项目一致

Fork自 milvus / milvus

通知 2
Star 0
Fork 0
milvus
milvus

体验新版 GitCode，发现更多精彩内容 >>

未验证 提交 a9df8376 编写于 作者: C Cai Yudong 提交者： GitHub
浏览文件
操作

clean util dependency (#2576) 

* move part of APIs from utils/ValidationUtil to server/ValidationUtils
Signed-off-by: Nyudong.cai <yudong.cai@zilliz.com>

* move part of APIs from utils/ValidationUtil to config/Utils
Signed-off-by: Nyudong.cai <yudong.cai@zilliz.com>

* move part of APIs from utils/CommonUtil to config/Utils
Signed-off-by: Nyudong.cai <yudong.cai@zilliz.com>

* move EraseFromCache from utils/CommonUtil to db/Utils
Signed-off-by: Nyudong.cai <yudong.cai@zilliz.com>

* remove SignalHandler dependency with server
Signed-off-by: Nyudong.cai <yudong.cai@zilliz.com>

* update fiu
Signed-off-by: Nyudong.cai <yudong.cai@zilliz.com>

* update changelog
Signed-off-by: Nyudong.cai <yudong.cai@zilliz.com>

* retry ci
Signed-off-by: Nyudong.cai <yudong.cai@zilliz.com>
上级 aa31b0c2
展开全部 隐藏空白更改
内联 并排
Showing with 655 addition and 574 deletion
CHANGELOG.md
浏览文件 @ a9df8376
# Changelog
Please mark all change in change log and use the issue from GitHub
Please mark all changes in change log and use the issue from GitHub
# Milvus 0.11.0 (TBD)
......@@ -14,6 +14,7 @@ Please mark all change in change log and use the issue from GitHub
## Improvement
- \#2543 Remove secondary_path related code
- \#2561 Clean util dependencies with other modules
## Task
......
core/src/config/Config.cpp
浏览文件 @ a9df8376
......@@ -30,10 +30,8 @@
#include "config/YamlConfigMgr.h"
#include "server/DBWrapper.h"
#include "thirdparty/nlohmann/json.hpp"
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include "utils/StringHelpFunctions.h"
#include "utils/ValidationUtil.h"
namespace milvus {
namespace server {
......@@ -875,7 +873,7 @@ Config::CheckConfigVersion(const std::string& value) {
/* cluster config */
Status
Config::CheckClusterConfigEnable(const std::string& value) {
return ValidationUtil::ValidateStringIsBool(value);
return ValidateStringIsBool(value);
}
Status
......@@ -900,7 +898,7 @@ Config::CheckGeneralConfigTimezone(const std::string& value) {
if (value.substr(0, 3) != "UTC") {
return Status(SERVER_INVALID_ARGUMENT, "Invalid general.timezone: " + value);
} else {
if (!ValidationUtil::IsNumber(value.substr(4))) {
if (!IsNumber(value.substr(4))) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid general.timezone: " + value);
}
}
......@@ -910,7 +908,7 @@ Config::CheckGeneralConfigTimezone(const std::string& value) {
Status
Config::CheckGeneralConfigMetaURI(const std::string& value) {
auto exist_error = !ValidationUtil::ValidateDbURI(value).ok();
auto exist_error = !ValidateDbURI(value).ok();
fiu_do_on("check_config_meta_uri_fail", exist_error = true);
if (exist_error) {
......@@ -925,7 +923,7 @@ Config::CheckGeneralConfigMetaURI(const std::string& value) {
/* network config */
Status
Config::CheckNetworkConfigBindAddress(const std::string& value) {
auto exist_error = !ValidationUtil::ValidateIpAddress(value).ok();
auto exist_error = !ValidateIpAddress(value).ok();
fiu_do_on("check_config_bind_address_fail", exist_error = true);
if (exist_error) {
......@@ -937,7 +935,7 @@ Config::CheckNetworkConfigBindAddress(const std::string& value) {
Status
Config::CheckNetworkConfigBindPort(const std::string& value) {
auto exist_error = !ValidationUtil::ValidateStringIsNumber(value).ok();
auto exist_error = !ValidateStringIsNumber(value).ok();
fiu_do_on("check_config_bind_port_fail", exist_error = true);
if (exist_error) {
......@@ -960,12 +958,12 @@ Config::CheckNetworkConfigBindPort(const std::string& value) {
Status
Config::CheckNetworkConfigHTTPEnable(const std::string& value) {
return ValidationUtil::ValidateStringIsBool(value);
return ValidateStringIsBool(value);
}
Status
Config::CheckNetworkConfigHTTPPort(const std::string& value) {
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
if (!ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid web server port: " + value + ". Possible reason: network.http.port is not a number.";
return Status(SERVER_INVALID_ARGUMENT, msg);
} else {
......@@ -986,7 +984,7 @@ Config::CheckNetworkConfigHTTPPort(const std::string& value) {
/* DB config */
Status
Config::CheckDBConfigArchiveDiskThreshold(const std::string& value) {
auto exist_error = !ValidationUtil::ValidateStringIsNumber(value).ok();
auto exist_error = !ValidateStringIsNumber(value).ok();
fiu_do_on("check_config_archive_disk_threshold_fail", exist_error = true);
if (exist_error) {
......@@ -999,7 +997,7 @@ Config::CheckDBConfigArchiveDiskThreshold(const std::string& value) {
Status
Config::CheckDBConfigArchiveDaysThreshold(const std::string& value) {
auto exist_error = !ValidationUtil::ValidateStringIsNumber(value).ok();
auto exist_error = !ValidateStringIsNumber(value).ok();
fiu_do_on("check_config_archive_days_threshold_fail", exist_error = true);
if (exist_error) {
......@@ -1018,12 +1016,12 @@ Config::CheckStorageConfigPath(const std::string& value) {
return Status(SERVER_INVALID_ARGUMENT, "storage.path is empty.");
}
return ValidationUtil::ValidateStoragePath(value);
return ValidateStoragePath(value);
}
Status
Config::CheckStorageConfigAutoFlushInterval(const std::string& value) {
auto exist_error = !ValidationUtil::ValidateStringIsNumber(value).ok();
auto exist_error = !ValidateStringIsNumber(value).ok();
fiu_do_on("check_config_auto_flush_interval_fail", exist_error = true);
if (exist_error) {
......@@ -1037,7 +1035,7 @@ Config::CheckStorageConfigAutoFlushInterval(const std::string& value) {
Status
Config::CheckStorageConfigFileCleanupTimeout(const std::string& value) {
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
if (!ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid file_cleanup_timeout: " + value +
". Possible reason: storage.file_cleanup_timeout is not a positive integer.";
return Status(SERVER_INVALID_ARGUMENT, msg);
......@@ -1058,7 +1056,7 @@ Config::CheckStorageConfigFileCleanupTimeout(const std::string& value) {
// Status
// Config::CheckStorageConfigS3Enable(const std::string& value) {
// if (!ValidationUtil::ValidateStringIsBool(value).ok()) {
// if (!ValidateStringIsBool(value).ok()) {
// std::string msg =
// "Invalid storage config: " + value + ". Possible reason: storage_config.s3_enable is not a boolean.";
// return Status(SERVER_INVALID_ARGUMENT, msg);
......@@ -1068,7 +1066,7 @@ Config::CheckStorageConfigFileCleanupTimeout(const std::string& value) {
//
// Status
// Config::CheckStorageConfigS3Address(const std::string& value) {
// if (!ValidationUtil::ValidateIpAddress(value).ok()) {
// if (!ValidateIpAddress(value).ok()) {
// std::string msg = "Invalid s3 address: " + value + ". Possible reason: storage_config.s3_address is invalid.";
// return Status(SERVER_INVALID_ARGUMENT, msg);
// }
......@@ -1077,7 +1075,7 @@ Config::CheckStorageConfigFileCleanupTimeout(const std::string& value) {
//
// Status
// Config::CheckStorageConfigS3Port(const std::string& value) {
// if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
// if (!ValidateStringIsNumber(value).ok()) {
// std::string msg = "Invalid s3 port: " + value + ". Possible reason: storage_config.s3_port is not a number.";
// return Status(SERVER_INVALID_ARGUMENT, msg);
// } else {
......@@ -1122,7 +1120,7 @@ Config::CheckStorageConfigFileCleanupTimeout(const std::string& value) {
/* metric config */
Status
Config::CheckMetricConfigEnableMonitor(const std::string& value) {
auto exist_error = !ValidationUtil::ValidateStringIsBool(value).ok();
auto exist_error = !ValidateStringIsBool(value).ok();
fiu_do_on("check_config_enable_monitor_fail", exist_error = true);
if (exist_error) {
......@@ -1134,7 +1132,7 @@ Config::CheckMetricConfigEnableMonitor(const std::string& value) {
Status
Config::CheckMetricConfigAddress(const std::string& value) {
if (!ValidationUtil::ValidateIpAddress(value).ok()) {
if (!ValidateIpAddress(value).ok()) {
std::string msg = "Invalid metric ip: " + value + ". Possible reason: metric.ip is invalid.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
......@@ -1143,7 +1141,7 @@ Config::CheckMetricConfigAddress(const std::string& value) {
Status
Config::CheckMetricConfigPort(const std::string& value) {
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
if (!ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid metric port: " + value + ". Possible reason: metric.port is not a number.";
return Status(SERVER_INVALID_ARGUMENT, msg);
} else {
......@@ -1178,7 +1176,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) {
}
int64_t total_mem = 0, free_mem = 0;
CommonUtil::GetSystemMemInfo(total_mem, free_mem);
GetSystemMemInfo(total_mem, free_mem);
if (cache_size >= total_mem) {
std::string msg =
"Invalid cpu cache size: " + value + ". Possible reason: cache.cache_size exceeds system memory.";
......@@ -1205,7 +1203,7 @@ Status
Config::CheckCacheConfigCpuCacheThreshold(const std::string& value) {
fiu_return_on("check_config_cpu_cache_threshold_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (!ValidationUtil::ValidateStringIsFloat(value).ok()) {
if (!ValidateStringIsFloat(value).ok()) {
std::string msg = "Invalid cpu cache threshold: " + value +
". Possible reason: cache_config.cpu_cache_threshold is not in range (0.0, 1.0].";
return Status(SERVER_INVALID_ARGUMENT, msg);
......@@ -1240,7 +1238,7 @@ Config::CheckCacheConfigInsertBufferSize(const std::string& value) {
int64_t cache_size = parse_bytes(str, err);
int64_t total_mem = 0, free_mem = 0;
CommonUtil::GetSystemMemInfo(total_mem, free_mem);
GetSystemMemInfo(total_mem, free_mem);
if (buffer_size + cache_size >= total_mem) {
std::string msg = "Invalid insert buffer size: " + value +
". Possible reason: sum of cache.cache_size and "
......@@ -1255,7 +1253,7 @@ Status
Config::CheckCacheConfigCacheInsertData(const std::string& value) {
fiu_return_on("check_config_cache_insert_data_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (!ValidationUtil::ValidateStringIsBool(value).ok()) {
if (!ValidateStringIsBool(value).ok()) {
std::string msg = "Invalid cache insert data option: " + value +
". Possible reason: cache_config.cache_insert_data is not a boolean.";
return Status(SERVER_INVALID_ARGUMENT, msg);
......@@ -1277,9 +1275,6 @@ Config::CheckCacheConfigPreloadCollection(const std::string& value) {
std::unordered_set<std::string> table_set;
for (auto& collection : tables) {
if (!ValidationUtil::ValidateCollectionName(collection).ok()) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid collection name: " + collection);
}
bool exist = false;
auto status = DBWrapper::DB()->HasNativeCollection(collection, exist);
if (!(status.ok() && exist)) {
......@@ -1303,7 +1298,7 @@ Status
Config::CheckEngineConfigUseBlasThreshold(const std::string& value) {
fiu_return_on("check_config_use_blas_threshold_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
if (!ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid use blas threshold: " + value +
". Possible reason: engine_config.use_blas_threshold is not a positive integer.";
return Status(SERVER_INVALID_ARGUMENT, msg);
......@@ -1315,7 +1310,7 @@ Status
Config::CheckEngineConfigOmpThreadNum(const std::string& value) {
fiu_return_on("check_config_omp_thread_num_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
if (!ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid omp thread num: " + value +
". Possible reason: engine_config.omp_thread_num is not a positive integer.";
return Status(SERVER_INVALID_ARGUMENT, msg);
......@@ -1323,7 +1318,7 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) {
int64_t omp_thread = std::stoll(value);
int64_t sys_thread_cnt = 8;
CommonUtil::GetSystemAvailableThreads(sys_thread_cnt);
GetSystemAvailableThreads(sys_thread_cnt);
if (omp_thread > sys_thread_cnt) {
std::string msg = "Invalid omp thread num: " + value +
". Possible reason: engine_config.omp_thread_num exceeds system cpu cores.";
......@@ -1349,7 +1344,7 @@ Status
Config::CheckGpuResourceConfigEnable(const std::string& value) {
fiu_return_on("check_config_gpu_resource_enable_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (!ValidationUtil::ValidateStringIsBool(value).ok()) {
if (!ValidateStringIsBool(value).ok()) {
std::string msg = "Invalid gpu resource config: " + value + ". Possible reason: gpu.enable is not a boolean.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
......@@ -1374,7 +1369,7 @@ Config::CheckGpuResourceConfigCacheCapacity(const std::string& value) {
for (int64_t gpu_id : gpu_ids) {
int64_t gpu_memory;
if (!ValidationUtil::GetGpuMemory(gpu_id, gpu_memory).ok()) {
if (!GetGpuMemory(gpu_id, gpu_memory).ok()) {
std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(gpu_id);
return Status(SERVER_UNEXPECTED_ERROR, msg);
} else if (gpu_cache_size >= gpu_memory) {
......@@ -1393,7 +1388,7 @@ Status
Config::CheckGpuResourceConfigCacheThreshold(const std::string& value) {
fiu_return_on("check_config_gpu_resource_cache_threshold_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (!ValidationUtil::ValidateStringIsFloat(value).ok()) {
if (!ValidateStringIsFloat(value).ok()) {
std::string msg = "Invalid gpu cache threshold: " + value +
". Possible reason: gpu.cache_threshold is not in range (0.0, 1.0].";
return Status(SERVER_INVALID_ARGUMENT, msg);
......@@ -1412,7 +1407,7 @@ Status
Config::CheckGpuResourceConfigGpuSearchThreshold(const std::string& value) {
fiu_return_on("check_config_gpu_search_threshold_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
if (!ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid gpu search threshold: " + value +
". Possible reason: gpu.gpu_search_threshold is not a positive integer.";
return Status(SERVER_INVALID_ARGUMENT, msg);
......@@ -1436,7 +1431,7 @@ CheckGpuResource(const std::string& value) {
if (s.compare(0, 3, "gpu") == 0) {
try {
int32_t gpu_index = std::stoi(s.substr(3));
if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) {
if (!ValidateGpuIndex(gpu_index).ok()) {
std::string msg =
"Invalid gpu resource: " + value + ". Possible reason: gpu does not match with the hardware.";
return Status(SERVER_INVALID_ARGUMENT, msg);
......@@ -1515,7 +1510,7 @@ Config::CheckTracingConfigJsonConfigPath(const std::string& value) {
/* wal config */
Status
Config::CheckWalConfigEnable(const std::string& value) {
auto exist_error = !ValidationUtil::ValidateStringIsBool(value).ok();
auto exist_error = !ValidateStringIsBool(value).ok();
fiu_do_on("check_config_wal_enable_fail", exist_error = true);
if (exist_error) {
......@@ -1527,7 +1522,7 @@ Config::CheckWalConfigEnable(const std::string& value) {
Status
Config::CheckWalConfigRecoveryErrorIgnore(const std::string& value) {
auto exist_error = !ValidationUtil::ValidateStringIsBool(value).ok();
auto exist_error = !ValidateStringIsBool(value).ok();
fiu_do_on("check_config_wal_recovery_error_ignore_fail", exist_error = true);
if (exist_error) {
......@@ -1560,7 +1555,7 @@ Config::CheckWalConfigWalPath(const std::string& value) {
return Status(SERVER_INVALID_ARGUMENT, "wal.path is empty!");
}
return ValidationUtil::ValidateStoragePath(value);
return ValidateStoragePath(value);
}
/* logs config */
......@@ -1570,12 +1565,12 @@ Config::CheckLogsLevel(const std::string& value) {
if (value.empty()) {
return Status(SERVER_INVALID_ARGUMENT, "logs.level is empty!");
}
return ValidationUtil::ValidateLogLevel(value);
return ValidateLogLevel(value);
}
Status
Config::CheckLogsTraceEnable(const std::string& value) {
auto exist_error = !ValidationUtil::ValidateStringIsBool(value).ok();
auto exist_error = !ValidateStringIsBool(value).ok();
fiu_do_on("check_logs_trace_enable_fail", exist_error = true);
if (exist_error) {
......@@ -1592,7 +1587,7 @@ Config::CheckLogsPath(const std::string& value) {
return Status(SERVER_INVALID_ARGUMENT, "logs.path is empty!");
}
return ValidationUtil::ValidateStoragePath(value);
return ValidateStoragePath(value);
}
Status
......@@ -1621,7 +1616,7 @@ Config::CheckLogsMaxLogFileSize(const std::string& value) {
Status
Config::CheckLogsLogRotateNum(const std::string& value) {
auto exist_error = !ValidationUtil::ValidateStringIsNumber(value).ok();
auto exist_error = !ValidateStringIsNumber(value).ok();
fiu_do_on("check_logs_log_rotate_num_fail", exist_error = true);
if (exist_error) {
......
core/src/config/Utils.cpp
浏览文件 @ a9df8376
......@@ -10,11 +10,32 @@
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "config/Utils.h"
#include "utils/Log.h"
#include "utils/StringHelpFunctions.h"
#include <arpa/inet.h>
#include <algorithm>
#include <sstream>
#include <cmath>
#ifdef MILVUS_GPU_VERSION
#include <cuda_runtime.h>
#endif
#include <fiu-local.h>
#include <sys/stat.h>
#include <sys/sysinfo.h>
#include <unistd.h>
#include <limits>
#include <regex>
#include <set>
#include <unordered_map>
#if defined(__x86_64__)
#define THREAD_MULTIPLY_CPU 1
#elif defined(__powerpc64__)
#define THREAD_MULTIPLY_CPU 4
#else
#define THREAD_MULTIPLY_CPU 1
#endif
namespace milvus {
namespace server {
......@@ -81,5 +102,214 @@ parse_bytes(const std::string& str, std::string& err) {
return 0;
}
bool
GetSystemMemInfo(int64_t& total_mem, int64_t& free_mem) {
struct sysinfo info;
int ret = sysinfo(&info);
total_mem = info.totalram;
free_mem = info.freeram;
return ret == 0; // succeed 0, failed -1
}
bool
GetSystemAvailableThreads(int64_t& thread_count) {
// threadCnt = std::thread::hardware_concurrency();
thread_count = sysconf(_SC_NPROCESSORS_CONF);
thread_count *= THREAD_MULTIPLY_CPU;
fiu_do_on("GetSystemAvailableThreads.zero_thread", thread_count = 0);
if (thread_count == 0) {
thread_count = 8;
}
return true;
}
Status
ValidateGpuIndex(int32_t gpu_index) {
#ifdef MILVUS_GPU_VERSION
int num_devices = 0;
auto cuda_err = cudaGetDeviceCount(&num_devices);
fiu_do_on("config.ValidateGpuIndex.get_device_count_fail", cuda_err = cudaError::cudaErrorUnknown);
if (cuda_err != cudaSuccess) {
std::string msg = "Failed to get gpu card number, cuda error:" + std::to_string(cuda_err);
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_UNEXPECTED_ERROR, msg);
}
if (gpu_index >= num_devices) {
std::string msg = "Invalid gpu index: " + std::to_string(gpu_index);
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_ARGUMENT, msg);
}
#endif
return Status::OK();
}
#ifdef MILVUS_GPU_VERSION
Status
GetGpuMemory(int32_t gpu_index, int64_t& memory) {
fiu_return_on("config.GetGpuMemory.return_error", Status(SERVER_UNEXPECTED_ERROR, ""));
cudaDeviceProp deviceProp;
auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index);
if (cuda_err) {
std::string msg = "Failed to get gpu properties for gpu" + std::to_string(gpu_index) +
" , cuda error:" + std::to_string(cuda_err);
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_UNEXPECTED_ERROR, msg);
}
memory = deviceProp.totalGlobalMem;
return Status::OK();
}
#endif
Status
ValidateIpAddress(const std::string& ip_address) {
struct in_addr address;
int result = inet_pton(AF_INET, ip_address.c_str(), &address);
fiu_do_on("config.ValidateIpAddress.error_ip_result", result = 2);
switch (result) {
case 1:
return Status::OK();
case 0: {
std::string msg = "Invalid IP address: " + ip_address;
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_ARGUMENT, msg);
}
default: {
std::string msg = "IP address conversion error: " + ip_address;
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_UNEXPECTED_ERROR, msg);
}
}
}
Status
ValidateStringIsNumber(const std::string& str) {
if (str.empty() || !std::all_of(str.begin(), str.end(), ::isdigit)) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid number");
}
try {
int64_t value = std::stol(str);
fiu_do_on("config.ValidateStringIsNumber.throw_exception", throw std::exception());
if (value < 0) {
return Status(SERVER_INVALID_ARGUMENT, "Negative number");
}
} catch (...) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid number");
}
return Status::OK();
}
Status
ValidateStringIsBool(const std::string& str) {
fiu_return_on("ValidateStringNotBool", Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str));
std::string s = str;
std::transform(s.begin(), s.end(), s.begin(), ::tolower);
if (s == "true" || s == "on" || s == "yes" || s == "1" || s == "false" || s == "off" || s == "no" || s == "0" ||
s.empty()) {
return Status::OK();
}
return Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str);
}
Status
ValidateStringIsFloat(const std::string& str) {
try {
float val = std::stof(str);
if (val < 0.0) {
return Status(SERVER_INVALID_ARGUMENT, "Negative float: " + str);
}
} catch (...) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid float: " + str);
}
return Status::OK();
}
Status
ValidateDbURI(const std::string& uri) {
std::string dialectRegex = "(.*)";
std::string usernameRegex = "(.*)";
std::string passwordRegex = "(.*)";
std::string hostRegex = "(.*)";
std::string portRegex = "(.*)";
std::string dbNameRegex = "(.*)";
std::string uriRegexStr = dialectRegex + "\\:\\/\\/" + usernameRegex + "\\:" + passwordRegex + "\\@" + hostRegex +
"\\:" + portRegex + "\\/" + dbNameRegex;
std::regex uriRegex(uriRegexStr);
std::smatch pieces_match;
bool okay = true;
if (std::regex_match(uri, pieces_match, uriRegex)) {
std::string dialect = pieces_match[1].str();
std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower);
if (dialect.find("mysql") == std::string::npos && dialect.find("sqlite") == std::string::npos) {
LOG_SERVER_ERROR_ << "Invalid dialect in URI: dialect = " << dialect;
okay = false;
}
/*
* Could be DNS, skip checking
*
std::string host = pieces_match[4].str();
if (!host.empty() && host != "localhost") {
if (ValidateIpAddress(host) != SERVER_SUCCESS) {
LOG_SERVER_ERROR_ << "Invalid host ip address in uri = " << host;
okay = false;
}
}
*/
std::string port = pieces_match[5].str();
if (!port.empty()) {
auto status = ValidateStringIsNumber(port);
if (!status.ok()) {
LOG_SERVER_ERROR_ << "Invalid port in uri = " << port;
okay = false;
}
}
} else {
LOG_SERVER_ERROR_ << "Wrong URI format: URI = " << uri;
okay = false;
}
return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Invalid db backend uri"));
}
Status
ValidateStoragePath(const std::string& path) {
// Validate storage path if is valid, only correct absolute path will be validated pass
// Invalid path only contain character[a-zA-Z], number[0-9], '-', and '_',
// and path must start with '/'.
// examples below are invalid
// '/a//a', '/a--/a', '/-a/a', '/a@#/a', 'aaa/sfs'
std::string path_pattern = "^\\/(\\w+-?\\/?)+$";
std::regex regex(path_pattern);
return std::regex_match(path, regex) ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Invalid file path");
}
Status
ValidateLogLevel(const std::string& level) {
std::set<std::string> supported_level{"debug", "info", "warning", "error", "fatal"};
return supported_level.find(level) != supported_level.end()
? Status::OK()
: Status(SERVER_INVALID_ARGUMENT, "Log level must be one of debug, info, warning, error and fatal.");
}
bool
IsNumber(const std::string& s) {
return !s.empty() && std::all_of(s.begin(), s.end(), ::isdigit);
}
} // namespace server
} // namespace milvus
core/src/config/Utils.h
浏览文件 @ a9df8376
......@@ -12,12 +12,52 @@
#pragma once
#include <string>
#include <vector>
#include "utils/Status.h"
namespace milvus {
namespace server {
int64_t
extern int64_t
parse_bytes(const std::string& str, std::string& err);
extern bool
GetSystemMemInfo(int64_t& total_mem, int64_t& free_mem);
extern bool
GetSystemAvailableThreads(int64_t& thread_count);
extern Status
ValidateGpuIndex(int32_t gpu_index);
#ifdef MILVUS_GPU_VERSION
extern Status
GetGpuMemory(int32_t gpu_index, int64_t& memory);
#endif
extern Status
ValidateIpAddress(const std::string& ip_address);
extern Status
ValidateStringIsNumber(const std::string& str);
extern Status
ValidateStringIsBool(const std::string& str);
extern Status
ValidateStringIsFloat(const std::string& str);
extern Status
ValidateDbURI(const std::string& uri);
extern Status
ValidateStoragePath(const std::string& path);
extern Status
ValidateLogLevel(const std::string& level);
extern bool
IsNumber(const std::string& s);
} // namespace server
} // namespace milvus
core/src/db/DBImpl.cpp
浏览文件 @ a9df8376
......@@ -31,6 +31,7 @@
#include "Utils.h"
#include "cache/CpuCacheMgr.h"
#include "cache/GpuCacheMgr.h"
#include "config/Utils.h"
#include "db/IDGenerator.h"
#include "db/merge/MergeManagerFactory.h"
#include "engine/EngineFactory.h"
......@@ -52,7 +53,6 @@
#include "utils/Log.h"
#include "utils/StringHelpFunctions.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include "wal/WalDefinations.h"
#include "search/TaskInst.h"
......
core/src/db/Utils.cpp
浏览文件 @ a9df8376
......@@ -20,6 +20,10 @@
#include <regex>
#include <vector>
#include "cache/CpuCacheMgr.h"
#ifdef MILVUS_GPU_VERSION
#include "cache/GpuCacheMgr.h"
#endif
#include "config/Config.h"
//#include "storage/s3/S3ClientWrapper.h"
#include "utils/CommonUtil.h"
......@@ -277,6 +281,25 @@ ExitOnWriteError(Status& status) {
}
}
void
EraseFromCache(const std::string& item_key) {
if (item_key.empty()) {
LOG_SERVER_ERROR_ << "Empty key cannot be erased from cache";
return;
}
cache::CpuCacheMgr::GetInstance()->EraseItem(item_key);
#ifdef MILVUS_GPU_VERSION
server::Config& config = server::Config::GetInstance();
std::vector<int64_t> gpus;
config.GetGpuResourceConfigSearchResources(gpus);
for (auto& gpu : gpus) {
cache::GpuCacheMgr::GetInstance(gpu)->EraseItem(item_key);
}
#endif
}
} // namespace utils
} // namespace engine
} // namespace milvus
core/src/db/Utils.h
浏览文件 @ a9df8376
......@@ -79,6 +79,9 @@ SendExitSignal();
void
ExitOnWriteError(Status& status);
void
EraseFromCache(const std::string& item_key);
} // namespace utils
} // namespace engine
} // namespace milvus
core/src/db/engine/ExecutionEngineImpl.cpp
浏览文件 @ a9df8376
......@@ -49,7 +49,6 @@
#include "utils/Log.h"
#include "utils/Status.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
namespace milvus {
namespace engine {
......
core/src/db/insert/MemTableFile.cpp
浏览文件 @ a9df8376
......@@ -23,7 +23,6 @@
#include "metrics/Metrics.h"
#include "segment/SegmentReader.h"
#include "utils/Log.h"
#include "utils/ValidationUtil.h"
namespace milvus {
namespace engine {
......
core/src/db/meta/MySQLMetaImpl.cpp
浏览文件 @ a9df8376
......@@ -34,11 +34,9 @@
#include "db/IDGenerator.h"
#include "db/Utils.h"
#include "metrics/Metrics.h"
#include "utils/CommonUtil.h"
#include "utils/Exception.h"
#include "utils/Log.h"
#include "utils/StringHelpFunctions.h"
#include "utils/ValidationUtil.h"
namespace milvus {
namespace engine {
......@@ -2615,7 +2613,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/)
// erase file data from cache
// because GetCollectionFilePath won't able to generate file path after the file is deleted
utils::GetCollectionFilePath(options_, collection_file);
server::CommonUtil::EraseFromCache(collection_file.location_);
utils::EraseFromCache(collection_file.location_);
if (collection_file.file_type_ == (int)SegmentSchema::TO_DELETE) {
// delete file from disk storage
......
core/src/db/meta/SqliteMetaImpl.cpp
浏览文件 @ a9df8376
......@@ -25,15 +25,13 @@
#include <sstream>
#include <unordered_map>
#include "MetaConsts.h"
#include "db/IDGenerator.h"
#include "db/Utils.h"
#include "db/meta/MetaConsts.h"
#include "metrics/Metrics.h"
#include "utils/CommonUtil.h"
#include "utils/Exception.h"
#include "utils/Log.h"
#include "utils/StringHelpFunctions.h"
#include "utils/ValidationUtil.h"
#define USING_SQLITE_WARNING \
LOG_ENGINE_WARNING_ << "You are using SQLite as the meta data management, which can't be used in production. " \
......@@ -1868,7 +1866,7 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/
// because GetCollectionFilePath won't able to generate file path after the file is deleted
// TODO(zhiru): clean up
utils::GetCollectionFilePath(options_, collection_file);
server::CommonUtil::EraseFromCache(collection_file.location_);
utils::EraseFromCache(collection_file.location_);
if (collection_file.file_type_ == (int)SegmentSchema::TO_DELETE) {
// delete file from meta
......
core/src/main.cpp
浏览文件 @ a9df8376
......@@ -18,7 +18,7 @@
#include "easyloggingpp/easylogging++.h"
#include "server/Server.h"
#include "src/version.h"
#include "utils/SignalUtil.h"
#include "utils/SignalHandler.h"
#include "utils/Status.h"
INITIALIZE_EASYLOGGINGPP
......@@ -59,6 +59,11 @@ print_banner() {
std::cout << std::endl;
}
/* register signal handler routine */
milvus::server::func_ptr milvus::server::SignalHandler::routine_func_ = []() {
milvus::server::Server::GetInstance().Stop();
};
int
main(int argc, char* argv[]) {
print_banner();
......@@ -117,12 +122,12 @@ main(int argc, char* argv[]) {
}
/* Handle Signal */
signal(SIGHUP, milvus::server::SignalUtil::HandleSignal);
signal(SIGINT, milvus::server::SignalUtil::HandleSignal);
signal(SIGUSR1, milvus::server::SignalUtil::HandleSignal);
signal(SIGSEGV, milvus::server::SignalUtil::HandleSignal);
signal(SIGUSR2, milvus::server::SignalUtil::HandleSignal);
signal(SIGTERM, milvus::server::SignalUtil::HandleSignal);
signal(SIGHUP, milvus::server::SignalHandler::HandleSignal);
signal(SIGINT, milvus::server::SignalHandler::HandleSignal);
signal(SIGUSR1, milvus::server::SignalHandler::HandleSignal);
signal(SIGSEGV, milvus::server::SignalHandler::HandleSignal);
signal(SIGUSR2, milvus::server::SignalHandler::HandleSignal);
signal(SIGTERM, milvus::server::SignalHandler::HandleSignal);
server.Init(start_daemonized, pid_filename, config_filename);
......
core/src/scheduler/task/BuildIndexTask.cpp
浏览文件 @ a9df8376
......@@ -26,7 +26,6 @@
#include "utils/Exception.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
namespace milvus {
namespace scheduler {
......
core/src/scheduler/task/SearchTask.cpp
浏览文件 @ a9df8376
......@@ -29,7 +29,6 @@
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
namespace milvus {
namespace scheduler {
......
core/src/server/DBWrapper.cpp
浏览文件 @ a9df8376
......@@ -19,6 +19,7 @@
#include <faiss/utils/distances.h>
#include "config/Config.h"
#include "config/Utils.h"
#include "db/DBFactory.h"
#include "utils/CommonUtil.h"
#include "utils/Log.h"
......@@ -144,7 +145,7 @@ DBWrapper::StartService() {
LOG_SERVER_DEBUG_ << "Specify openmp thread number: " << omp_thread;
} else {
int64_t sys_thread_cnt = 8;
if (CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) {
if (GetSystemAvailableThreads(sys_thread_cnt)) {
omp_thread = static_cast<int32_t>(ceil(sys_thread_cnt * 0.5));
omp_set_num_threads(omp_thread);
}
......
core/src/server/Server.cpp
浏览文件 @ a9df8376
......@@ -33,7 +33,7 @@
#include "tracing/TracerUtil.h"
#include "utils/Log.h"
#include "utils/LogUtil.h"
#include "utils/SignalUtil.h"
#include "utils/SignalHandler.h"
#include "utils/TimeRecorder.h"
#include "search/TaskInst.h"
......
core/src/utils/ValidationUtil.cpp → core/src/server/ValidationUtil.cpp
浏览文件 @ a9df8376
......@@ -9,35 +9,22 @@
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "utils/ValidationUtil.h"
#include "Log.h"
#include "db/Types.h"
#include "server/ValidationUtil.h"
#include "db/Utils.h"
#include "db/engine/ExecutionEngine.h"
#include "knowhere/index/vector_index/ConfAdapter.h"
#include "knowhere/index/vector_index/helpers/IndexParameter.h"
#include "utils/Log.h"
#include "utils/StringHelpFunctions.h"
#include <arpa/inet.h>
#ifdef MILVUS_GPU_VERSION
#include <cuda_runtime.h>
#endif
#include <fiu-local.h>
#include <algorithm>
#include <cmath>
#include <limits>
#include <regex>
#include <set>
#include <string>
namespace milvus {
namespace server {
namespace {
constexpr size_t COLLECTION_NAME_SIZE_LIMIT = 255;
constexpr int64_t COLLECTION_DIMENSION_LIMIT = 32768;
constexpr int32_t INDEX_FILE_SIZE_LIMIT = 4096; // index trigger size max = 4096 MB
......@@ -104,7 +91,7 @@ CheckParameterExistence(const milvus::json& json_params, const std::string& para
} // namespace
Status
ValidationUtil::ValidateCollectionName(const std::string& collection_name) {
ValidateCollectionName(const std::string& collection_name) {
// Collection name shouldn't be empty.
if (collection_name.empty()) {
std::string msg = "Collection name should not be empty.";
......@@ -142,7 +129,7 @@ ValidationUtil::ValidateCollectionName(const std::string& collection_name) {
}
Status
ValidationUtil::ValidateTableDimension(int64_t dimension, int64_t metric_type) {
ValidateTableDimension(int64_t dimension, int64_t metric_type) {
if (dimension <= 0 || dimension > COLLECTION_DIMENSION_LIMIT) {
std::string msg = "Invalid collection dimension: " + std::to_string(dimension) + ". " +
"The collection dimension must be within the range of 1 ~ " +
......@@ -164,7 +151,7 @@ ValidationUtil::ValidateTableDimension(int64_t dimension, int64_t metric_type) {
}
Status
ValidationUtil::ValidateCollectionIndexType(int32_t index_type) {
ValidateCollectionIndexType(int32_t index_type) {
int engine_type = static_cast<int>(engine::EngineType(index_type));
if (engine_type <= 0 || engine_type > static_cast<int>(engine::EngineType::MAX_VALUE)) {
std::string msg = "Invalid index type: " + std::to_string(index_type) + ". " +
......@@ -186,8 +173,8 @@ ValidationUtil::ValidateCollectionIndexType(int32_t index_type) {
}
Status
ValidationUtil::ValidateIndexParams(const milvus::json& index_params,
const engine::meta::CollectionSchema& collection_schema, int32_t index_type) {
ValidateIndexParams(const milvus::json& index_params, const engine::meta::CollectionSchema& collection_schema,
int32_t index_type) {
switch (index_type) {
case (int32_t)engine::EngineType::FAISS_IDMAP:
case (int32_t)engine::EngineType::FAISS_BIN_IDMAP: {
......@@ -283,8 +270,8 @@ ValidationUtil::ValidateIndexParams(const milvus::json& index_params,
}
Status
ValidationUtil::ValidateSearchParams(const milvus::json& search_params,
const engine::meta::CollectionSchema& collection_schema, int64_t topk) {
ValidateSearchParams(const milvus::json& search_params, const engine::meta::CollectionSchema& collection_schema,
int64_t topk) {
switch (collection_schema.engine_type_) {
case (int32_t)engine::EngineType::FAISS_IDMAP:
case (int32_t)engine::EngineType::FAISS_BIN_IDMAP: {
......@@ -328,8 +315,7 @@ ValidationUtil::ValidateSearchParams(const milvus::json& search_params,
}
Status
ValidationUtil::ValidateVectorData(const engine::VectorsData& vectors,
const engine::meta::CollectionSchema& collection_schema) {
ValidateVectorData(const engine::VectorsData& vectors, const engine::meta::CollectionSchema& collection_schema) {
uint64_t vector_count = vectors.vector_count_;
if ((vectors.float_data_.empty() && vectors.binary_data_.empty()) || vector_count == 0) {
return Status(SERVER_INVALID_ROWRECORD_ARRAY,
......@@ -364,8 +350,7 @@ ValidationUtil::ValidateVectorData(const engine::VectorsData& vectors,
}
Status
ValidationUtil::ValidateVectorDataSize(const engine::VectorsData& vectors,
const engine::meta::CollectionSchema& collection_schema) {
ValidateVectorDataSize(const engine::VectorsData& vectors, const engine::meta::CollectionSchema& collection_schema) {
std::string msg =
"The amount of data inserted each time cannot exceed " + std::to_string(MAX_INSERT_DATA_SIZE / M_BYTE) + " MB";
if (engine::utils::IsBinaryMetricType(collection_schema.metric_type_)) {
......@@ -383,7 +368,7 @@ ValidationUtil::ValidateVectorDataSize(const engine::VectorsData& vectors,
}
Status
ValidationUtil::ValidateCollectionIndexFileSize(int64_t index_file_size) {
ValidateCollectionIndexFileSize(int64_t index_file_size) {
if (index_file_size <= 0 || index_file_size > INDEX_FILE_SIZE_LIMIT) {
std::string msg = "Invalid index file size: " + std::to_string(index_file_size) + ". " +
"The index file size must be within the range of 1 ~ " +
......@@ -396,7 +381,7 @@ ValidationUtil::ValidateCollectionIndexFileSize(int64_t index_file_size) {
}
Status
ValidationUtil::ValidateCollectionIndexMetricType(int32_t metric_type) {
ValidateCollectionIndexMetricType(int32_t metric_type) {
if (metric_type <= 0 || metric_type > static_cast<int32_t>(engine::MetricType::MAX_VALUE)) {
std::string msg = "Invalid index metric type: " + std::to_string(metric_type) + ". " +
"Make sure the metric type is in MetricType list.";
......@@ -407,7 +392,7 @@ ValidationUtil::ValidateCollectionIndexMetricType(int32_t metric_type) {
}
Status
ValidationUtil::ValidateSearchTopk(int64_t top_k) {
ValidateSearchTopk(int64_t top_k) {
if (top_k <= 0 || top_k > QUERY_MAX_TOPK) {
std::string msg =
"Invalid topk: " + std::to_string(top_k) + ". " + "The topk must be within the range of 1 ~ 2048.";
......@@ -419,7 +404,7 @@ ValidationUtil::ValidateSearchTopk(int64_t top_k) {
}
Status
ValidationUtil::ValidatePartitionName(const std::string& partition_name) {
ValidatePartitionName(const std::string& partition_name) {
if (partition_name.empty()) {
std::string msg = "Partition name should not be empty.";
LOG_SERVER_ERROR_ << msg;
......@@ -456,7 +441,7 @@ ValidationUtil::ValidatePartitionName(const std::string& partition_name) {
}
Status
ValidationUtil::ValidatePartitionTags(const std::vector<std::string>& partition_tags) {
ValidatePartitionTags(const std::vector<std::string>& partition_tags) {
for (const std::string& tag : partition_tags) {
// trim side-blank of tag, only compare valid characters
// for example: " ab cd " is treated as "ab cd"
......@@ -479,192 +464,5 @@ ValidationUtil::ValidatePartitionTags(const std::vector<std::string>& partition_
return Status::OK();
}
Status
ValidationUtil::ValidateGpuIndex(int32_t gpu_index) {
#ifdef MILVUS_GPU_VERSION
int num_devices = 0;
auto cuda_err = cudaGetDeviceCount(&num_devices);
fiu_do_on("ValidationUtil.ValidateGpuIndex.get_device_count_fail", cuda_err = cudaError::cudaErrorUnknown);
if (cuda_err != cudaSuccess) {
std::string msg = "Failed to get gpu card number, cuda error:" + std::to_string(cuda_err);
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_UNEXPECTED_ERROR, msg);
}
if (gpu_index >= num_devices) {
std::string msg = "Invalid gpu index: " + std::to_string(gpu_index);
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_ARGUMENT, msg);
}
#endif
return Status::OK();
}
#ifdef MILVUS_GPU_VERSION
Status
ValidationUtil::GetGpuMemory(int32_t gpu_index, int64_t& memory) {
fiu_return_on("ValidationUtil.GetGpuMemory.return_error", Status(SERVER_UNEXPECTED_ERROR, ""));
cudaDeviceProp deviceProp;
auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index);
if (cuda_err) {
std::string msg = "Failed to get gpu properties for gpu" + std::to_string(gpu_index) +
" , cuda error:" + std::to_string(cuda_err);
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_UNEXPECTED_ERROR, msg);
}
memory = deviceProp.totalGlobalMem;
return Status::OK();
}
#endif
Status
ValidationUtil::ValidateIpAddress(const std::string& ip_address) {
struct in_addr address;
int result = inet_pton(AF_INET, ip_address.c_str(), &address);
fiu_do_on("ValidationUtil.ValidateIpAddress.error_ip_result", result = 2);
switch (result) {
case 1:
return Status::OK();
case 0: {
std::string msg = "Invalid IP address: " + ip_address;
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_ARGUMENT, msg);
}
default: {
std::string msg = "IP address conversion error: " + ip_address;
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_UNEXPECTED_ERROR, msg);
}
}
}
Status
ValidationUtil::ValidateStringIsNumber(const std::string& str) {
if (str.empty() || !std::all_of(str.begin(), str.end(), ::isdigit)) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid number");
}
try {
int64_t value = std::stol(str);
fiu_do_on("ValidationUtil.ValidateStringIsNumber.throw_exception", throw std::exception());
if (value < 0) {
return Status(SERVER_INVALID_ARGUMENT, "Negative number");
}
} catch (...) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid number");
}
return Status::OK();
}
Status
ValidationUtil::ValidateStringIsBool(const std::string& str) {
fiu_return_on("ValidateStringNotBool", Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str));
std::string s = str;
std::transform(s.begin(), s.end(), s.begin(), ::tolower);
if (s == "true" || s == "on" || s == "yes" || s == "1" || s == "false" || s == "off" || s == "no" || s == "0" ||
s.empty()) {
return Status::OK();
}
return Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str);
}
Status
ValidationUtil::ValidateStringIsFloat(const std::string& str) {
try {
float val = std::stof(str);
if (val < 0.0) {
return Status(SERVER_INVALID_ARGUMENT, "Negative float: " + str);
}
} catch (...) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid float: " + str);
}
return Status::OK();
}
Status
ValidationUtil::ValidateDbURI(const std::string& uri) {
std::string dialectRegex = "(.*)";
std::string usernameRegex = "(.*)";
std::string passwordRegex = "(.*)";
std::string hostRegex = "(.*)";
std::string portRegex = "(.*)";
std::string dbNameRegex = "(.*)";
std::string uriRegexStr = dialectRegex + "\\:\\/\\/" + usernameRegex + "\\:" + passwordRegex + "\\@" + hostRegex +
"\\:" + portRegex + "\\/" + dbNameRegex;
std::regex uriRegex(uriRegexStr);
std::smatch pieces_match;
bool okay = true;
if (std::regex_match(uri, pieces_match, uriRegex)) {
std::string dialect = pieces_match[1].str();
std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower);
if (dialect.find("mysql") == std::string::npos && dialect.find("sqlite") == std::string::npos) {
LOG_SERVER_ERROR_ << "Invalid dialect in URI: dialect = " << dialect;
okay = false;
}
/*
* Could be DNS, skip checking
*
std::string host = pieces_match[4].str();
if (!host.empty() && host != "localhost") {
if (ValidateIpAddress(host) != SERVER_SUCCESS) {
LOG_SERVER_ERROR_ << "Invalid host ip address in uri = " << host;
okay = false;
}
}
*/
std::string port = pieces_match[5].str();
if (!port.empty()) {
auto status = ValidateStringIsNumber(port);
if (!status.ok()) {
LOG_SERVER_ERROR_ << "Invalid port in uri = " << port;
okay = false;
}
}
} else {
LOG_SERVER_ERROR_ << "Wrong URI format: URI = " << uri;
okay = false;
}
return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Invalid db backend uri"));
}
Status
ValidationUtil::ValidateStoragePath(const std::string& path) {
// Validate storage path if is valid, only correct absolute path will be validated pass
// Invalid path only contain character[a-zA-Z], number[0-9], '-', and '_',
// and path must start with '/'.
// examples below are invalid
// '/a//a', '/a--/a', '/-a/a', '/a@#/a', 'aaa/sfs'
std::string path_pattern = "^\\/(\\w+-?\\/?)+$";
std::regex regex(path_pattern);
return std::regex_match(path, regex) ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Invalid file path");
}
Status
ValidationUtil::ValidateLogLevel(const std::string& level) {
std::set<std::string> supported_level{"debug", "info", "warning", "error", "fatal"};
return supported_level.find(level) != supported_level.end()
? Status::OK()
: Status(SERVER_INVALID_ARGUMENT, "Log level must be one of debug, info, warning, error and fatal.");
}
bool
ValidationUtil::IsNumber(const std::string& s) {
return !s.empty() && std::all_of(s.begin(), s.end(), ::isdigit);
}
} // namespace server
} // namespace milvus
core/src/utils/ValidationUtil.h → core/src/server/ValidationUtil.h
浏览文件 @ a9df8376
......@@ -24,81 +24,42 @@ namespace server {
constexpr int64_t QUERY_MAX_TOPK = 2048;
class ValidationUtil {
private:
ValidationUtil() = default;
extern Status
ValidateCollectionName(const std::string& collection_name);
public:
static Status
ValidateCollectionName(const std::string& collection_name);
extern Status
ValidateTableDimension(int64_t dimension, int64_t metric_type);
static Status
ValidateTableDimension(int64_t dimension, int64_t metric_type);
extern Status
ValidateCollectionIndexType(int32_t index_type);
static Status
ValidateCollectionIndexType(int32_t index_type);
extern Status
ValidateIndexParams(const milvus::json& index_params, const engine::meta::CollectionSchema& collection_schema,
int32_t index_type);
static Status
ValidateIndexParams(const milvus::json& index_params, const engine::meta::CollectionSchema& collection_schema,
int32_t index_type);
extern Status
ValidateSearchParams(const milvus::json& search_params, const engine::meta::CollectionSchema& collection_schema,
int64_t topk);
static Status
ValidateSearchParams(const milvus::json& search_params, const engine::meta::CollectionSchema& collection_schema,
int64_t topk);
extern Status
ValidateVectorData(const engine::VectorsData& vectors, const engine::meta::CollectionSchema& collection_schema);
static Status
ValidateVectorData(const engine::VectorsData& vectors, const engine::meta::CollectionSchema& collection_schema);
extern Status
ValidateVectorDataSize(const engine::VectorsData& vectors, const engine::meta::CollectionSchema& collection_schema);
static Status
ValidateVectorDataSize(const engine::VectorsData& vectors, const engine::meta::CollectionSchema& collection_schema);
extern Status
ValidateCollectionIndexFileSize(int64_t index_file_size);
static Status
ValidateCollectionIndexFileSize(int64_t index_file_size);
extern Status
ValidateCollectionIndexMetricType(int32_t metric_type);
static Status
ValidateCollectionIndexMetricType(int32_t metric_type);
extern Status
ValidateSearchTopk(int64_t top_k);
static Status
ValidateSearchTopk(int64_t top_k);
static Status
ValidatePartitionName(const std::string& partition_name);
static Status
ValidatePartitionTags(const std::vector<std::string>& partition_tags);
static Status
ValidateGpuIndex(int32_t gpu_index);
#ifdef MILVUS_GPU_VERSION
static Status
GetGpuMemory(int32_t gpu_index, int64_t& memory);
#endif
static Status
ValidateIpAddress(const std::string& ip_address);
static Status
ValidateStringIsNumber(const std::string& str);
static Status
ValidateStringIsBool(const std::string& str);
static Status
ValidateStringIsFloat(const std::string& str);
static Status
ValidateDbURI(const std::string& uri);
static Status
ValidateStoragePath(const std::string& path);
static Status
ValidateLogLevel(const std::string& level);
static bool
IsNumber(const std::string& s);
};
extern Status
ValidatePartitionName(const std::string& partition_name);
extern Status
ValidatePartitionTags(const std::vector<std::string>& partition_tags);
} // namespace server
} // namespace milvus
core/src/server/delivery/hybrid_request/CreateHybridCollectionRequest.cpp
浏览文件 @ a9df8376
......@@ -12,11 +12,11 @@
#include "server/delivery/hybrid_request/CreateHybridCollectionRequest.h"
#include "db/Utils.h"
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
#include "server/delivery/request/BaseRequest.h"
#include "server/web_impl/Constants.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
......@@ -56,7 +56,7 @@ CreateHybridCollectionRequest::OnExecute() {
try {
// step 1: check arguments
auto status = ValidationUtil::ValidateCollectionName(collection_name_);
auto status = ValidateCollectionName(collection_name_);
fiu_do_on("CreateHybridCollectionRequest.OnExecute.invalid_collection_name",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
......
core/src/server/delivery/hybrid_request/DescribeHybridCollectionRequest.cpp
浏览文件 @ a9df8376
......@@ -16,7 +16,6 @@
#include "server/web_impl/Constants.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
......
core/src/server/delivery/hybrid_request/GetEntityByIDRequest.cpp
浏览文件 @ a9df8376
......@@ -17,9 +17,9 @@
#include "server/delivery/hybrid_request/GetEntityByIDRequest.h"
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <memory>
#include <vector>
......@@ -63,7 +63,7 @@ GetEntityByIDRequest::OnExecute() {
return Status(SERVER_INVALID_ARGUMENT, msg);
}
auto status = ValidationUtil::ValidateCollectionName(collection_name_);
auto status = ValidateCollectionName(collection_name_);
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/hybrid_request/HybridSearchRequest.cpp
浏览文件 @ a9df8376
......@@ -12,10 +12,10 @@
#include "server/delivery/hybrid_request/HybridSearchRequest.h"
#include "db/Utils.h"
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
......@@ -62,7 +62,7 @@ HybridSearchRequest::OnExecute() {
TimeRecorder rc(hdr);
// step 1: check table name
auto status = ValidationUtil::ValidateCollectionName(collection_name_);
auto status = ValidateCollectionName(collection_name_);
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/hybrid_request/InsertEntityRequest.cpp
浏览文件 @ a9df8376
......@@ -12,10 +12,10 @@
#include "server/delivery/hybrid_request/InsertEntityRequest.h"
#include "db/Utils.h"
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
......@@ -61,7 +61,7 @@ InsertEntityRequest::OnExecute() {
TimeRecorder rc(hdr);
// step 1: check arguments
auto status = ValidationUtil::ValidateCollectionName(collection_name_);
auto status = ValidateCollectionName(collection_name_);
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/CompactRequest.cpp
浏览文件 @ a9df8376
......@@ -17,9 +17,9 @@
#include "server/delivery/request/CompactRequest.h"
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <memory>
......@@ -46,7 +46,7 @@ CompactRequest::OnExecute() {
TimeRecorderAuto rc(hdr);
// step 1: check arguments
auto status = ValidationUtil::ValidateCollectionName(collection_name_);
auto status = ValidateCollectionName(collection_name_);
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/CountCollectionRequest.cpp
浏览文件 @ a9df8376
......@@ -12,9 +12,9 @@
#include "server/delivery/request/CountCollectionRequest.h"
#include "BaseRequest.h"
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
......@@ -40,7 +40,7 @@ CountCollectionRequest::OnExecute() {
TimeRecorderAuto rc(hdr);
// step 1: check arguments
auto status = ValidationUtil::ValidateCollectionName(collection_name_);
auto status = ValidateCollectionName(collection_name_);
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/CreateCollectionRequest.cpp
浏览文件 @ a9df8376
......@@ -12,10 +12,10 @@
#include "server/delivery/request/CreateCollectionRequest.h"
#include "db/Utils.h"
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
#include "server/delivery/request/BaseRequest.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
......@@ -50,24 +50,24 @@ CreateCollectionRequest::OnExecute() {
try {
// step 1: check arguments
auto status = ValidationUtil::ValidateCollectionName(collection_name_);
auto status = ValidateCollectionName(collection_name_);
if (!status.ok()) {
return status;
}
status = ValidationUtil::ValidateTableDimension(dimension_, metric_type_);
status = ValidateTableDimension(dimension_, metric_type_);
if (!status.ok()) {
return status;
}
status = ValidationUtil::ValidateCollectionIndexFileSize(index_file_size_);
status = ValidateCollectionIndexFileSize(index_file_size_);
fiu_do_on("CreateCollectionRequest.OnExecute.invalid_index_file_size",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
status = ValidationUtil::ValidateCollectionIndexMetricType(metric_type_);
status = ValidateCollectionIndexMetricType(metric_type_);
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/CreateIndexRequest.cpp
浏览文件 @ a9df8376
......@@ -13,9 +13,9 @@
#include "config/Config.h"
#include "db/Utils.h"
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
......@@ -46,7 +46,7 @@ CreateIndexRequest::OnExecute() {
TimeRecorderAuto rc(hdr);
// step 1: check arguments
auto status = ValidationUtil::ValidateCollectionName(collection_name_);
auto status = ValidateCollectionName(collection_name_);
if (!status.ok()) {
return status;
}
......@@ -70,12 +70,12 @@ CreateIndexRequest::OnExecute() {
}
}
status = ValidationUtil::ValidateCollectionIndexType(index_type_);
status = ValidateCollectionIndexType(index_type_);
if (!status.ok()) {
return status;
}
status = ValidationUtil::ValidateIndexParams(json_params_, collection_schema, index_type_);
status = ValidateIndexParams(json_params_, collection_schema, index_type_);
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/CreatePartitionRequest.cpp
浏览文件 @ a9df8376
......@@ -11,9 +11,9 @@
#include "server/delivery/request/CreatePartitionRequest.h"
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
......@@ -43,7 +43,7 @@ CreatePartitionRequest::OnExecute() {
try {
// step 1: check arguments
auto status = ValidationUtil::ValidateCollectionName(collection_name_);
auto status = ValidateCollectionName(collection_name_);
fiu_do_on("CreatePartitionRequest.OnExecute.invalid_collection_name",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
......@@ -54,7 +54,7 @@ CreatePartitionRequest::OnExecute() {
return Status(SERVER_INVALID_PARTITION_TAG, "'_default' is built-in partition tag");
}
status = ValidationUtil::ValidatePartitionTags({tag_});
status = ValidatePartitionTags({tag_});
fiu_do_on("CreatePartitionRequest.OnExecute.invalid_partition_name",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
......
core/src/server/delivery/request/DeleteByIDRequest.cpp
浏览文件 @ a9df8376
......@@ -22,9 +22,9 @@
#include <vector>
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
namespace milvus {
namespace server {
......@@ -46,7 +46,7 @@ DeleteByIDRequest::OnExecute() {
TimeRecorderAuto rc("DeleteByIDRequest");
// step 1: check arguments
auto status = ValidationUtil::ValidateCollectionName(collection_name_);
auto status = ValidateCollectionName(collection_name_);
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/DescribeCollectionRequest.cpp
浏览文件 @ a9df8376
......@@ -11,9 +11,9 @@
#include "server/delivery/request/DescribeCollectionRequest.h"
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
......@@ -39,7 +39,7 @@ DescribeCollectionRequest::OnExecute() {
try {
// step 1: check arguments
auto status = ValidationUtil::ValidateCollectionName(collection_name_);
auto status = ValidateCollectionName(collection_name_);
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/DescribeIndexRequest.cpp
浏览文件 @ a9df8376
......@@ -11,9 +11,9 @@
#include "server/delivery/request/DescribeIndexRequest.h"
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
......@@ -40,7 +40,7 @@ DescribeIndexRequest::OnExecute() {
TimeRecorderAuto rc(hdr);
// step 1: check arguments
auto status = ValidationUtil::ValidateCollectionName(collection_name_);
auto status = ValidateCollectionName(collection_name_);
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/DropCollectionRequest.cpp
浏览文件 @ a9df8376
......@@ -11,9 +11,9 @@
#include "server/delivery/request/DropCollectionRequest.h"
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
......@@ -40,7 +40,7 @@ DropCollectionRequest::OnExecute() {
TimeRecorder rc(hdr);
// step 1: check arguments
auto status = ValidationUtil::ValidateCollectionName(collection_name_);
auto status = ValidateCollectionName(collection_name_);
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/DropIndexRequest.cpp
浏览文件 @ a9df8376
......@@ -11,9 +11,9 @@
#include "server/delivery/request/DropIndexRequest.h"
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
......@@ -39,7 +39,7 @@ DropIndexRequest::OnExecute() {
TimeRecorderAuto rc(hdr);
// step 1: check arguments
auto status = ValidationUtil::ValidateCollectionName(collection_name_);
auto status = ValidateCollectionName(collection_name_);
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/DropPartitionRequest.cpp
浏览文件 @ a9df8376
......@@ -11,9 +11,9 @@
#include "server/delivery/request/DropPartitionRequest.h"
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
......@@ -42,7 +42,7 @@ DropPartitionRequest::OnExecute() {
std::string partition_tag = tag_;
// step 1: check collection name
auto status = ValidationUtil::ValidateCollectionName(collection_name);
auto status = ValidateCollectionName(collection_name);
fiu_do_on("DropPartitionRequest.OnExecute.invalid_collection_name",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
......@@ -56,7 +56,7 @@ DropPartitionRequest::OnExecute() {
return Status(SERVER_INVALID_COLLECTION_NAME, msg);
}
status = ValidationUtil::ValidatePartitionTags({partition_tag});
status = ValidatePartitionTags({partition_tag});
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/GetVectorIDsRequest.cpp
浏览文件 @ a9df8376
......@@ -17,9 +17,9 @@
#include "server/delivery/request/GetVectorIDsRequest.h"
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <memory>
#include <vector>
......@@ -49,7 +49,7 @@ GetVectorIDsRequest::OnExecute() {
TimeRecorderAuto rc(hdr);
// step 1: check arguments
auto status = ValidationUtil::ValidateCollectionName(collection_name_);
auto status = ValidateCollectionName(collection_name_);
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/GetVectorsByIDRequest.cpp
浏览文件 @ a9df8376
......@@ -17,9 +17,9 @@
#include "server/delivery/request/GetVectorsByIDRequest.h"
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <memory>
#include <vector>
......@@ -61,7 +61,7 @@ GetVectorsByIDRequest::OnExecute() {
return Status(SERVER_INVALID_ARGUMENT, msg);
}
auto status = ValidationUtil::ValidateCollectionName(collection_name_);
auto status = ValidateCollectionName(collection_name_);
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/HasCollectionRequest.cpp
浏览文件 @ a9df8376
......@@ -11,9 +11,9 @@
#include "server/delivery/request/HasCollectionRequest.h"
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
......@@ -41,7 +41,7 @@ HasCollectionRequest::OnExecute() {
TimeRecorderAuto rc(hdr);
// step 1: check arguments
auto status = ValidationUtil::ValidateCollectionName(collection_name_);
auto status = ValidateCollectionName(collection_name_);
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/HasPartitionRequest.cpp
浏览文件 @ a9df8376
......@@ -11,9 +11,9 @@
#include "server/delivery/request/HasPartitionRequest.h"
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
......@@ -46,7 +46,7 @@ HasPartitionRequest::OnExecute() {
has_partition_ = false;
// step 1: check arguments
auto status = ValidationUtil::ValidateCollectionName(collection_name_);
auto status = ValidateCollectionName(collection_name_);
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/InsertRequest.cpp
浏览文件 @ a9df8376
......@@ -12,10 +12,10 @@
#include "server/delivery/request/InsertRequest.h"
#include "db/Utils.h"
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
......@@ -55,7 +55,7 @@ InsertRequest::OnExecute() {
TimeRecorder rc(LogOut("[%s][%ld] %s", "insert", 0, hdr.c_str()));
// step 1: check arguments
auto status = ValidationUtil::ValidateCollectionName(collection_name_);
auto status = ValidateCollectionName(collection_name_);
if (!status.ok()) {
LOG_SERVER_ERROR_ << LogOut("[%s][%ld] Invalid collection name: %s", "insert", 0, status.message().c_str());
return status;
......@@ -129,14 +129,14 @@ InsertRequest::OnExecute() {
ProfilerStart(fname.c_str());
#endif
// step 4: some metric type doesn't support float vectors
status = ValidationUtil::ValidateVectorData(vectors_data_, collection_schema);
status = ValidateVectorData(vectors_data_, collection_schema);
if (!status.ok()) {
LOG_SERVER_ERROR_ << LogOut("[%s][%d] Invalid vector data: %s", "insert", 0, status.message().c_str());
return status;
}
// step 5: check insert data limitation
status = ValidationUtil::ValidateVectorDataSize(vectors_data_, collection_schema);
status = ValidateVectorDataSize(vectors_data_, collection_schema);
if (!status.ok()) {
LOG_SERVER_ERROR_ << LogOut("[%s][%d] Invalid vector data: %s", "insert", 0, status.message().c_str());
return status;
......
core/src/server/delivery/request/PreloadCollectionRequest.cpp
浏览文件 @ a9df8376
......@@ -11,9 +11,9 @@
#include "server/delivery/request/PreloadCollectionRequest.h"
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
......@@ -39,7 +39,7 @@ PreloadCollectionRequest::OnExecute() {
TimeRecorderAuto rc(hdr);
// step 1: check arguments
auto status = ValidationUtil::ValidateCollectionName(collection_name_);
auto status = ValidateCollectionName(collection_name_);
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/ReLoadSegmentsRequest.cpp
浏览文件 @ a9df8376
......@@ -15,8 +15,8 @@
#include "config/Config.h"
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
namespace milvus {
namespace server {
......@@ -52,7 +52,7 @@ ReLoadSegmentsRequest::OnExecute() {
TimeRecorderAuto rc(hdr);
// step 1: check arguments
auto status = ValidationUtil::ValidateCollectionName(collection_name_);
auto status = ValidateCollectionName(collection_name_);
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/SearchByIDRequest.cpp
浏览文件 @ a9df8376
......@@ -21,10 +21,10 @@
#include "config/Config.h"
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#ifdef ENABLE_CPU_PROFILING
#include <gperftools/profiler.h>
......@@ -70,13 +70,13 @@ SearchByIDRequest::OnExecute() {
}
// step 2: check collection name
auto status = ValidationUtil::ValidateCollectionName(collection_name_);
auto status = ValidateCollectionName(collection_name_);
if (!status.ok()) {
return status;
}
// step 3: check search topk
status = ValidationUtil::ValidateSearchTopk(topk_);
status = ValidateSearchTopk(topk_);
if (!status.ok()) {
return status;
}
......@@ -99,7 +99,7 @@ SearchByIDRequest::OnExecute() {
}
// step 5: check search parameters
status = ValidationUtil::ValidateSearchParams(extra_params_, collection_schema, topk_);
status = ValidateSearchParams(extra_params_, collection_schema, topk_);
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/SearchCombineRequest.cpp
浏览文件 @ a9df8376
......@@ -12,11 +12,11 @@
#include "server/delivery/request/SearchCombineRequest.h"
#include "db/Utils.h"
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
#include "server/context/Context.h"
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <memory>
#include <set>
......@@ -106,12 +106,12 @@ SearchCombineRequest::Combine(const SearchRequestPtr& request) {
// reset some parameters in necessary
if (request_list_.empty()) {
// validate first request input
auto status = ValidationUtil::ValidateCollectionName(request->CollectionName());
auto status = ValidateCollectionName(request->CollectionName());
if (!status.ok()) {
return status;
}
status = ValidationUtil::ValidateSearchTopk(request->TopK());
status = ValidateSearchTopk(request->TopK());
if (!status.ok()) {
return status;
}
......@@ -267,7 +267,7 @@ SearchCombineRequest::OnExecute() {
std::vector<SearchRequestPtr>::iterator iter = request_list_.begin();
for (; iter != request_list_.end();) {
SearchRequestPtr& request = *iter;
status = ValidationUtil::ValidateSearchTopk(request->TopK());
status = ValidateSearchTopk(request->TopK());
if (!status.ok()) {
// check failed, erase request and let it return error status
FreeRequest(request, status);
......@@ -275,7 +275,7 @@ SearchCombineRequest::OnExecute() {
continue;
}
status = ValidationUtil::ValidateSearchParams(extra_params_, collection_schema, request->TopK());
status = ValidateSearchParams(extra_params_, collection_schema, request->TopK());
if (!status.ok()) {
// check failed, erase request and let it return error status
FreeRequest(request, status);
......@@ -283,7 +283,7 @@ SearchCombineRequest::OnExecute() {
continue;
}
status = ValidationUtil::ValidateVectorData(request->VectorsData(), collection_schema);
status = ValidateVectorData(request->VectorsData(), collection_schema);
if (!status.ok()) {
// check failed, erase request and let it return error status
FreeRequest(request, status);
......@@ -291,7 +291,7 @@ SearchCombineRequest::OnExecute() {
continue;
}
status = ValidationUtil::ValidatePartitionTags(request->PartitionList());
status = ValidatePartitionTags(request->PartitionList());
if (!status.ok()) {
// check failed, erase request and let it return error status
FreeRequest(request, status);
......
core/src/server/delivery/request/SearchRequest.cpp
浏览文件 @ a9df8376
......@@ -17,10 +17,10 @@
#include "db/Utils.h"
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#ifdef ENABLE_CPU_PROFILING
#include <gperftools/profiler.h>
......@@ -60,21 +60,21 @@ SearchRequest::OnPreExecute() {
milvus::server::ContextChild tracer_pre(context_, "Pre Query");
// step 1: check collection name
auto status = ValidationUtil::ValidateCollectionName(collection_name_);
auto status = ValidateCollectionName(collection_name_);
if (!status.ok()) {
LOG_SERVER_ERROR_ << LogOut("[%s][%ld] %s", "search", 0, status.message().c_str());
return status;
}
// step 2: check search topk
status = ValidationUtil::ValidateSearchTopk(topk_);
status = ValidateSearchTopk(topk_);
if (!status.ok()) {
LOG_SERVER_ERROR_ << LogOut("[%s][%ld] %s", "search", 0, status.message().c_str());
return status;
}
// step 3: check partition tags
status = ValidationUtil::ValidatePartitionTags(partition_list_);
status = ValidatePartitionTags(partition_list_);
fiu_do_on("SearchRequest.OnExecute.invalid_partition_tags", status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
LOG_SERVER_ERROR_ << LogOut("[%s][%ld] %s", "search", 0, status.message().c_str());
......@@ -120,14 +120,14 @@ SearchRequest::OnExecute() {
}
// step 5: check search parameters
status = ValidationUtil::ValidateSearchParams(extra_params_, collection_schema_, topk_);
status = ValidateSearchParams(extra_params_, collection_schema_, topk_);
if (!status.ok()) {
LOG_SERVER_ERROR_ << LogOut("[%s][%ld] Invalid search params: %s", "search", 0, status.message().c_str());
return status;
}
// step 6: check vector data according to metric type
status = ValidationUtil::ValidateVectorData(vectors_data_, collection_schema_);
status = ValidateVectorData(vectors_data_, collection_schema_);
if (!status.ok()) {
LOG_SERVER_ERROR_ << LogOut("[%s][%ld] Invalid vector data: %s", "search", 0, status.message().c_str());
return status;
......
core/src/server/delivery/request/ShowCollectionInfoRequest.cpp
浏览文件 @ a9df8376
......@@ -17,9 +17,9 @@
#include "server/delivery/request/ShowCollectionInfoRequest.h"
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <memory>
#include <vector>
......@@ -46,7 +46,7 @@ ShowCollectionInfoRequest::OnExecute() {
TimeRecorderAuto rc(hdr);
// step 1: check collection name
auto status = ValidationUtil::ValidateCollectionName(collection_name_);
auto status = ValidateCollectionName(collection_name_);
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/ShowPartitionsRequest.cpp
浏览文件 @ a9df8376
......@@ -11,9 +11,9 @@
#include "server/delivery/request/ShowPartitionsRequest.h"
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
......@@ -42,7 +42,7 @@ ShowPartitionsRequest::OnExecute() {
TimeRecorderAuto rc(hdr);
// step 1: check collection name
auto status = ValidationUtil::ValidateCollectionName(collection_name_);
auto status = ValidateCollectionName(collection_name_);
fiu_do_on("ShowPartitionsRequest.OnExecute.invalid_collection_name",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
......
core/src/server/web_impl/handler/WebRequestHandler.cpp
浏览文件 @ a9df8376
......@@ -20,6 +20,7 @@
#include <fiu-local.h>
#include "config/Config.h"
#include "config/Utils.h"
#include "metrics/SystemInfo.h"
#include "server/delivery/request/BaseRequest.h"
#include "server/web_impl/Constants.h"
......@@ -28,7 +29,6 @@
#include "server/web_impl/utils/Util.h"
#include "thirdparty/nlohmann/json.hpp"
#include "utils/StringHelpFunctions.h"
#include "utils/ValidationUtil.h"
namespace milvus {
namespace server {
......@@ -885,7 +885,7 @@ WebRequestHandler::DeleteByIDs(const std::string& collection_name, const nlohman
for (auto& id : ids) {
auto id_str = id.get<std::string>();
if (!ValidationUtil::ValidateStringIsNumber(id_str).ok()) {
if (!ValidateStringIsNumber(id_str).ok()) {
return Status(ILLEGAL_BODY, "Members in \"ids\" must be integer string");
}
vector_ids.emplace_back(std::stol(id_str));
......@@ -1520,7 +1520,7 @@ WebRequestHandler::ShowPartitions(const OString& collection_name, const OQueryPa
auto required = query_params.get("all_required");
if (nullptr != required.get()) {
auto required_str = required->std_str();
if (!ValidationUtil::ValidateStringIsBool(required_str).ok()) {
if (!ValidateStringIsBool(required_str).ok()) {
RETURN_STATUS_DTO(ILLEGAL_QUERY_PARAM, "Query param \'all_required\' must be a bool")
}
all_required = required_str == "True" || required_str == "true";
......@@ -1596,7 +1596,7 @@ WebRequestHandler::ShowSegments(const OString& collection_name, const OQueryPara
auto required = query_params.get("all_required");
if (nullptr != required.get()) {
auto required_str = required->std_str();
if (!ValidationUtil::ValidateStringIsBool(required_str).ok()) {
if (!ValidateStringIsBool(required_str).ok()) {
RETURN_STATUS_DTO(ILLEGAL_QUERY_PARAM, "Query param \'all_required\' must be a bool")
}
all_required = required_str == "True" || required_str == "true";
......
core/src/server/web_impl/utils/Util.cpp
浏览文件 @ a9df8376
......@@ -11,7 +11,7 @@
#include "server/web_impl/utils/Util.h"
#include "utils/ValidationUtil.h"
#include "config/Utils.h"
namespace milvus {
namespace server {
......@@ -22,7 +22,7 @@ ParseQueryInteger(const OQueryParams& query_params, const std::string& key, int6
auto query = query_params.get(key.c_str());
if (nullptr != query.get() && query->getSize() > 0) {
std::string value_str = query->std_str();
if (!ValidationUtil::ValidateStringIsNumber(value_str).ok()) {
if (!ValidateStringIsNumber(value_str).ok()) {
return Status(ILLEGAL_QUERY_PARAM,
"Query param \'offset\' is illegal, only non-negative integer supported");
}
......@@ -52,7 +52,7 @@ ParseQueryBool(const OQueryParams& query_params, const std::string& key, bool& v
auto query = query_params.get(key.c_str());
if (nullptr != query.get() && query->getSize() > 0) {
std::string value_str = query->std_str();
if (!ValidationUtil::ValidateStringIsBool(value_str).ok()) {
if (!ValidateStringIsBool(value_str).ok()) {
return Status(ILLEGAL_QUERY_PARAM, "Query param \'all_required\' must be a bool");
}
value = value_str == "True" || value_str == "true";
......
core/src/utils/CommonUtil.cpp
浏览文件 @ a9df8376
......@@ -10,63 +10,23 @@
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "utils/CommonUtil.h"
#include "cache/CpuCacheMgr.h"
#include "cache/GpuCacheMgr.h"
#include "config/Config.h"
#include "utils/Log.h"
#include <dirent.h>
#include <fiu-local.h>
#include <pwd.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/sysinfo.h>
#include <time.h>
#include <unistd.h>
#include <boost/filesystem.hpp>
#include <iostream>
#include <thread>
#include <vector>
#include "boost/filesystem.hpp"
#if defined(__x86_64__)
#define THREAD_MULTIPLY_CPU 1
#elif defined(__powerpc64__)
#define THREAD_MULTIPLY_CPU 4
#else
#define THREAD_MULTIPLY_CPU 1
#endif
#include <fiu-local.h>
namespace milvus {
namespace server {
namespace fs = boost::filesystem;
bool
CommonUtil::GetSystemMemInfo(int64_t& total_mem, int64_t& free_mem) {
struct sysinfo info;
int ret = sysinfo(&info);
total_mem = info.totalram;
free_mem = info.freeram;
return ret == 0; // succeed 0, failed -1
}
bool
CommonUtil::GetSystemAvailableThreads(int64_t& thread_count) {
// threadCnt = std::thread::hardware_concurrency();
thread_count = sysconf(_SC_NPROCESSORS_CONF);
thread_count *= THREAD_MULTIPLY_CPU;
fiu_do_on("CommonUtil.GetSystemAvailableThreads.zero_thread", thread_count = 0);
if (thread_count == 0) {
thread_count = 8;
}
return true;
}
bool
CommonUtil::IsDirectoryExist(const std::string& path) {
DIR* dp = nullptr;
......@@ -244,24 +204,5 @@ CommonUtil::GetCurrentTimeStr() {
}
#endif
void
CommonUtil::EraseFromCache(const std::string& item_key) {
if (item_key.empty()) {
LOG_SERVER_ERROR_ << "Empty key cannot be erased from cache";
return;
}
cache::CpuCacheMgr::GetInstance()->EraseItem(item_key);
#ifdef MILVUS_GPU_VERSION
server::Config& config = server::Config::GetInstance();
std::vector<int64_t> gpus;
config.GetGpuResourceConfigSearchResources(gpus);
for (auto& gpu : gpus) {
cache::GpuCacheMgr::GetInstance(gpu)->EraseItem(item_key);
}
#endif
}
} // namespace server
} // namespace milvus
core/src/utils/CommonUtil.h
浏览文件 @ a9df8376
......@@ -21,11 +21,6 @@ namespace server {
class CommonUtil {
public:
static bool
GetSystemMemInfo(int64_t& total_mem, int64_t& free_mem);
static bool
GetSystemAvailableThreads(int64_t& thread_count);
static bool
IsFileExist(const std::string& path);
static uint64_t
......@@ -55,9 +50,6 @@ class CommonUtil {
static std::string
GetCurrentTimeStr();
#endif
static void
EraseFromCache(const std::string& item_key);
};
} // namespace server
......
core/src/utils/SignalUtil.cpp → core/src/utils/SignalHandler.cpp
浏览文件 @ a9df8376
......@@ -9,8 +9,7 @@
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "utils/SignalUtil.h"
#include "src/server/Server.h"
#include "utils/SignalHandler.h"
#include "utils/Log.h"
#include <execinfo.h>
......@@ -21,31 +20,29 @@ namespace milvus {
namespace server {
void
SignalUtil::HandleSignal(int signum) {
SignalHandler::HandleSignal(int signum) {
switch (signum) {
case SIGINT:
case SIGUSR2: {
LOG_SERVER_INFO_ << "Server received signal: " << signum;
server::Server& server = server::Server::GetInstance();
server.Stop();
if (routine_func_ != nullptr) {
(*routine_func_)();
}
exit(0);
}
default: {
LOG_SERVER_INFO_ << "Server received critical signal: " << signum;
SignalUtil::PrintStacktrace();
server::Server& server = server::Server::GetInstance();
server.Stop();
SignalHandler::PrintStacktrace();
if (routine_func_ != nullptr) {
(*routine_func_)();
}
exit(1);
}
}
}
void
SignalUtil::PrintStacktrace() {
SignalHandler::PrintStacktrace() {
LOG_SERVER_INFO_ << "Call stack:";
const int size = 32;
......
core/src/utils/SignalUtil.h → core/src/utils/SignalHandler.h
浏览文件 @ a9df8376
......@@ -14,12 +14,18 @@
namespace milvus {
namespace server {
class SignalUtil {
typedef void (*func_ptr)(void);
class SignalHandler {
public:
static void
HandleSignal(int signum);
static void
PrintStacktrace();
private:
static func_ptr routine_func_;
};
} // namespace server
......
core/src/utils/StringHelpFunctions.cpp
浏览文件 @ a9df8376
......@@ -16,8 +16,6 @@
#include <regex>
#include <string>
#include "utils/ValidationUtil.h"
namespace milvus {
namespace server {
......@@ -156,11 +154,6 @@ StringHelpFunctions::IsRegexMatch(const std::string& target_str, const std::stri
Status
StringHelpFunctions::ConvertToBoolean(const std::string& str, bool& value) {
auto status = ValidationUtil::ValidateStringIsBool(str);
if (!status.ok()) {
return status;
}
std::string s = str;
std::transform(s.begin(), s.end(), s.begin(), ::tolower);
value = s == "true" || s == "on" || s == "yes" || s == "1";
......
core/unittest/CMakeLists.txt
浏览文件 @ a9df8376
......@@ -133,7 +133,6 @@ set(helper_files
${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp
${MILVUS_ENGINE_SRC}/utils/Status.cpp
${MILVUS_ENGINE_SRC}/utils/StringHelpFunctions.cpp
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp
${MILVUS_THIRDPARTY_SRC}/easyloggingpp/easylogging++.cc
)
......
core/unittest/server/CMakeLists.txt
浏览文件 @ a9df8376
......@@ -26,7 +26,7 @@ link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
set(util_files
${MILVUS_ENGINE_SRC}/utils/StringHelpFunctions.cpp
${MILVUS_ENGINE_SRC}/utils/LogUtil.cpp
${MILVUS_ENGINE_SRC}/utils/SignalUtil.cpp)
${MILVUS_ENGINE_SRC}/utils/SignalHandler.cpp)
set(server_test_files
${common_files}
......
core/unittest/server/test_config.cpp
浏览文件 @ a9df8376
......@@ -19,10 +19,9 @@
#include "config/Config.h"
#include "config/YamlConfigMgr.h"
#include "config/Utils.h"
#include "server/utils.h"
#include "utils/CommonUtil.h"
#include "utils/StringHelpFunctions.h"
#include "utils/ValidationUtil.h"
namespace {
......@@ -259,7 +258,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
{
// #2564
int64_t total_mem = 0, free_mem = 0;
milvus::server::CommonUtil::GetSystemMemInfo(total_mem, free_mem);
milvus::server::GetSystemMemInfo(total_mem, free_mem);
ASSERT_TRUE(config.SetCacheConfigInsertBufferSize("1GB").ok());
int64_t cache_cpu_cache_size = total_mem / 2;
float cache_cpu_cache_threshold = 0.7;
......@@ -271,7 +270,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
{
int64_t total_mem = 0, free_mem = 0;
milvus::server::CommonUtil::GetSystemMemInfo(total_mem, free_mem);
milvus::server::GetSystemMemInfo(total_mem, free_mem);
ASSERT_TRUE(config.SetCacheConfigInsertBufferSize("1GB").ok());
int64_t cache_cpu_cache_size = total_mem - 1073741824 - 1; // total_size - 1GB - 1
ASSERT_TRUE(config.SetCacheConfigCpuCacheCapacity(std::to_string(cache_cpu_cache_size)).ok());
......@@ -935,10 +934,10 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_FAIL_TEST) {
ASSERT_FALSE(s.ok());
fiu_disable("Config.GetGpuResourceConfigCacheCapacity.diable_gpu_resource");
fiu_enable("ValidationUtil.GetGpuMemory.return_error", 1, NULL, 0);
fiu_enable("config.GetGpuMemory.return_error", 1, NULL, 0);
s = config.GetGpuResourceConfigCacheCapacity(value);
ASSERT_FALSE(s.ok());
fiu_disable("ValidationUtil.GetGpuMemory.return_error");
fiu_disable("config.GetGpuMemory.return_error");
// fiu_enable("check_config_insert_buffer_size_fail", 1, NULL, 0);
// s = config.GetCacheConfigCpuCacheCapacity(value);
......
core/unittest/server/test_util.cpp
浏览文件 @ a9df8376
此差异已折叠。
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或