Skip to content

M
milvus

milvus / milvus
11 个月 前同步成功

通知 261
Star 22476
Fork 2472
M
milvus

体验新版 GitCode，发现更多精彩内容 >>

未验证 提交 b09e56ff 编写于 作者: C Cai Yudong 提交者： GitHub
浏览文件
操作

Remove unused directory (#16419) 

Signed-off-by: Nyudong.cai <yudong.cai@zilliz.com>
上级 bb7a0766
展开全部 隐藏空白更改
内联 并排
Showing with 0 addition and 3402 deletion
internal/core/src/value/CMakeLists.txt 已删除 100644 → 0
浏览文件 @ bb7a0766
#-------------------------------------------------------------------------------
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
#-------------------------------------------------------------------------------
# library
set( VALUE_SRCS config/ConfigInit.cpp
config/ConfigMgr.h
config/ConfigMgr.cpp
config/ServerConfig.h
config/ServerConfig.cpp
status/StatusInit.cpp
status/StatusMgr.h
status/StatusMgr.cpp
status/ServerStatus.h
status/ServerStatus.cpp
Value.h
ValueMgr.h
ValueMgr.cpp
ValueType.h
ValueType.cpp
)
set( VALUE_LIBS yaml-cpp)
create_library(
TARGET value
SRCS ${VALUE_SRCS}
LIBS ${VALUE_LIBS}
)
if ( BUILD_UNIT_TEST )
create_library(
TARGET value-test
SRCS ${VALUE_SRCS}
LIBS ${VALUE_LIBS})
set(GTEST_LIBS gtest gtest_main gmock gmock_main)
create_executable(
TARGET ConfigMgrTest
SRCS config/ConfigMgrTest
LIBS value-test ${GTEST_LIBS}
DEFS ""
)
add_test ( NAME ConfigMgrTest
COMMAND $<TARGET_FILE:ConfigMgrTest>
)
create_executable(
TARGET ServerConfigTest
SRCS config/ServerConfigTest
LIBS value-fiu ${GTEST_LIBS}
DEFS ""
)
add_test ( NAME ServerConfigTest
COMMAND $<TARGET_FILE:ServerConfigTest>
)
create_executable(
TARGET ValueTypeTest
SRCS ValueTypeTest1 ValueTypeTest2
LIBS value-test ${GTEST_LIBS}
DEFS ""
)
add_test ( NAME ValueTypeTest
COMMAND $<TARGET_FILE:ValueTypeTest>
)
endif()
internal/core/src/value/Value.h 已删除 100644 → 0
浏览文件 @ bb7a0766
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <mutex>
namespace milvus {
template <typename T>
class Value {
public:
const T&
operator()() {
std::lock_guard<std::mutex> lock(mutex_);
return value_;
}
Value&
operator=(T value) {
std::lock_guard<std::mutex> lock(mutex_);
value_ = value;
return *this;
}
private:
std::mutex mutex_;
T value_;
};
} // namespace milvus
internal/core/src/value/ValueMgr.cpp 已删除 100644 → 0
浏览文件 @ bb7a0766
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "value/ValueMgr.h"
#include "utils/Json.h"
namespace milvus {
void
BaseValueMgr::Attach(const std::string& name, ValueObserver* observer) {
std::lock_guard<std::mutex> lock(observer_mutex_);
observers_[name].push_back(observer);
}
void
BaseValueMgr::Detach(const std::string& name, ValueObserver* observer) {
std::lock_guard<std::mutex> lock(observer_mutex_);
if (observers_.find(name) == observers_.end()) {
return;
}
auto& ob_list = observers_[name];
ob_list.remove(observer);
}
void
BaseValueMgr::Notify(const std::string& name) {
std::lock_guard<std::mutex> lock(observer_mutex_);
if (observers_.find(name) == observers_.end()) {
return;
}
auto& ob_list = observers_[name];
for (auto& ob : ob_list) {
ob->ValueUpdate(name);
}
}
void
ValueMgr::Init() {
for (auto& kv : value_list_) {
kv.second->Init();
}
}
std::string
ValueMgr::Dump() const {
std::stringstream ss;
for (auto& kv : value_list_) {
auto& config = kv.second;
ss << config->name_ << ": " << config->Get() << std::endl;
}
return ss.str();
}
std::string
ValueMgr::JsonDump() const {
json config_list;
for (auto& kv : value_list_) {
auto& config = kv.second;
config_list[config->name_] = config->Get();
}
return config_list.dump();
}
} // namespace milvus
internal/core/src/value/ValueMgr.h 已删除 100644 → 0
浏览文件 @ bb7a0766
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <list>
#include <mutex>
#include <set>
#include <string>
#include <unordered_map>
#include <utility>
#include "value/ValueType.h"
namespace milvus {
class ValueObserver {
public:
virtual ~ValueObserver() = default;
virtual void
ValueUpdate(const std::string& name) = 0;
};
class BaseValueMgr {
protected:
BaseValueMgr() = default;
public:
// Shared pointer should not be used here
void
Attach(const std::string& name, ValueObserver* observer);
void
Detach(const std::string& name, ValueObserver* observer);
protected:
virtual void
Notify(const std::string& name);
private:
std::unordered_map<std::string, std::list<ValueObserver*>> observers_;
std::mutex observer_mutex_;
};
class ValueMgr : public BaseValueMgr {
public:
explicit ValueMgr(std::unordered_map<std::string, BaseValuePtr> init_list) : value_list_(std::move(init_list)) {
}
ValueMgr(const ValueMgr&) = delete;
ValueMgr&
operator=(const ValueMgr&) = delete;
ValueMgr(ValueMgr&&) = delete;
ValueMgr&
operator=(ValueMgr&&) = delete;
public:
void
Init();
virtual void
Set(const std::string& name, const std::string& value, bool update) = 0;
virtual std::string
Get(const std::string& name) const = 0;
std::string
Dump() const;
std::string
JsonDump() const;
protected:
const std::unordered_map<std::string, BaseValuePtr> value_list_;
};
} // namespace milvus
internal/core/src/value/ValueType.cpp 已删除 100644 → 0
浏览文件 @ bb7a0766
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "value/ValueType.h"
#include <strings.h>
#include <algorithm>
#include <cassert>
#include <functional>
#include <map>
#include <regex>
#include <sstream>
#include <string>
namespace {
std::unordered_map<std::string, int64_t> BYTE_UNITS = {
{"b", 1},
{"k", 1024},
{"m", 1024 * 1024},
{"g", 1024 * 1024 * 1024},
};
std::map<std::string, int64_t> TIME_UNITS = {
// {"seconds", 1ll},
// {"minutes", 1ll * 60},
{"hours", 1ll * 60 * 60},
{"days", 1ll * 60 * 60 * 24},
};
bool
is_integer(const std::string& s) {
if (not s.empty() && (std::isdigit(s[0]) || s[0] == '-')) {
auto ss = s.substr(1);
return std::find_if(ss.begin(), ss.end(), [](unsigned char c) { return !std::isdigit(c); }) == ss.end();
}
return false;
}
bool
is_number(const std::string& s) {
return !s.empty() && std::find_if(s.begin(), s.end(), [](unsigned char c) { return !std::isdigit(c); }) == s.end();
}
bool
is_alpha(const std::string& s) {
return !s.empty() && std::find_if(s.begin(), s.end(), [](unsigned char c) { return !std::isalpha(c); }) == s.end();
}
template <typename T>
bool
boundary_check(T val, T lower_bound, T upper_bound) {
return lower_bound <= val && val <= upper_bound;
}
bool
parse_bool(const std::string& str, std::string& err) {
if (!strcasecmp(str.c_str(), "true")) {
return true;
} else if (!strcasecmp(str.c_str(), "false")) {
return false;
} else {
err = "The specified value must be true or false";
return false;
}
}
std::string
str_tolower(std::string s) {
std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return std::tolower(c); });
return s;
}
int64_t
parse_bytes(const std::string& str, std::string& err) {
try {
if (str.find_first_of('-') != std::string::npos) {
std::stringstream ss;
ss << "The specified value for memory (" << str << ") should be a positive integer.";
err = ss.str();
return 0;
}
std::string s = str;
if (is_number(s)) {
return std::stoll(s);
}
if (s.length() == 0) {
return 0;
}
auto last_two = s.substr(s.length() - 2, 2);
auto last_one = s.substr(s.length() - 1);
if (is_alpha(last_two) && is_alpha(last_one)) {
if (last_one == "b" or last_one == "B") {
s = s.substr(0, s.length() - 1);
}
}
auto& units = BYTE_UNITS;
auto suffix = str_tolower(s.substr(s.length() - 1));
std::string digits_part;
if (is_number(suffix)) {
digits_part = s;
suffix = 'b';
} else {
digits_part = s.substr(0, s.length() - 1);
}
if (is_number(digits_part) && (units.find(suffix) != units.end() || is_number(suffix))) {
auto digits = std::stoll(digits_part);
return digits * units[suffix];
} else {
std::stringstream ss;
ss << "The specified value for memory (" << str << ") should specify the units."
<< " The postfix should be one of the `b` `k` `m` `g` characters.";
err = ss.str();
}
} catch (...) {
err = "Unknown error happened on parse bytes.";
}
return 0;
}
int64_t
parse_time(const std::string& str, std::string& err) {
try {
const std::regex regex(R"(\s*([0-9]+)\s*(seconds|minutes|hours|days)\s*)");
std::smatch base_match;
auto& units = TIME_UNITS;
if (std::regex_match(str, base_match, regex) && base_match.size() == 3 &&
units.find(base_match[2].str()) != units.end()) {
return stoll(base_match[1].str()) * units[base_match[2].str()];
} else {
std::stringstream ss;
ss << "The specified value for time (" << str << ") should specify the units."
<< " The postfix should be one of the ";
for (auto& pair : units) {
ss << "`" << pair.first << "` ";
}
ss << "words.";
err = ss.str();
}
} catch (...) {
err = "Unknown error happened on parse time.";
}
return 0;
}
} // namespace
// Use (void) to silent unused warnings.
#define assertm(exp, msg) assert(((void)msg, exp))
namespace milvus {
std::vector<std::string>
OptionValue(const valueEnum& ce) {
std::vector<std::string> ret;
for (auto& e : ce) {
ret.emplace_back(e.first);
}
return ret;
}
BaseValue::BaseValue(const char* name, const char* alias, bool modifiable)
: name_(name), alias_(alias), modifiable_(modifiable) {
}
void
BaseValue::Init() {
assertm(not inited_, "already initialized");
inited_ = true;
}
BoolValue::BoolValue(const char* name,
const char* alias,
bool modifiable,
Value<bool>& config,
bool default_value,
std::function<bool(bool val, std::string& err)> is_valid_fn)
: BaseValue(name, alias, modifiable),
config_(config),
default_value_(default_value),
is_valid_fn_(std::move(is_valid_fn)) {
}
void
BoolValue::Init() {
BaseValue::Init();
config_ = default_value_;
}
void
BoolValue::Set(const std::string& val, bool update) {
assertm(inited_, "uninitialized");
try {
/* Check modifiable */
if (update and not modifiable_) {
throw Immutable(name_, val);
}
/* Parse from string */
std::string err;
bool value = parse_bool(val, err);
if (not err.empty()) {
throw Invalid(name_, val, err);
}
/* Validate */
if (is_valid_fn_ && not is_valid_fn_(value, err)) {
throw Invalid(name_, val, err);
}
/* Set value */
config_ = value;
} catch (ValueError& e) {
throw;
} catch (...) {
throw Unexpected(name_, val);
}
}
std::string
BoolValue::Get() {
assertm(inited_, "uninitialized");
return config_() ? "true" : "false";
}
StringValue::StringValue(const char* name,
const char* alias,
bool modifiable,
Value<std::string>& config,
const char* default_value,
std::function<bool(const std::string& val, std::string& err)> is_valid_fn)
: BaseValue(name, alias, modifiable),
config_(config),
default_value_(default_value),
is_valid_fn_(std::move(is_valid_fn)) {
}
void
StringValue::Init() {
BaseValue::Init();
config_ = default_value_;
}
void
StringValue::Set(const std::string& val, bool update) {
assertm(inited_, "uninitialized");
try {
/* Check modifiable */
if (update and not modifiable_) {
throw Immutable(name_, val);
}
/* Validate */
std::string err;
if (is_valid_fn_ && not is_valid_fn_(val, err)) {
throw Invalid(name_, val, err);
}
/* Set value */
config_ = val;
} catch (ValueError& e) {
throw;
} catch (...) {
throw Unexpected(name_, val);
}
}
std::string
StringValue::Get() {
assertm(inited_, "uninitialized");
return config_();
}
EnumValue::EnumValue(const char* name,
const char* alias,
bool modifiable,
valueEnum* enumd,
Value<int64_t>& config,
int64_t default_value,
std::function<bool(int64_t val, std::string& err)> is_valid_fn)
: BaseValue(name, alias, modifiable),
config_(config),
enum_value_(enumd),
default_value_(default_value),
is_valid_fn_(std::move(is_valid_fn)) {
}
void
EnumValue::Init() {
BaseValue::Init();
assert(enum_value_ != nullptr);
assertm(not enum_value_->empty(), "enum value empty");
config_ = default_value_;
}
void
EnumValue::Set(const std::string& val, bool update) {
assertm(inited_, "uninitialized");
try {
/* Check modifiable */
if (update and not modifiable_) {
throw Immutable(name_, val);
}
/* Check if value exist */
if (enum_value_->find(val) == enum_value_->end()) {
auto option_values = OptionValue(*enum_value_);
throw EnumValueNotFound(name_, val, std::move(option_values));
}
int64_t value = enum_value_->at(val);
/* Validate */
std::string err;
if (is_valid_fn_ && not is_valid_fn_(value, err)) {
throw Invalid(name_, val, err);
}
/* Set value */
config_ = value;
} catch (ValueError& e) {
throw;
} catch (...) {
throw Unexpected(name_, val);
}
}
std::string
EnumValue::Get() {
assertm(inited_, "uninitialized");
auto val = config_();
for (auto& it : *enum_value_) {
if (val == it.second) {
return it.first;
}
}
return "unknown";
}
IntegerValue::IntegerValue(const char* name,
const char* alias,
bool modifiable,
int64_t lower_bound,
int64_t upper_bound,
Value<int64_t>& config,
int64_t default_value,
std::function<bool(int64_t val, std::string& err)> is_valid_fn)
: BaseValue(name, alias, modifiable),
config_(config),
lower_bound_(lower_bound),
upper_bound_(upper_bound),
default_value_(default_value),
is_valid_fn_(std::move(is_valid_fn)) {
}
void
IntegerValue::Init() {
BaseValue::Init();
config_ = default_value_;
}
void
IntegerValue::Set(const std::string& val, bool update) {
assertm(inited_, "uninitialized");
try {
/* Check modifiable */
if (update and not modifiable_) {
throw Immutable(name_, val);
}
/* Check if it is an integer */
if (not is_integer(val)) {
throw Invalid(name_, val, "Not an integer.");
}
/* Parse from string */
int64_t value = std::stoll(val);
/* Boundary check */
if (not boundary_check<int64_t>(value, lower_bound_, upper_bound_)) {
throw OutOfRange<int64_t>(name_, val, lower_bound_, upper_bound_);
}
/* Validate */
std::string err;
if (is_valid_fn_ && not is_valid_fn_(value, err)) {
throw Invalid(name_, val, err);
}
/* Set value */
config_ = value;
} catch (ValueError& e) {
throw;
} catch (...) {
throw Unexpected(name_, val);
}
}
std::string
IntegerValue::Get() {
assertm(inited_, "uninitialized");
return std::to_string(config_());
}
FloatingValue::FloatingValue(const char* name,
const char* alias,
bool modifiable,
double lower_bound,
double upper_bound,
Value<double>& config,
double default_value,
std::function<bool(double val, std::string& err)> is_valid_fn)
: BaseValue(name, alias, modifiable),
config_(config),
lower_bound_(lower_bound),
upper_bound_(upper_bound),
default_value_(default_value),
is_valid_fn_(std::move(is_valid_fn)) {
}
void
FloatingValue::Init() {
BaseValue::Init();
config_ = default_value_;
}
void
FloatingValue::Set(const std::string& val, bool update) {
assertm(inited_, "uninitialized");
try {
/* Check modifiable */
if (update and not modifiable_) {
throw Immutable(name_, val);
}
/* Parse from string */
double value = std::stod(val);
/* Boundary check */
if (not boundary_check<double>(value, lower_bound_, upper_bound_)) {
throw OutOfRange<double>(name_, val, lower_bound_, upper_bound_);
}
/* Validate */
std::string err;
if (is_valid_fn_ && not is_valid_fn_(value, err)) {
throw Invalid(name_, val, err);
}
/* Set value */
config_ = value;
} catch (ValueError& e) {
throw;
} catch (...) {
throw Unexpected(name_, val);
}
}
std::string
FloatingValue::Get() {
assertm(inited_, "uninitialized");
return std::to_string(config_());
}
SizeValue::SizeValue(const char* name,
const char* alias,
bool modifiable,
int64_t lower_bound,
int64_t upper_bound,
Value<int64_t>& config,
int64_t default_value,
std::function<bool(int64_t val, std::string& err)> is_valid_fn)
: BaseValue(name, alias, modifiable),
config_(config),
lower_bound_(lower_bound),
upper_bound_(upper_bound),
default_value_(default_value),
is_valid_fn_(std::move(is_valid_fn)) {
}
void
SizeValue::Init() {
BaseValue::Init();
config_ = default_value_;
}
void
SizeValue::Set(const std::string& val, bool update) {
assertm(inited_, "uninitialized");
try {
/* Check modifiable */
if (update and not modifiable_) {
throw Immutable(name_, val);
}
/* Parse from string */
std::string err;
int64_t value = parse_bytes(val, err);
if (not err.empty()) {
throw Invalid(name_, val, err);
}
/* Boundary check */
if (not boundary_check<int64_t>(value, lower_bound_, upper_bound_)) {
throw OutOfRange<int64_t>(name_, val, lower_bound_, upper_bound_);
}
/* Validate */
if (is_valid_fn_ && not is_valid_fn_(value, err)) {
throw Invalid(name_, val, err);
}
/* Set value */
config_ = value;
} catch (ValueError& e) {
throw;
} catch (...) {
throw Unexpected(name_, val);
}
}
std::string
SizeValue::Get() {
assertm(inited_, "uninitialized");
auto val = config_();
const int64_t gb = 1024ll * 1024 * 1024;
const int64_t mb = 1024ll * 1024;
const int64_t kb = 1024ll;
if (val % gb == 0) {
return std::to_string(val / gb) + "GB";
} else if (val % mb == 0) {
return std::to_string(val / mb) + "MB";
} else if (val % kb == 0) {
return std::to_string(val / kb) + "KB";
} else {
return std::to_string(val);
}
}
TimeValue::TimeValue(const char* name,
const char* alias,
bool modifiable,
int64_t lower_bound,
int64_t upper_bound,
Value<int64_t>& config,
int64_t default_value,
std::function<bool(int64_t val, std::string& err)> is_valid_fn)
: BaseValue(name, alias, modifiable),
config_(config),
lower_bound_(lower_bound),
upper_bound_(upper_bound),
default_value_(default_value),
is_valid_fn_(std::move(is_valid_fn)) {
}
void
TimeValue::Init() {
BaseValue::Init();
config_ = default_value_;
}
void
TimeValue::Set(const std::string& val, bool update) {
assertm(inited_, "uninitialized");
try {
/* Check modifiable */
if (update and not modifiable_) {
throw Immutable(name_, val);
}
/* Parse from string */
std::string err;
int64_t value = parse_time(val, err);
if (not err.empty()) {
throw Invalid(name_, val, err);
}
/* Boundary check */
if (not boundary_check<int64_t>(value, lower_bound_, upper_bound_)) {
throw OutOfRange<int64_t>(name_, val, lower_bound_, upper_bound_);
}
/* Validate */
if (is_valid_fn_ && not is_valid_fn_(value, err)) {
throw Invalid(name_, val, err);
}
/* Set value */
config_ = value;
} catch (ValueError& e) {
throw;
} catch (...) {
throw Unexpected(name_, val);
}
}
std::string
TimeValue::Get() {
assertm(inited_, "uninitialized");
auto val = config_();
const int64_t second = 1ll;
const int64_t minute = second * 60;
const int64_t hour = minute * 60;
const int64_t day = hour * 24;
if (val % day == 0) {
return std::to_string(val / day) + "days";
} else if (val % hour == 0) {
return std::to_string(val / hour) + "hours";
} else if (val % minute == 0) {
return std::to_string(val / minute) + "minutes";
} else {
return std::to_string(val) + "seconds";
}
}
} // namespace milvus
internal/core/src/value/ValueType.h 已删除 100644 → 0
浏览文件 @ bb7a0766
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <exception>
#include <functional>
#include <memory>
#include <sstream>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
#include "value/Value.h"
namespace milvus {
using valueEnum = const std::unordered_map<std::string, int64_t>;
std::vector<std::string>
OptionValue(const valueEnum& ce);
struct ValueError : public std::exception {
explicit ValueError(const std::string& name, const std::string& value) : name_(name), value_(value) {
}
virtual std::string
message() = 0;
protected:
const std::string name_;
const std::string value_;
};
struct Immutable : public ValueError {
explicit Immutable(const std::string& name, const std::string& value) : ValueError(name, value) {
}
std::string
message() override {
return "Config " + name_ + " is immutable.";
}
};
struct EnumValueNotFound : public ValueError {
EnumValueNotFound(const std::string& name, const std::string& value, std::vector<std::string> option_values)
: ValueError(name, value), option_values_(std::move(option_values)) {
}
std::string
message() override {
std::stringstream ss;
ss << "Config " << name_ << "(" << value_ << ") must be one of following: ";
for (size_t i = 0; i < option_values_.size() - 1; ++i) {
ss << option_values_[i] << ", ";
}
return ss.str();
}
private:
std::vector<std::string> option_values_;
};
struct Invalid : public ValueError {
Invalid(const std::string& name, const std::string& value, const std::string& reason)
: ValueError(name, value), reason_(reason) {
}
std::string
message() override {
return value_ + " is invalid for config " + name_ + ": " + reason_;
}
private:
const std::string reason_;
};
template <typename T>
struct OutOfRange : public ValueError {
OutOfRange(const std::string& name, const std::string& value, T lower_bound, T upper_bound)
: ValueError(name, value), lower_bound_(lower_bound), upper_bound_(upper_bound) {
}
std::string
message() override {
return "Config " + name_ + "(" + value_ + ") must in range [" + std::to_string(lower_bound_) + ", " +
std::to_string(upper_bound_) + "].";
}
private:
T lower_bound_;
T upper_bound_;
};
struct Unexpected : public ValueError {
Unexpected(const std::string& name, const std::string& value) : ValueError(name, value) {
}
std::string
message() override {
return "An unknown error occurred while setting " + name_ + " as " + value_;
}
};
class BaseValue {
public:
BaseValue(const char* name, const char* alias, bool modifiable);
virtual ~BaseValue() = default;
public:
bool inited_ = false;
const char* name_;
const char* alias_;
const bool modifiable_;
public:
virtual void
Init();
virtual void
Set(const std::string& value, bool update) = 0;
virtual std::string
Get() = 0;
};
using BaseValuePtr = std::shared_ptr<BaseValue>;
class BoolValue : public BaseValue {
public:
BoolValue(const char* name,
const char* alias,
bool modifiable,
Value<bool>& config,
bool default_value,
std::function<bool(bool val, std::string& err)> is_valid_fn = nullptr);
private:
Value<bool>& config_;
const bool default_value_;
std::function<bool(bool val, std::string& err)> is_valid_fn_;
public:
void
Init() override;
void
Set(const std::string& value, bool update) override;
std::string
Get() override;
};
class StringValue : public BaseValue {
public:
StringValue(const char* name,
const char* alias,
bool modifiable,
Value<std::string>& config,
const char* default_value,
std::function<bool(const std::string& val, std::string& err)> is_valid_fn = nullptr);
private:
Value<std::string>& config_;
const char* default_value_;
std::function<bool(const std::string& val, std::string& err)> is_valid_fn_;
public:
void
Init() override;
void
Set(const std::string& value, bool update) override;
std::string
Get() override;
};
class EnumValue : public BaseValue {
public:
EnumValue(const char* name,
const char* alias,
bool modifiable,
valueEnum* enumd,
Value<int64_t>& config,
int64_t default_value,
std::function<bool(int64_t val, std::string& err)> is_valid_fn = nullptr);
private:
Value<int64_t>& config_;
valueEnum* enum_value_;
const int64_t default_value_;
std::function<bool(int64_t val, std::string& err)> is_valid_fn_;
public:
void
Init() override;
void
Set(const std::string& value, bool update) override;
std::string
Get() override;
};
class IntegerValue : public BaseValue {
public:
IntegerValue(const char* name,
const char* alias,
bool modifiable,
int64_t lower_bound,
int64_t upper_bound,
Value<int64_t>& config,
int64_t default_value,
std::function<bool(int64_t val, std::string& err)> is_valid_fn = nullptr);
private:
Value<int64_t>& config_;
int64_t lower_bound_;
int64_t upper_bound_;
const int64_t default_value_;
std::function<bool(int64_t val, std::string& err)> is_valid_fn_;
public:
void
Init() override;
void
Set(const std::string& value, bool update) override;
std::string
Get() override;
};
class FloatingValue : public BaseValue {
public:
FloatingValue(const char* name,
const char* alias,
bool modifiable,
double lower_bound,
double upper_bound,
Value<double>& config,
double default_value,
std::function<bool(double val, std::string& err)> is_valid_fn = nullptr);
private:
Value<double>& config_;
double lower_bound_;
double upper_bound_;
const double default_value_;
std::function<bool(double val, std::string& err)> is_valid_fn_;
public:
void
Init() override;
void
Set(const std::string& value, bool update) override;
std::string
Get() override;
};
class SizeValue : public BaseValue {
public:
SizeValue(const char* name,
const char* alias,
bool modifiable,
int64_t lower_bound,
int64_t upper_bound,
Value<int64_t>& config,
int64_t default_value,
std::function<bool(int64_t val, std::string& err)> is_valid_fn = nullptr);
private:
Value<int64_t>& config_;
int64_t lower_bound_;
int64_t upper_bound_;
const int64_t default_value_;
std::function<bool(int64_t val, std::string& err)> is_valid_fn_;
public:
void
Init() override;
void
Set(const std::string& value, bool update) override;
std::string
Get() override;
};
class TimeValue : public BaseValue {
public:
TimeValue(const char* name,
const char* alias,
bool modifiable,
int64_t lower_bound,
int64_t upper_bound,
Value<int64_t>& config,
int64_t default_value,
std::function<bool(int64_t val, std::string& err)> is_valid_fn = nullptr);
private:
Value<int64_t>& config_;
int64_t lower_bound_;
int64_t upper_bound_;
const int64_t default_value_;
std::function<bool(int64_t val, std::string& err)> is_valid_fn_;
public:
void
Init() override;
void
Set(const std::string& value, bool update) override;
std::string
Get() override;
};
/* create config with {is_valid} function */
#define CreateBoolValue(name, modifiable, config_addr, default, is_valid) \
std::make_shared<BoolValue>(name, nullptr, modifiable, config_addr, (default), is_valid)
#define CreateStringValue(name, modifiable, config_addr, default, is_valid) \
std::make_shared<StringValue>(name, nullptr, modifiable, config_addr, (default), is_valid)
#define CreateEnumValue(name, modifiable, enumd, config_addr, default, is_valid) \
std::make_shared<EnumValue>(name, nullptr, modifiable, enumd, config_addr, (default), is_valid)
#define CreateIntegerValue(name, modifiable, lower_bound, upper_bound, config_addr, default, is_valid) \
std::make_shared<IntegerValue>(name, nullptr, modifiable, lower_bound, upper_bound, config_addr, (default), \
is_valid)
#define CreateFloatingValue(name, modifiable, lower_bound, upper_bound, config_addr, default, is_valid) \
std::make_shared<FloatingValue>(name, nullptr, modifiable, lower_bound, upper_bound, config_addr, (default), \
is_valid)
#define CreateSizeValue(name, modifiable, lower_bound, upper_bound, config_addr, default, is_valid) \
std::make_shared<SizeValue>(name, nullptr, modifiable, lower_bound, upper_bound, config_addr, (default), is_valid)
#define CreateTimeValue(name, modifiable, lower_bound, upper_bound, config_addr, default, is_valid) \
std::make_shared<TimeValue>(name, nullptr, modifiable, lower_bound, upper_bound, config_addr, (default), is_valid)
} // namespace milvus
internal/core/src/value/ValueTypeTest1.cpp 已删除 100644 → 0
浏览文件 @ bb7a0766
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include <cstring>
#include <functional>
#include "gtest/gtest.h"
#include "value/ValueType.h"
namespace milvus {
#define _MODIFIABLE (true)
#define _IMMUTABLE (false)
template <typename T>
class Utils {
public:
bool
validate_fn(const T& value, std::string& err) {
validate_value = value;
return true;
}
protected:
T validate_value;
};
/* ValidBoolValueTest */
class ValidBoolValueTest : public testing::Test, public Utils<bool> {
protected:
};
TEST_F(ValidBoolValueTest, init_load_update_get_test) {
auto validate = std::bind(&ValidBoolValueTest::validate_fn, this, std::placeholders::_1, std::placeholders::_2);
Value<bool> bool_value;
auto bool_config = CreateBoolValue("b", _MODIFIABLE, bool_value, false, validate);
ASSERT_EQ(bool_config->modifiable_, true);
bool_config->Init();
ASSERT_EQ(bool_value(), false);
ASSERT_EQ(bool_config->Get(), "false");
{
// now `bool_value` is `false`, calling Set(update=false) to set it to `true`
validate_value = false;
EXPECT_NO_THROW(bool_config->Set("true", false));
EXPECT_EQ(bool_value(), true);
EXPECT_EQ(bool_config->Get(), "true");
// expect change
EXPECT_EQ(validate_value, true);
}
{
// now `bool_value` is `true`, calling Set(update=true) to set it to `false`
validate_value = true;
EXPECT_NO_THROW(bool_config->Set("false", true));
EXPECT_EQ(bool_value(), false);
EXPECT_EQ(bool_config->Get(), "false");
// expect change
EXPECT_EQ(validate_value, false);
}
}
/* ValidStringValueTest */
class ValidStringValueTest : public testing::Test, public Utils<std::string> {
protected:
};
TEST_F(ValidStringValueTest, init_load_update_get_test) {
auto validate = std::bind(&ValidStringValueTest::validate_fn, this, std::placeholders::_1, std::placeholders::_2);
Value<std::string> string_value;
auto string_config = CreateStringValue("s", _MODIFIABLE, string_value, "Magic", validate);
ASSERT_EQ(string_config->modifiable_, true);
string_config->Init();
ASSERT_EQ(string_value(), "Magic");
ASSERT_EQ(string_config->Get(), "Magic");
{
// now `string_value` is `Magic`, calling Set(update=false) to set it to `cigaM`
validate_value = "";
EXPECT_NO_THROW(string_config->Set("cigaM", false));
EXPECT_EQ(string_value(), "cigaM");
EXPECT_EQ(string_config->Get(), "cigaM");
// expect change
EXPECT_EQ(validate_value, "cigaM");
}
{
// now `string_value` is `cigaM`, calling Set(update=true) to set it to `Check`
validate_value = "";
EXPECT_NO_THROW(string_config->Set("Check", true));
EXPECT_EQ(string_value(), "Check");
EXPECT_EQ(string_config->Get(), "Check");
// expect change
EXPECT_EQ(validate_value, "Check");
}
}
/* ValidIntegerValueTest */
class ValidIntegerValueTest : public testing::Test, public Utils<int64_t> {
protected:
};
TEST_F(ValidIntegerValueTest, init_load_update_get_test) {
auto validate = std::bind(&ValidIntegerValueTest::validate_fn, this, std::placeholders::_1, std::placeholders::_2);
Value<int64_t> integer_value;
auto integer_config = CreateIntegerValue("i", _MODIFIABLE, -100, 100, integer_value, 42, validate);
ASSERT_EQ(integer_config->modifiable_, true);
integer_config->Init();
ASSERT_EQ(integer_value(), 42);
ASSERT_EQ(integer_config->Get(), "42");
{
// now `integer_value` is `42`, calling Set(update=false) to set it to `24`
validate_value = 0;
EXPECT_NO_THROW(integer_config->Set("24", false));
EXPECT_EQ(integer_value(), 24);
EXPECT_EQ(integer_config->Get(), "24");
// expect change
EXPECT_EQ(validate_value, 24);
}
{
// now `integer_value` is `24`, calling Set(update=true) to set it to `36`
validate_value = 0;
EXPECT_NO_THROW(integer_config->Set("36", true));
EXPECT_EQ(integer_value(), 36);
EXPECT_EQ(integer_config->Get(), "36");
// expect change
EXPECT_EQ(validate_value, 36);
}
}
/* ValidFloatingValueTest */
class ValidFloatingValueTest : public testing::Test, public Utils<double> {
protected:
};
TEST_F(ValidFloatingValueTest, init_load_update_get_test) {
auto validate = std::bind(&ValidFloatingValueTest::validate_fn, this, std::placeholders::_1, std::placeholders::_2);
Value<double> floating_value;
auto floating_config = CreateFloatingValue("f", _MODIFIABLE, -10.0, 10.0, floating_value, 3.14, validate);
ASSERT_EQ(floating_config->modifiable_, true);
floating_config->Init();
ASSERT_FLOAT_EQ(floating_value(), 3.14);
ASSERT_FLOAT_EQ(std::stof(floating_config->Get()), 3.14);
{
// now `floating_value` is `3.14`, calling Set(update=false) to set it to `6.22`
validate_value = 0.0;
EXPECT_NO_THROW(floating_config->Set("6.22", false));
ASSERT_FLOAT_EQ(floating_value(), 6.22);
ASSERT_FLOAT_EQ(std::stof(floating_config->Get()), 6.22);
// expect change
ASSERT_FLOAT_EQ(validate_value, 6.22);
}
{
// now `integer_value` is `6.22`, calling Set(update=true) to set it to `-3.14`
validate_value = 0.0;
EXPECT_NO_THROW(floating_config->Set("-3.14", true));
ASSERT_FLOAT_EQ(floating_value(), -3.14);
ASSERT_FLOAT_EQ(std::stof(floating_config->Get()), -3.14);
// expect change
ASSERT_FLOAT_EQ(validate_value, -3.14);
}
}
/* ValidEnumValueTest */
class ValidEnumValueTest : public testing::Test, public Utils<int64_t> {
protected:
};
// template <>
// int64_t Utils<int64_t>::validate_value = 0;
// template <>
// int64_t Utils<int64_t>::new_value = 0;
// template <>
// int64_t Utils<int64_t>::prev_value = 0;
TEST_F(ValidEnumValueTest, init_load_update_get_test) {
auto validate = std::bind(&ValidEnumValueTest::validate_fn, this, std::placeholders::_1, std::placeholders::_2);
valueEnum testEnum{
{"a", 1},
{"b", 2},
{"c", 3},
};
Value<int64_t> enum_value;
auto enum_config = CreateEnumValue("e", _MODIFIABLE, &testEnum, enum_value, 1, validate);
ASSERT_EQ(enum_config->modifiable_, true);
enum_config->Init();
ASSERT_EQ(enum_value(), 1);
ASSERT_EQ(enum_config->Get(), "a");
{
// now `enum_value` is `a`, calling Set(update=false) to set it to `b`
validate_value = 0;
EXPECT_NO_THROW(enum_config->Set("b", false));
ASSERT_EQ(enum_value(), 2);
ASSERT_EQ(enum_config->Get(), "b");
// expect change
ASSERT_EQ(validate_value, 2);
}
{
// now `enum_value` is `b`, calling Set(update=true) to set it to `c`
validate_value = 0;
EXPECT_NO_THROW(enum_config->Set("c", true));
ASSERT_EQ(enum_value(), 3);
ASSERT_EQ(enum_config->Get(), "c");
// expect change
ASSERT_EQ(validate_value, 3);
}
}
/* ValidSizeValueTest */
class ValidSizeValueTest : public testing::Test, public Utils<int64_t> {
protected:
};
// template <>
// int64_t Utils<int64_t>::validate_value = 0;
// template <>
// int64_t Utils<int64_t>::new_value = 0;
// template <>
// int64_t Utils<int64_t>::prev_value = 0;
TEST_F(ValidSizeValueTest, init_load_update_get_test) {
auto validate = std::bind(&ValidSizeValueTest::validate_fn, this, std::placeholders::_1, std::placeholders::_2);
Value<int64_t> size_value;
auto size_config = CreateSizeValue("i", _MODIFIABLE, 0, 1024 * 1024, size_value, 1024, validate);
ASSERT_EQ(size_config->modifiable_, true);
size_config->Init();
ASSERT_EQ(size_value(), 1024);
ASSERT_EQ(size_config->Get(), "1KB");
{
// now `size_value` is `1024`, calling Set(update=false) to set it to `4096`
validate_value = 0;
EXPECT_NO_THROW(size_config->Set("4096", false));
EXPECT_EQ(size_value(), 4096);
EXPECT_EQ(size_config->Get(), "4KB");
// expect change
EXPECT_EQ(validate_value, 4096);
}
{
// now `size_value` is `4096`, calling Set(update=true) to set it to `256kb`
validate_value = 0;
EXPECT_NO_THROW(size_config->Set("256kb", true));
EXPECT_EQ(size_value(), 256 * 1024);
EXPECT_EQ(size_config->Get(), "256KB");
// expect change
EXPECT_EQ(validate_value, 262144);
}
}
class ValidTest : public testing::Test {
protected:
valueEnum family{
{"ipv4", 1},
{"ipv6", 2},
};
struct Server {
Value<bool> running;
Value<std::string> hostname;
Value<int64_t> family;
Value<int64_t> port;
Value<double> uptime;
};
Server server;
protected:
void
SetUp() override {
config_list = {
CreateBoolValue("running", true, server.running, true, nullptr),
CreateStringValue("hostname", true, server.hostname, "Magic", nullptr),
CreateEnumValue("socket_family", false, &family, server.family, 2, nullptr),
CreateIntegerValue("port", true, 1024, 65535, server.port, 19530, nullptr),
CreateFloatingValue("uptime", true, 0, 9999.0, server.uptime, 0, nullptr),
};
}
void
TearDown() override {
}
protected:
void
Init() {
for (auto& config : config_list) {
config->Init();
}
}
void
Load() {
std::unordered_map<std::string, std::string> config_file{
{"running", "false"},
};
for (auto& c : config_file) Set(c.first, c.second, false);
}
void
Set(const std::string& name, const std::string& value, bool update = true) {
for (auto& config : config_list) {
if (std::strcmp(name.c_str(), config->name_) == 0) {
config->Set(value, update);
return;
}
}
throw "Config " + name + " not found.";
}
std::string
Get(const std::string& name) {
for (auto& config : config_list) {
if (std::strcmp(name.c_str(), config->name_) == 0) {
return config->Get();
}
}
throw "Config " + name + " not found.";
}
std::vector<BaseValuePtr> config_list;
};
} // namespace milvus
internal/core/src/value/ValueTypeTest2.cpp 已删除 100644 → 0
浏览文件 @ bb7a0766
此差异已折叠。
internal/core/src/value/config/ConfigInit.cpp 已删除 100644 → 0
浏览文件 @ bb7a0766
此差异已折叠。
internal/core/src/value/config/ConfigMgr.cpp 已删除 100644 → 0
浏览文件 @ bb7a0766
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include <yaml-cpp/yaml.h>
#include <fstream>
#include <regex>
#include <unordered_map>
#include "value/config/ConfigMgr.h"
namespace {
void
Flatten(const YAML::Node& node, std::unordered_map<std::string, std::string>& target, const std::string& prefix) {
for (auto& it : node) {
auto key = prefix.empty() ? it.first.as<std::string>() : prefix + "." + it.first.as<std::string>();
switch (it.second.Type()) {
case YAML::NodeType::Null: {
target[key] = "";
break;
}
case YAML::NodeType::Scalar: {
target[key] = it.second.as<std::string>();
break;
}
case YAML::NodeType::Sequence: {
std::string value;
for (auto& sub : it.second) value += sub.as<std::string>() + ",";
target[key] = value;
break;
}
case YAML::NodeType::Map: {
Flatten(it.second, target, key);
break;
}
case YAML::NodeType::Undefined: {
std::runtime_error("Undefined YAML Node is not supported in Flatten.");
}
default:
break;
}
}
}
}; // namespace
namespace milvus {
extern std::unordered_map<std::string, BaseValuePtr>
InitConfig();
extern const char* config_file_template;
ConfigMgr ConfigMgr::instance;
ConfigMgr::ConfigMgr() : ValueMgr(InitConfig()) {
effective_immediately_ = {
"cache.cache_size",
"gpu.cache_size",
"gpu.gpu_search_threshold",
"storage.auto_flush_interval",
"engine.build_index_threshold",
"engine.search_combine_nq",
"engine.use_blas_threshold",
"engine.omp_thread_num",
};
}
void
ConfigMgr::LoadFile(const std::string& path) {
try {
/* load from milvus.yaml */
auto yaml = YAML::LoadFile(path);
/* make it flattened */
std::unordered_map<std::string, std::string> flattened;
Flatten(yaml, flattened, "");
/* update config */
for (auto& it : flattened) Set(it.first, it.second, false);
config_file_ = path;
} catch (std::exception& ex) {
throw;
} catch (...) {
throw std::runtime_error("Unknown error occurred.");
}
}
void
ConfigMgr::LoadMemory(const std::string& yaml_string) {
try {
auto yaml = YAML::Load(yaml_string);
/* make it flattened */
std::unordered_map<std::string, std::string> flattened;
Flatten(yaml, flattened, "");
/* update config */
for (auto& it : flattened) Set(it.first, it.second, false);
} catch (std::exception& ex) {
throw;
} catch (...) {
throw std::runtime_error("Unknown error occurred.");
}
}
void
ConfigMgr::Set(const std::string& name, const std::string& value, bool update) {
/* Check if existed */
if (config_list_.find(name) == config_list_.end()) {
throw std::runtime_error("Config " + name + " not found.");
}
auto old_value = config_list_.at(name)->Get();
try {
/* Set value, throws ValueError only. */
config_list_.at(name)->Set(value, update);
if (update) {
/* Save file */
Save();
/* Notify who observe this value */
Notify(name);
/* Update flag */
if (effective_immediately_.find(name) == effective_immediately_.end()) {
require_restart_ |= true;
}
}
} catch (ValueError& e) {
/* Convert to std::runtime_error. */
throw std::runtime_error(e.message());
} catch (SaveValueError& e) {
/* Save config failed, rollback and convert to std::runtime_error. */
config_list_.at(name)->Set(old_value, false);
throw std::runtime_error(e.message);
} catch (...) {
/* Unexpected exception, output config and value. */
throw std::runtime_error("Unexpected exception happened when setting " + value + " to " + name + ".");
}
}
std::string
ConfigMgr::Get(const std::string& name) const {
try {
auto& config = config_list_.at(name);
return config->Get();
} catch (std::out_of_range& ex) {
throw std::runtime_error("Config " + name + " not found.");
} catch (...) {
throw std::runtime_error("Unexpected exception happened when getting config " + name + ".");
}
}
void
ConfigMgr::Save() {
if (config_file_.empty()) {
throw SaveValueError("Cannot save config into empty path.");
}
std::string file_content(config_file_template);
for (auto& config_pair : config_list_) {
auto placeholder = "@" + config_pair.first + "@";
file_content = std::regex_replace(file_content, std::regex(placeholder), config_pair.second->Get());
}
std::ofstream config_file(config_file_);
config_file << file_content;
config_file.close();
if (config_file.fail()) {
throw SaveValueError("Cannot save config into file: " + config_file_ + ".");
}
}
} // namespace milvus
internal/core/src/value/config/ConfigMgr.h 已删除 100644 → 0
浏览文件 @ bb7a0766
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <list>
#include <memory>
#include <mutex>
#include <set>
#include <string>
#include <unordered_map>
#include <vector>
#include "value/ValueMgr.h"
#include "value/ValueType.h"
namespace milvus {
class ConfigObserver : public ValueObserver {
public:
void
ValueUpdate(const std::string& name) override {
ConfigUpdate(name);
}
virtual void
ConfigUpdate(const std::string& name) = 0;
};
class ConfigMgr : public ValueMgr {
public:
static ConfigMgr&
GetInstance() {
return instance;
}
private:
static ConfigMgr instance;
public:
bool
RequireRestart() {
return require_restart_;
}
public:
ConfigMgr();
/* throws std::exception only */
void
LoadFile(const std::string& path);
/* for testing */
/* throws std::exception only */
void
LoadMemory(const std::string& yaml_string);
/* throws std::exception only */
void
Set(const std::string& name, const std::string& value, bool update) override;
/* throws std::exception only */
std::string
Get(const std::string& name) const override;
private:
struct SaveValueError : public std::exception {
explicit SaveValueError(const std::string& msg) : message(msg) {
}
const std::string message;
};
void
Save();
private:
const std::unordered_map<std::string, BaseValuePtr>& config_list_ = value_list_;
std::string config_file_;
bool require_restart_ = false;
std::set<std::string> effective_immediately_;
};
} // namespace milvus
internal/core/src/value/config/ServerConfig.cpp 已删除 100644 → 0
浏览文件 @ bb7a0766
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include <iostream>
#include <sstream>
#include <string>
#include <unordered_set>
#include <vector>
#include "value/config/ServerConfig.h"
namespace milvus {
std::mutex config_mutex;
ServerConfig config;
std::vector<std::string>
ParsePreloadCollection(const std::string& str) {
std::stringstream ss(str);
std::vector<std::string> collections;
std::string collection;
while (std::getline(ss, collection, ',')) {
collections.push_back(collection);
}
return collections;
}
std::vector<int64_t>
ParseGPUDevices(const std::string& str) {
std::stringstream ss(str);
std::vector<int64_t> devices;
std::unordered_set<int64_t> device_set;
std::string device;
while (std::getline(ss, device, ',')) {
if (device.length() < 4) {
/* Invalid format string */
return {};
}
device_set.insert(std::stoll(device.substr(3)));
}
devices.reserve(device_set.size());
for (auto dev : device_set) {
devices.push_back(dev);
}
return devices;
}
} // namespace milvus
internal/core/src/value/config/ServerConfig.h 已删除 100644 → 0
浏览文件 @ bb7a0766
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <mutex>
#include <string>
#include <utility>
#include <vector>
#include "value/Value.h"
#include "value/ValueType.h"
namespace milvus {
enum ClusterRole {
RW = 1,
RO,
};
const valueEnum ClusterRoleMap{
{"rw", ClusterRole::RW},
{"ro", ClusterRole::RO},
};
enum SimdType {
AUTO = 1,
SSE4_2,
AVX2,
AVX512,
};
const valueEnum SimdMap{
{"auto", SimdType::AUTO},
{"sse4_2", SimdType::SSE4_2},
{"avx2", SimdType::AVX2},
{"avx512", SimdType::AVX512},
};
enum ClusteringType {
K_MEANS = 1,
K_MEANS_PLUS_PLUS,
};
const valueEnum ClusteringMap{
{"k-means", ClusteringType::K_MEANS},
{"k-means++", ClusteringType::K_MEANS_PLUS_PLUS},
};
struct ServerConfig {
using String = Value<std::string>;
using Bool = Value<bool>;
using Integer = Value<int64_t>;
using Floating = Value<double>;
String version;
struct Cluster {
Bool enable;
Integer role;
String node_id;
} cluster;
struct General {
String timezone;
String meta_uri;
Integer stale_snapshots_count;
Integer stale_snapshots_duration;
} general;
struct Network {
struct Bind {
String address;
Integer port;
} bind;
struct Http {
Bool enable;
Integer port;
} http;
} network;
struct Storage {
String path;
Integer auto_flush_interval;
} storage;
struct Cache {
Integer cache_size;
Floating cpu_cache_threshold;
Integer insert_buffer_size;
Bool cache_insert_data;
String preload_collection;
Integer max_concurrent_insert_request_size;
} cache;
struct Engine {
Integer max_partition_num;
Integer build_index_threshold;
Integer search_combine_nq;
Integer use_blas_threshold;
Integer omp_thread_num;
Integer clustering_type;
Integer simd_type;
Integer statistics_level;
} engine;
struct GPU {
Bool enable;
Integer cache_size;
Floating cache_threshold;
Integer gpu_search_threshold;
String search_devices;
String build_index_devices;
} gpu;
struct Tracing {
String json_config_path;
} tracing;
struct WAL {
Bool enable;
Bool sync_mode;
Bool recovery_error_ignore;
Integer buffer_size;
String path;
} wal;
struct Logs {
struct Trace {
Bool enable;
} trace;
String path;
Integer max_log_file_size;
Integer log_rotate_num;
Bool log_to_stdout;
Bool log_to_file;
} logs;
struct Log {
String min_messages;
Integer rotation_age;
Integer rotation_size;
} log;
struct System {
struct Lock {
Bool enable;
} lock;
} system;
struct Transcript {
Bool enable;
String replay;
} transcript;
};
extern ServerConfig config;
std::vector<std::string>
ParsePreloadCollection(const std::string&);
std::vector<int64_t>
ParseGPUDevices(const std::string&);
} // namespace milvus
internal/core/src/value/config/ServerConfigTest.cpp 已删除 100644 → 0
浏览文件 @ bb7a0766
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include <gtest/gtest.h>
#include "value/config/ServerConfig.h"
TEST(ServerConfigTest, parse_invalid_devices) {
auto collections = milvus::ParseGPUDevices("gpu0,gpu1");
ASSERT_EQ(collections.size(), 0);
}
internal/core/src/value/status/ServerStatus.cpp 已删除 100644 → 0
浏览文件 @ bb7a0766
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "value/status/ServerStatus.h"
namespace milvus {
ServerStatus server_status;
} // namespace milvus
internal/core/src/value/status/ServerStatus.h 已删除 100644 → 0
浏览文件 @ bb7a0766
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <string>
#include "value/Value.h"
#include "value/ValueType.h"
namespace milvus {
struct ServerStatus {
using String = Value<std::string>;
using Bool = Value<bool>;
using Integer = Value<int64_t>;
using Floating = Value<double>;
Bool indexing;
};
extern ServerStatus server_status;
} // namespace milvus
internal/core/src/value/status/StatusInit.cpp 已删除 100644 → 0
浏览文件 @ bb7a0766
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include <string>
#include <unordered_map>
#include "value/status/ServerStatus.h"
namespace milvus {
#define Bool_(name, modifiable, default, is_valid) \
{ #name, CreateBoolValue(#name, modifiable, server_status.name, default, is_valid) }
#define String_(name, modifiable, default, is_valid) \
{ #name, CreateStringValue(#name, modifiable, server_status.name, default, is_valid) }
#define Enum_(name, modifiable, enumd, default, is_valid) \
{ #name, CreateEnumValue(#name, modifiable, enumd, server_status.name, default, is_valid) }
#define Integer_(name, modifiable, lower_bound, upper_bound, default, is_valid) \
{ #name, CreateIntegerValue(#name, modifiable, lower_bound, upper_bound, server_status.name, default, is_valid) }
#define Floating_(name, modifiable, lower_bound, upper_bound, default, is_valid) \
{ #name, CreateFloatingValue(#name, modifiable, lower_bound, upper_bound, server_status.name, default, is_valid) }
#define Size_(name, modifiable, lower_bound, upper_bound, default, is_valid) \
{ #name, CreateSizeValue(#name, modifiable, lower_bound, upper_bound, server_status.name, default, is_valid) }
#define Bool(name, default) Bool_(name, true, default, nullptr)
#define String(name, default) String_(name, true, default, nullptr)
#define Enum(name, enumd, default) Enum_(name, true, enumd, default, nullptr)
#define Integer(name, lower_bound, upper_bound, default) \
Integer_(name, true, lower_bound, upper_bound, default, nullptr)
#define Floating(name, lower_bound, upper_bound, default) \
Floating_(name, true, lower_bound, upper_bound, default, nullptr)
#define Size(name, lower_bound, upper_bound, default) Size_(name, true, lower_bound, upper_bound, default, nullptr)
std::unordered_map<std::string, BaseValuePtr>
InitStatus() {
return std::unordered_map<std::string, BaseValuePtr>{
Bool(indexing, false),
};
}
} // namespace milvus
internal/core/src/value/status/StatusMgr.cpp 已删除 100644 → 0
浏览文件 @ bb7a0766
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "value/status/StatusMgr.h"
namespace milvus {
extern std::unordered_map<std::string, BaseValuePtr>
InitStatus();
StatusMgr StatusMgr::instance;
StatusMgr::StatusMgr() : ValueMgr(InitStatus()) {
}
void
StatusMgr::Set(const std::string& name, const std::string& value, bool update) {
/* Check if existed */
if (status_list_.find(name) == status_list_.end()) {
throw std::runtime_error("Status " + name + " not found.");
}
try {
/* Set value, throws ValueError only. */
status_list_.at(name)->Set(value, update);
} catch (ValueError& e) {
/* Convert to std::runtime_error. */
throw std::runtime_error(e.message());
} catch (...) {
/* Unexpected exception, output status and value. */
throw std::runtime_error("Unexpected exception happened when setting " + value + " to " + name + ".");
}
}
std::string
StatusMgr::Get(const std::string& name) const {
try {
auto& status = status_list_.at(name);
return status->Get();
} catch (std::out_of_range& ex) {
throw std::runtime_error("Status " + name + " not found.");
} catch (...) {
throw std::runtime_error("Unexpected exception happened when getting status " + name + ".");
}
}
} // namespace milvus
internal/core/src/value/status/StatusMgr.h 已删除 100644 → 0
浏览文件 @ bb7a0766
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <string>
#include <unordered_map>
#include "value/ValueMgr.h"
#include "value/ValueType.h"
namespace milvus {
class StatusMgr : public ValueMgr {
public:
static ValueMgr&
GetInstance() {
return instance;
}
private:
static StatusMgr instance;
public:
StatusMgr();
/* throws std::exception only */
void
Set(const std::string& name, const std::string& value, bool update) override;
/* throws std::exception only */
std::string
Get(const std::string& name) const override;
private:
const std::unordered_map<std::string, BaseValuePtr>& status_list_ = value_list_;
};
} // namespace milvus
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或