diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 731ff306302c3befa5d3ae8c70579dab8a69d14e..feafe74a6444887ef0b8157f47d0114857e88a5a 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -14,6 +14,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-56 - Add version information when server is started - MS-64 - Different table can have different index type - MS-52 - Return search score +- MS-66 - Support time range query ## Task diff --git a/cpp/src/metrics/PrometheusMetrics.cpp b/cpp/src/metrics/PrometheusMetrics.cpp index 7b93a33e9b87700ddab3f607e00557543d2d50b9..693051f52cb95990bc0c1466af76e51b245cfcee 100644 --- a/cpp/src/metrics/PrometheusMetrics.cpp +++ b/cpp/src/metrics/PrometheusMetrics.cpp @@ -5,7 +5,7 @@ ******************************************************************************/ #include "PrometheusMetrics.h" - +#include "utils/Log.h" namespace zilliz { namespace vecwise { @@ -13,18 +13,23 @@ namespace server { ServerError PrometheusMetrics::Init() { - ConfigNode& configNode = ServerConfig::GetInstance().GetConfig(CONFIG_METRIC); - startup_ = configNode.GetValue(CONFIG_METRIC_IS_STARTUP) == "true" ? true:false; - // Following should be read from config file. - const std::string bind_address = configNode.GetChild(CONFIG_PROMETHEUS).GetValue(CONFIG_METRIC_PROMETHEUS_PORT); - const std::string uri = std::string("/metrics"); - const std::size_t num_threads = 2; - - // Init Exposer - exposer_ptr_ = std::make_shared(bind_address, uri, num_threads); - - // Exposer Registry - exposer_ptr_->RegisterCollectable(registry_); + try { + ConfigNode &configNode = ServerConfig::GetInstance().GetConfig(CONFIG_METRIC); + startup_ = configNode.GetValue(CONFIG_METRIC_IS_STARTUP) == "true" ? true : false; + // Following should be read from config file. + const std::string bind_address = configNode.GetChild(CONFIG_PROMETHEUS).GetValue(CONFIG_METRIC_PROMETHEUS_PORT); + const std::string uri = std::string("/metrics"); + const std::size_t num_threads = 2; + + // Init Exposer + exposer_ptr_ = std::make_shared(bind_address, uri, num_threads); + + // Exposer Registry + exposer_ptr_->RegisterCollectable(registry_); + } catch (std::exception& ex) { + SERVER_LOG_ERROR << "Failed to connect prometheus server: " << std::string(ex.what()); + return SERVER_UNEXPECTED_ERROR; + } return SERVER_SUCCESS; } diff --git a/cpp/src/sdk/examples/simple/src/ClientTest.cpp b/cpp/src/sdk/examples/simple/src/ClientTest.cpp index 3eccb6bfa6f70c0336570ce5b3225db7390713da..5f6bdff07fdb35dc0712952363e1e6e9829e83b1 100644 --- a/cpp/src/sdk/examples/simple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/simple/src/ClientTest.cpp @@ -75,6 +75,18 @@ namespace { return str; } + std::string CurrentTmDate() { + time_t tt; + time( &tt ); + tt = tt + 8*3600; + tm* t= gmtime( &tt ); + + std::string str = std::to_string(t->tm_year + 1900) + "-" + std::to_string(t->tm_mon + 1) + + "-" + std::to_string(t->tm_mday); + + return str; + } + std::string GetTableName() { static std::string s_id(CurrentTime()); return s_id; @@ -170,6 +182,10 @@ ClientTest::Test(const std::string& address, const std::string& port) { BuildVectors(SEARCH_TARGET, SEARCH_TARGET + 10, record_array); std::vector query_range_array; + Range rg; + rg.start_value = CurrentTmDate(); + rg.end_value = CurrentTmDate(); + query_range_array.emplace_back(rg); std::vector topk_query_result_array; Status stat = conn->SearchVector(TABLE_NAME, record_array, query_range_array, TOP_K, topk_query_result_array); std::cout << "SearchVector function call status: " << stat.ToString() << std::endl; diff --git a/cpp/src/sdk/src/client/ClientProxy.cpp b/cpp/src/sdk/src/client/ClientProxy.cpp index 8805329c87cde8211530ae331a62463b861bc866..95d6d33426f5ec1ad619d235fadbb52a115e7541 100644 --- a/cpp/src/sdk/src/client/ClientProxy.cpp +++ b/cpp/src/sdk/src/client/ClientProxy.cpp @@ -159,6 +159,8 @@ ClientProxy::SearchVector(const std::string &table_name, } try { + + //step 1: convert vectors data std::vector thrift_records; for(auto& record : query_record_array) { thrift::RowRecord thrift_record; @@ -172,10 +174,21 @@ ClientProxy::SearchVector(const std::string &table_name, thrift_records.emplace_back(thrift_record); } + //step 2: convert range array std::vector thrift_ranges; + for(auto& range : query_range_array) { + thrift::Range thrift_range; + thrift_range.__set_start_value(range.start_value); + thrift_range.__set_end_value(range.end_value); + + thrift_ranges.emplace_back(thrift_range); + } + + //step 3: search vectors std::vector result_array; ClientPtr()->interface()->SearchVector(result_array, table_name, thrift_records, thrift_ranges, topk); + //step 4: convert result array for(auto& thrift_topk_result : result_array) { TopKQueryResult result; diff --git a/cpp/src/server/MegasearchTask.cpp b/cpp/src/server/MegasearchTask.cpp index 7c78b100468a3269400df7292af175180ac97202..dd459f16fa929a28ae98a1d15e0a115c704c1efb 100644 --- a/cpp/src/server/MegasearchTask.cpp +++ b/cpp/src/server/MegasearchTask.cpp @@ -75,6 +75,74 @@ namespace { return map_type[type]; } + + ServerError + ConvertRowRecordToFloatArray(const std::vector& record_array, + uint64_t dimension, + std::vector& float_array) { + ServerError error_code; + uint64_t vec_count = record_array.size(); + float_array.resize(vec_count*dimension);//allocate enough memory + for(uint64_t i = 0; i < vec_count; i++) { + const auto& record = record_array[i]; + if(record.vector_data.empty()) { + error_code = SERVER_INVALID_ARGUMENT; + SERVER_LOG_ERROR << "No vector provided in record"; + return error_code; + } + uint64_t vec_dim = record.vector_data.size()/sizeof(double);//how many double value? + if(vec_dim != dimension) { + SERVER_LOG_ERROR << "Invalid vector dimension: " << vec_dim + << " vs. group dimension:" << dimension; + error_code = SERVER_INVALID_VECTOR_DIMENSION; + return error_code; + } + + //convert double array to float array(thrift has no float type) + const double* d_p = reinterpret_cast(record.vector_data.data()); + for(uint64_t d = 0; d < vec_dim; d++) { + float_array[i*vec_dim + d] = (float)(d_p[d]); + } + } + + return SERVER_SUCCESS; + } + + static constexpr long DAY_SECONDS = 86400; + + ServerError + ConvertTimeRangeToDBDates(const std::vector &range_array, + std::vector& dates) { + dates.clear(); + ServerError error_code; + for(auto& range : range_array) { + time_t tt_start, tt_end; + tm tm_start, tm_end; + if(!CommonUtil::TimeStrToTime(range.start_value, tt_start, tm_start)){ + error_code = SERVER_INVALID_TIME_RANGE; + SERVER_LOG_ERROR << "Invalid time range: " << range.start_value; + return error_code; + } + + if(!CommonUtil::TimeStrToTime(range.end_value, tt_end, tm_end)){ + error_code = SERVER_INVALID_TIME_RANGE; + SERVER_LOG_ERROR << "Invalid time range: " << range.end_value; + return error_code; + } + + long days = (tt_end > tt_start) ? (tt_end - tt_start)/DAY_SECONDS : (tt_start - tt_end)/DAY_SECONDS; + for(long i = 0; i <= days; i++) { + time_t tt_day = tt_start + DAY_SECONDS*i; + tm tm_day; + CommonUtil::ConvertTime(tt_day, tm_day); + + long date = tm_day.tm_year*10000 + tm_day.tm_mon*100 + tm_day.tm_mday;//according to db logic + dates.push_back(date); + } + } + + return SERVER_SUCCESS; + } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -237,37 +305,17 @@ ServerError AddVectorTask::OnExecute() { rc.Record("check validation"); //step 2: prepare float data - uint64_t vec_count = (uint64_t)record_array_.size(); - uint64_t group_dim = table_info.dimension_; std::vector vec_f; - vec_f.resize(vec_count*group_dim);//allocate enough memory - for(uint64_t i = 0; i < vec_count; i++) { - const auto& record = record_array_[i]; - if(record.vector_data.empty()) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "No vector provided in record"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; - } - uint64_t vec_dim = record.vector_data.size()/sizeof(double);//how many double value? - if(vec_dim != group_dim) { - SERVER_LOG_ERROR << "Invalid vector dimension: " << vec_dim - << " vs. group dimension:" << group_dim; - error_code_ = SERVER_INVALID_VECTOR_DIMENSION; - error_msg_ = "Engine failed: " + stat.ToString(); - return error_code_; - } - - //convert double array to float array(thrift has no float type) - const double* d_p = reinterpret_cast(record.vector_data.data()); - for(uint64_t d = 0; d < vec_dim; d++) { - vec_f[i*vec_dim + d] = (float)(d_p[d]); - } + error_code_ = ConvertRowRecordToFloatArray(record_array_, table_info.dimension_, vec_f); + if(error_code_ != SERVER_SUCCESS) { + error_msg_ = "Invalid row record data"; + return error_code_; } rc.Record("prepare vectors data"); //step 3: insert vectors + uint64_t vec_count = (uint64_t)record_array_.size(); stat = DB()->InsertVectors(table_name_, vec_count, vec_f.data(), record_ids_); rc.Record("add vectors to engine"); if(!stat.ok()) { @@ -342,44 +390,29 @@ ServerError SearchVectorTask::OnExecute() { return error_code_; } + //step 3: check date range, and convert to db dates + std::vector dates; + error_code_ = ConvertTimeRangeToDBDates(range_array_, dates); + if(error_code_ != SERVER_SUCCESS) { + error_msg_ = "Invalid query range"; + return error_code_; + } + rc.Record("check validation"); //step 3: prepare float data std::vector vec_f; - uint64_t record_count = (uint64_t)record_array_.size(); - vec_f.resize(record_count*table_info.dimension_); - - for(uint64_t i = 0; i < record_array_.size(); i++) { - const auto& record = record_array_[i]; - if (record.vector_data.empty()) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Query record has no vector"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; - } - - uint64_t vec_dim = record.vector_data.size() / sizeof(double);//how many double value? - if (vec_dim != table_info.dimension_) { - SERVER_LOG_ERROR << "Invalid vector dimension: " << vec_dim - << " vs. group dimension:" << table_info.dimension_; - error_code_ = SERVER_INVALID_VECTOR_DIMENSION; - error_msg_ = "Engine failed: " + stat.ToString(); - return error_code_; - } - - //convert double array to float array(thrift has no float type) - const double* d_p = reinterpret_cast(record.vector_data.data()); - for(uint64_t d = 0; d < vec_dim; d++) { - vec_f[i*vec_dim + d] = (float)(d_p[d]); - } + error_code_ = ConvertRowRecordToFloatArray(record_array_, table_info.dimension_, vec_f); + if(error_code_ != SERVER_SUCCESS) { + error_msg_ = "Invalid row record data"; + return error_code_; } rc.Record("prepare vector data"); - //step 4: search vectors - std::vector dates; engine::QueryResults results; + uint64_t record_count = (uint64_t)record_array_.size(); stat = DB()->Query(table_name_, (size_t)top_k_, record_count, vec_f.data(), dates, results); rc.Record("search vectors from engine"); if(!stat.ok()) { diff --git a/cpp/src/utils/CommonUtil.cpp b/cpp/src/utils/CommonUtil.cpp index e095c291518aee23d34d61f2a4be14e96ee1b6a6..910fed0f79b6f0ab54d772d8f08b512edb3e0111 100644 --- a/cpp/src/utils/CommonUtil.cpp +++ b/cpp/src/utils/CommonUtil.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include "boost/filesystem.hpp" @@ -150,15 +151,39 @@ std::string CommonUtil::GetExePath() { return exe_path; } -void CommonUtil::ConvertTime(int year, int month, int day, int hour, int minute, int second, time_t& t_t) { - tm t_m; - t_m.tm_year = year; - t_m.tm_mon = month; - t_m.tm_mday = day; - t_m.tm_hour = hour; - t_m.tm_min = minute; - t_m.tm_sec = second; - t_t = mktime(&t_m); +bool CommonUtil::TimeStrToTime(const std::string& time_str, + time_t &time_integer, + tm &time_struct, + const std::string& format) { + time_integer = 0; + memset(&time_struct, 0, sizeof(tm)); + + int ret = sscanf(time_str.c_str(), + format.c_str(), + &(time_struct.tm_year), + &(time_struct.tm_mon), + &(time_struct.tm_mday), + &(time_struct.tm_hour), + &(time_struct.tm_min), + &(time_struct.tm_sec)); + if(ret <= 0) { + return false; + } + + time_struct.tm_year -= 1900; + time_struct.tm_mon--; + time_integer = mktime(&time_struct); + + return true; +} + +void CommonUtil::ConvertTime(time_t time_integer, tm &time_struct) { + tm* t_m = localtime (&time_integer); + memcpy(&time_struct, t_m, sizeof(tm)); +} + +void ConvertTime(tm time_struct, time_t &time_integer) { + time_integer = mktime(&time_struct); } } diff --git a/cpp/src/utils/CommonUtil.h b/cpp/src/utils/CommonUtil.h index 0a458cf9eb6ed871290bbf80b189262b5fa86a98..4a8c9e4e842c2a5376db78492a67df9c7a701592 100755 --- a/cpp/src/utils/CommonUtil.h +++ b/cpp/src/utils/CommonUtil.h @@ -26,7 +26,13 @@ class CommonUtil { static std::string GetExePath(); - static void ConvertTime(int year, int month, int day, int hour, int minute, int second, time_t& t_t); + static bool TimeStrToTime(const std::string& time_str, + time_t &time_integer, + tm &time_struct, + const std::string& format = "%d-%d-%d %d:%d:%d"); + + static void ConvertTime(time_t time_integer, tm &time_struct); + static void ConvertTime(tm time_struct, time_t &time_integer); }; }