diff --git a/cpp/.gitignore b/cpp/.gitignore index e99e0273f35bccb8fb3c80dbc3435a742b688b69..88c9c4c2f480b9b5c6bb0f24f49b7f0ebc17b74d 100644 --- a/cpp/.gitignore +++ b/cpp/.gitignore @@ -4,3 +4,4 @@ third_party/bzip2-1.0.6/ third_party/sqlite3/ megasearch/ conf/server_config.yaml +version.h diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index cf501e00d9be774f6bb42bd7429f3e1299fc9642..db388802918b7563333c36ae43c696e78916ffbf 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -11,6 +11,9 @@ Please mark all change in change log and use the ticket from JIRA. ## New Feature - MS-57 - Implement index load/search pipeline +- MS-56 - Add version information when server is started +- MS-64 - Different table can have different index type +- MS-52 - Return search score ## Task @@ -45,4 +48,4 @@ Please mark all change in change log and use the ticket from JIRA. - MS-1 - Add CHANGELOG.md - MS-4 - Refactor the vecwise_engine code structure - +- MS-62 - Search range to all if no date specified diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 58ba3aea7de7454f34e94748619fdee060273ca3..7362dc75971e96566365571659cb337ca9aa0da2 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -7,11 +7,35 @@ cmake_minimum_required(VERSION 3.14) message(STATUS "Building using CMake version: ${CMAKE_VERSION}") -set(MEGASEARCH_VERSION "0.1.0") +MACRO (GET_CURRENT_TIME CURRENT_TIME) + execute_process(COMMAND "date" +"%Y-%m-%d %H:%M.%S" OUTPUT_VARIABLE ${CURRENT_TIME}) +ENDMACRO (GET_CURRENT_TIME) -string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" MEGASEARCH_BASE_VERSION "${MEGASEARCH_VERSION}") +GET_CURRENT_TIME(BUILD_TIME) +string(REGEX REPLACE "\n" "" BUILD_TIME ${BUILD_TIME}) +message(STATUS "Build time = ${BUILD_TIME}") -project(megasearch VERSION "${MEGASEARCH_BASE_VERSION}") +MACRO (GET_GIT_BRANCH_NAME GIT_BRANCH_NAME) + execute_process(COMMAND "git" symbolic-ref --short HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) +ENDMACRO (GET_GIT_BRANCH_NAME) + +GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME) +string(REGEX REPLACE "\n" "" GIT_BRANCH_NAME ${GIT_BRANCH_NAME}) + +set(MEGASEARCH_VERSION "${GIT_BRANCH_NAME}") +string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]" MEGASEARCH_VERSION "${MEGASEARCH_VERSION}") +message(STATUS "Build version = ${MEGASEARCH_VERSION}") + +if(CMAKE_BUILD_TYPE STREQUAL "Release") + set(BUILD_TYPE "release") +else() + set(BUILD_TYPE "debug") +endif() +message(STATUS "Build type = ${BUILD_TYPE}") + +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/version.h.macro ${CMAKE_CURRENT_SOURCE_DIR}/version.h) + +project(megasearch VERSION "${MEGASEARCH_VERSION}") project(vecwise_engine LANGUAGES CUDA CXX) set(MEGASEARCH_VERSION_MAJOR "${megasearch_VERSION_MAJOR}") diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index d63ee9cb12f9717070db45227c5ef2f3f1eec861..2c1219b506e9c918842b2cbbb806bc8e6b2e3d85 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -86,7 +86,6 @@ set(third_party_libs prometheus-cpp-pull prometheus-cpp-core civetweb - rocksdb boost_system_static boost_filesystem_static boost_serialization_static @@ -169,7 +168,6 @@ endif () set(server_libs vecwise_engine - librocksdb.a libthrift.a pthread libyaml-cpp.a diff --git a/cpp/src/db/DB.cpp b/cpp/src/db/DB.cpp index 610e353358078496c685d1be27843fcf9d3b622e..bfb83a446c061ebc3dd6dbb97f2249a953a4f071 100644 --- a/cpp/src/db/DB.cpp +++ b/cpp/src/db/DB.cpp @@ -16,15 +16,7 @@ namespace engine { DB::~DB() {} void DB::Open(const Options& options, DB** dbptr) { - *dbptr = nullptr; - -#ifdef GPU_VERSION - std::string default_index_type{"Faiss,IVF"}; -#else - std::string default_index_type{"Faiss,IDMap"}; -#endif - - *dbptr = DBFactory::Build(options, default_index_type); + *dbptr = DBFactory::Build(options); return; } diff --git a/cpp/src/db/DBImpl.inl b/cpp/src/db/DBImpl.cpp similarity index 69% rename from cpp/src/db/DBImpl.inl rename to cpp/src/db/DBImpl.cpp index 878466f402adb026ed2dc0b11ee22f6e852d3667..068e2a8d2485a4352c6fb4d5e6f44881081643e3 100644 --- a/cpp/src/db/DBImpl.inl +++ b/cpp/src/db/DBImpl.cpp @@ -3,11 +3,10 @@ * Unauthorized copying of this file, via any medium is strictly prohibited. * Proprietary and confidential. ******************************************************************************/ -#pragma once - #include "DBImpl.h" #include "DBMetaImpl.h" #include "Env.h" +#include "EngineFactory.h" #include "metrics/Metrics.h" #include "scheduler/SearchScheduler.h" @@ -23,81 +22,109 @@ namespace zilliz { namespace vecwise { namespace engine { +namespace { + +void CollectInsertMetrics(double total_time, size_t n, bool succeed) { + double avg_time = total_time / n; + for (int i = 0; i < n; ++i) { + server::Metrics::GetInstance().AddVectorsDurationHistogramOberve(avg_time); + } -template -DBImpl::DBImpl(const Options& options) +// server::Metrics::GetInstance().add_vector_duration_seconds_quantiles().Observe((average_time)); + if (succeed) { + server::Metrics::GetInstance().AddVectorsSuccessTotalIncrement(n); + server::Metrics::GetInstance().AddVectorsSuccessGaugeSet(n); + } + else { + server::Metrics::GetInstance().AddVectorsFailTotalIncrement(n); + server::Metrics::GetInstance().AddVectorsFailGaugeSet(n); + } +} + +void CollectQueryMetrics(double total_time, size_t nq) { + for (int i = 0; i < nq; ++i) { + server::Metrics::GetInstance().QueryResponseSummaryObserve(total_time); + } + auto average_time = total_time / nq; + server::Metrics::GetInstance().QueryVectorResponseSummaryObserve(average_time, nq); + server::Metrics::GetInstance().QueryVectorResponsePerSecondGaugeSet(double (nq) / total_time); +} + +void CollectFileMetrics(int file_type, size_t file_size, double total_time) { + switch(file_type) { + case meta::TableFileSchema::RAW: + case meta::TableFileSchema::TO_INDEX: { + server::Metrics::GetInstance().SearchRawDataDurationSecondsHistogramObserve(total_time); + server::Metrics::GetInstance().RawFileSizeHistogramObserve(file_size); + server::Metrics::GetInstance().RawFileSizeTotalIncrement(file_size); + server::Metrics::GetInstance().RawFileSizeGaugeSet(file_size); + break; + } + default: { + server::Metrics::GetInstance().SearchIndexDataDurationSecondsHistogramObserve(total_time); + server::Metrics::GetInstance().IndexFileSizeHistogramObserve(file_size); + server::Metrics::GetInstance().IndexFileSizeTotalIncrement(file_size); + server::Metrics::GetInstance().IndexFileSizeGaugeSet(file_size); + break; + } + } +} + +} + + +DBImpl::DBImpl(const Options& options) : env_(options.env), options_(options), bg_compaction_scheduled_(false), shutting_down_(false), bg_build_index_started_(false), pMeta_(new meta::DBMetaImpl(options_.meta)), - pMemMgr_(new MemManager(pMeta_, options_)) { + pMemMgr_(new MemManager(pMeta_, options_)) { StartTimerTasks(options_.memory_sync_interval); } -template -Status DBImpl::CreateTable(meta::TableSchema& table_schema) { +Status DBImpl::CreateTable(meta::TableSchema& table_schema) { return pMeta_->CreateTable(table_schema); } -template -Status DBImpl::DescribeTable(meta::TableSchema& table_schema) { +Status DBImpl::DescribeTable(meta::TableSchema& table_schema) { return pMeta_->DescribeTable(table_schema); } -template -Status DBImpl::HasTable(const std::string& table_id, bool& has_or_not) { +Status DBImpl::HasTable(const std::string& table_id, bool& has_or_not) { return pMeta_->HasTable(table_id, has_or_not); } -template -Status DBImpl::InsertVectors(const std::string& table_id_, +Status DBImpl::InsertVectors(const std::string& table_id_, size_t n, const float* vectors, IDNumbers& vector_ids_) { auto start_time = METRICS_NOW_TIME; Status status = pMemMgr_->InsertVectors(table_id_, n, vectors, vector_ids_); auto end_time = METRICS_NOW_TIME; - + double total_time = METRICS_MICROSECONDS(start_time,end_time); // std::chrono::microseconds time_span = std::chrono::duration_cast(end_time - start_time); // double average_time = double(time_span.count()) / n; - double total_time = METRICS_MICROSECONDS(start_time,end_time); - double avg_time = total_time / double(n); - for (int i = 0; i < n; ++i) { - server::Metrics::GetInstance().AddVectorsDurationHistogramOberve(avg_time); - } + CollectInsertMetrics(total_time, n, status.ok()); + return status; -// server::Metrics::GetInstance().add_vector_duration_seconds_quantiles().Observe((average_time)); - if (!status.ok()) { - server::Metrics::GetInstance().AddVectorsFailTotalIncrement(n); - server::Metrics::GetInstance().AddVectorsFailGaugeSet(n); - return status; - } - server::Metrics::GetInstance().AddVectorsSuccessTotalIncrement(n); - server::Metrics::GetInstance().AddVectorsSuccessGaugeSet(n); } -template -Status DBImpl::Query(const std::string &table_id, size_t k, size_t nq, +Status DBImpl::Query(const std::string &table_id, size_t k, size_t nq, const float *vectors, QueryResults &results) { auto start_time = METRICS_NOW_TIME; meta::DatesT dates = {meta::Meta::GetDate()}; Status result = Query(table_id, k, nq, vectors, dates, results); auto end_time = METRICS_NOW_TIME; - auto total_time = METRICS_MICROSECONDS(start_time, end_time); - auto average_time = total_time / nq; - for (int i = 0; i < nq; ++i) { - server::Metrics::GetInstance().QueryResponseSummaryObserve(total_time); - } - server::Metrics::GetInstance().QueryVectorResponseSummaryObserve(average_time, nq); - server::Metrics::GetInstance().QueryVectorResponsePerSecondGaugeSet(double (nq) / total_time); - server::Metrics::GetInstance().QueryResponsePerSecondGaugeSet(1.0 / total_time); + auto total_time = METRICS_MICROSECONDS(start_time,end_time); + + CollectQueryMetrics(total_time, nq); + return result; } -template -Status DBImpl::Query(const std::string& table_id, size_t k, size_t nq, +Status DBImpl::Query(const std::string& table_id, size_t k, size_t nq, const float* vectors, const meta::DatesT& dates, QueryResults& results) { #if 0 return QuerySync(table_id, k, nq, vectors, dates, results); @@ -106,8 +133,7 @@ Status DBImpl::Query(const std::string& table_id, size_t k, size_t nq, #endif } -template -Status DBImpl::QuerySync(const std::string& table_id, size_t k, size_t nq, +Status DBImpl::QuerySync(const std::string& table_id, size_t k, size_t nq, const float* vectors, const meta::DatesT& dates, QueryResults& results) { meta::DatePartionedTableFilesSchema files; auto status = pMeta_->FilesToSearch(table_id, dates, files); @@ -119,16 +145,16 @@ Status DBImpl::QuerySync(const std::string& table_id, size_t k, size_t meta::TableFilesSchema raw_files; for (auto &day_files : files) { for (auto &file : day_files.second) { - file.file_type == meta::TableFileSchema::INDEX ? + file.file_type_ == meta::TableFileSchema::INDEX ? index_files.push_back(file) : raw_files.push_back(file); } } int dim = 0; if (!index_files.empty()) { - dim = index_files[0].dimension; + dim = index_files[0].dimension_; } else if (!raw_files.empty()) { - dim = raw_files[0].dimension; + dim = raw_files[0].dimension_; } else { LOG(DEBUG) << "no files to search"; return Status::OK(); @@ -161,38 +187,20 @@ Status DBImpl::QuerySync(const std::string& table_id, size_t k, size_t auto search_in_index = [&](meta::TableFilesSchema& file_vec) -> void { for (auto &file : file_vec) { - EngineT index(file.dimension, file.location); - index.Load(); - auto file_size = index.PhysicalSize()/(1024*1024); + ExecutionEnginePtr index = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_); + index->Load(); + auto file_size = index->PhysicalSize(); search_set_size += file_size; - LOG(DEBUG) << "Search file_type " << file.file_type << " Of Size: " - << file_size << " M"; + LOG(DEBUG) << "Search file_type " << file.file_type_ << " Of Size: " + << file_size/(1024*1024) << " M"; - int inner_k = index.Count() < k ? index.Count() : k; + int inner_k = index->Count() < k ? index->Count() : k; auto start_time = METRICS_NOW_TIME; - index.Search(nq, vectors, inner_k, output_distence, output_ids); + index->Search(nq, vectors, inner_k, output_distence, output_ids); auto end_time = METRICS_NOW_TIME; auto total_time = METRICS_MICROSECONDS(start_time, end_time); - if(file.file_type == meta::TableFileSchema::RAW) { - server::Metrics::GetInstance().SearchRawDataDurationSecondsHistogramObserve(total_time); - server::Metrics::GetInstance().RawFileSizeHistogramObserve(file_size*1024*1024); - server::Metrics::GetInstance().RawFileSizeTotalIncrement(file_size*1024*1024); - server::Metrics::GetInstance().RawFileSizeGaugeSet(file_size*1024*1024); - - } else if(file.file_type == meta::TableFileSchema::TO_INDEX) { - - server::Metrics::GetInstance().SearchRawDataDurationSecondsHistogramObserve(total_time); - server::Metrics::GetInstance().RawFileSizeHistogramObserve(file_size*1024*1024); - server::Metrics::GetInstance().RawFileSizeTotalIncrement(file_size*1024*1024); - server::Metrics::GetInstance().RawFileSizeGaugeSet(file_size*1024*1024); - - } else { - server::Metrics::GetInstance().SearchIndexDataDurationSecondsHistogramObserve(total_time); - server::Metrics::GetInstance().IndexFileSizeHistogramObserve(file_size*1024*1024); - server::Metrics::GetInstance().IndexFileSizeTotalIncrement(file_size*1024*1024); - server::Metrics::GetInstance().IndexFileSizeGaugeSet(file_size*1024*1024); - } + CollectFileMetrics(file.file_type_, file_size, total_time); cluster(output_ids, output_distence, inner_k); // cluster to each query memset(output_distence, 0, k * nq * sizeof(float)); memset(output_ids, 0, k * nq * sizeof(long)); @@ -234,7 +242,7 @@ Status DBImpl::QuerySync(const std::string& table_id, size_t k, size_t int inner_k = dis.size() < k ? dis.size() : k; for (int i = 0; i < inner_k; ++i) { - res.emplace_back(nns[output_ids[i]]); // mapping + res.emplace_back(std::make_pair(nns[output_ids[i]], output_distence[i])); // mapping } results.push_back(res); // append to result list res.clear(); @@ -259,9 +267,10 @@ Status DBImpl::QuerySync(const std::string& table_id, size_t k, size_t return Status::OK(); } -template -Status DBImpl::QueryAsync(const std::string& table_id, size_t k, size_t nq, +Status DBImpl::QueryAsync(const std::string& table_id, size_t k, size_t nq, const float* vectors, const meta::DatesT& dates, QueryResults& results) { + + //step 1: get files to search meta::DatePartionedTableFilesSchema files; auto status = pMeta_->FilesToSearch(table_id, dates, files); if (!status.ok()) { return status; } @@ -277,29 +286,25 @@ Status DBImpl::QueryAsync(const std::string& table_id, size_t k, size_t } } + //step 2: put search task to scheduler SearchScheduler& scheduler = SearchScheduler::GetInstance(); scheduler.ScheduleSearchTask(context); context->WaitResult(); + + //step 3: construct results auto& context_result = context->GetResult(); - for(auto& topk_result : context_result) { - QueryResult ids; - for(auto& pair : topk_result) { - ids.push_back(pair.second); - } - results.emplace_back(ids); - } + results.swap(context_result); return Status::OK(); } -template -void DBImpl::StartTimerTasks(int interval) { - bg_timer_thread_ = std::thread(&DBImpl::BackgroundTimerTask, this, interval); +void DBImpl::StartTimerTasks(int interval) { + bg_timer_thread_ = std::thread(&DBImpl::BackgroundTimerTask, this, interval); } -template -void DBImpl::BackgroundTimerTask(int interval) { + +void DBImpl::BackgroundTimerTask(int interval) { Status status; server::SystemInfo::GetInstance().Init(); while (true) { @@ -322,22 +327,19 @@ void DBImpl::BackgroundTimerTask(int interval) { } } -template -void DBImpl::TrySchedule() { +void DBImpl::TrySchedule() { if (bg_compaction_scheduled_) return; if (!bg_error_.ok()) return; bg_compaction_scheduled_ = true; - env_->Schedule(&DBImpl::BGWork, this); + env_->Schedule(&DBImpl::BGWork, this); } -template -void DBImpl::BGWork(void* db_) { +void DBImpl::BGWork(void* db_) { reinterpret_cast(db_)->BackgroundCall(); } -template -void DBImpl::BackgroundCall() { +void DBImpl::BackgroundCall() { std::lock_guard lock(mutex_); assert(bg_compaction_scheduled_); @@ -350,13 +352,11 @@ void DBImpl::BackgroundCall() { bg_work_finish_signal_.notify_all(); } - -template -Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, +Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, const meta::TableFilesSchema& files) { meta::TableFileSchema table_file; - table_file.table_id = table_id; - table_file.date = date; + table_file.table_id_ = table_id; + table_file.date_ = date; Status status = pMeta_->CreateTableFile(table_file); if (!status.ok()) { @@ -364,7 +364,8 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::Date return status; } - EngineT index(table_file.dimension, table_file.location); + ExecutionEnginePtr index = + EngineFactory::Build(table_file.dimension_, table_file.location_, (EngineType)table_file.engine_type_); meta::TableFilesSchema updated; long index_size = 0; @@ -372,41 +373,40 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::Date for (auto& file : files) { auto start_time = METRICS_NOW_TIME; - index.Merge(file.location); + index->Merge(file.location_); auto file_schema = file; auto end_time = METRICS_NOW_TIME; auto total_time = METRICS_MICROSECONDS(start_time,end_time); server::Metrics::GetInstance().MemTableMergeDurationSecondsHistogramObserve(total_time); - file_schema.file_type = meta::TableFileSchema::TO_DELETE; + file_schema.file_type_ = meta::TableFileSchema::TO_DELETE; updated.push_back(file_schema); - LOG(DEBUG) << "Merging file " << file_schema.file_id; - index_size = index.Size(); + LOG(DEBUG) << "Merging file " << file_schema.file_id_; + index_size = index->Size(); if (index_size >= options_.index_trigger_size) break; } - index.Serialize(); + index->Serialize(); if (index_size >= options_.index_trigger_size) { - table_file.file_type = meta::TableFileSchema::TO_INDEX; + table_file.file_type_ = meta::TableFileSchema::TO_INDEX; } else { - table_file.file_type = meta::TableFileSchema::RAW; + table_file.file_type_ = meta::TableFileSchema::RAW; } - table_file.size = index_size; + table_file.size_ = index_size; updated.push_back(table_file); status = pMeta_->UpdateTableFiles(updated); - LOG(DEBUG) << "New merged file " << table_file.file_id << - " of size=" << index.PhysicalSize()/(1024*1024) << " M"; + LOG(DEBUG) << "New merged file " << table_file.file_id_ << + " of size=" << index->PhysicalSize()/(1024*1024) << " M"; - index.Cache(); + index->Cache(); return status; } -template -Status DBImpl::BackgroundMergeFiles(const std::string& table_id) { +Status DBImpl::BackgroundMergeFiles(const std::string& table_id) { meta::DatePartionedTableFilesSchema raw_files; auto status = pMeta_->FilesToMerge(table_id, raw_files); if (!status.ok()) { @@ -433,37 +433,36 @@ Status DBImpl::BackgroundMergeFiles(const std::string& table_id) { return Status::OK(); } -template -Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { +Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { meta::TableFileSchema table_file; - table_file.table_id = file.table_id; - table_file.date = file.date; + table_file.table_id_ = file.table_id_; + table_file.date_ = file.date_; Status status = pMeta_->CreateTableFile(table_file); if (!status.ok()) { return status; } - EngineT to_index(file.dimension, file.location); + ExecutionEnginePtr to_index = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_); - to_index.Load(); + to_index->Load(); auto start_time = METRICS_NOW_TIME; - auto index = to_index.BuildIndex(table_file.location); + auto index = to_index->BuildIndex(table_file.location_); auto end_time = METRICS_NOW_TIME; auto total_time = METRICS_MICROSECONDS(start_time, end_time); server::Metrics::GetInstance().BuildIndexDurationSecondsHistogramObserve(total_time); - table_file.file_type = meta::TableFileSchema::INDEX; - table_file.size = index->Size(); + table_file.file_type_ = meta::TableFileSchema::INDEX; + table_file.size_ = index->Size(); auto to_remove = file; - to_remove.file_type = meta::TableFileSchema::TO_DELETE; + to_remove.file_type_ = meta::TableFileSchema::TO_DELETE; meta::TableFilesSchema update_files = {to_remove, table_file}; pMeta_->UpdateTableFiles(update_files); - LOG(DEBUG) << "New index file " << table_file.file_id << " of size " + LOG(DEBUG) << "New index file " << table_file.file_id_ << " of size " << index->PhysicalSize()/(1024*1024) << " M" - << " from file " << to_remove.file_id; + << " from file " << to_remove.file_id_; index->Cache(); pMeta_->Archive(); @@ -471,8 +470,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { return Status::OK(); } -template -void DBImpl::BackgroundBuildIndex() { +void DBImpl::BackgroundBuildIndex() { std::lock_guard lock(build_index_mutex_); assert(bg_build_index_started_); meta::TableFilesSchema to_index_files; @@ -492,18 +490,16 @@ void DBImpl::BackgroundBuildIndex() { bg_build_index_finish_signal_.notify_all(); } -template -Status DBImpl::TryBuildIndex() { +Status DBImpl::TryBuildIndex() { if (bg_build_index_started_) return Status::OK(); if (shutting_down_.load(std::memory_order_acquire)) return Status::OK(); bg_build_index_started_ = true; - std::thread build_index_task(&DBImpl::BackgroundBuildIndex, this); + std::thread build_index_task(&DBImpl::BackgroundBuildIndex, this); build_index_task.detach(); return Status::OK(); } -template -void DBImpl::BackgroundCompaction() { +void DBImpl::BackgroundCompaction() { std::vector table_ids; pMemMgr_->Serialize(table_ids); @@ -517,18 +513,15 @@ void DBImpl::BackgroundCompaction() { } } -template -Status DBImpl::DropAll() { +Status DBImpl::DropAll() { return pMeta_->DropAll(); } -template -Status DBImpl::Size(long& result) { +Status DBImpl::Size(long& result) { return pMeta_->Size(result); } -template -DBImpl::~DBImpl() { +DBImpl::~DBImpl() { { std::unique_lock lock(mutex_); shutting_down_.store(true, std::memory_order_release); diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index b67aa8a0a9ac54176d85150783e509d8202648de..4f9ab54cfbebd4e64e72a262e8fb3ba9bb9d0173 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -8,12 +8,12 @@ #include "DB.h" #include "MemManager.h" #include "Types.h" -#include "Traits.h" #include #include #include #include +#include namespace zilliz { namespace vecwise { @@ -25,11 +25,10 @@ namespace meta { class Meta; } -template class DBImpl : public DB { public: using MetaPtr = meta::Meta::Ptr; - using MemManagerPtr = typename MemManager::Ptr; + using MemManagerPtr = typename MemManager::Ptr; DBImpl(const Options& options); @@ -100,5 +99,3 @@ private: } // namespace engine } // namespace vecwise } // namespace zilliz - -#include "DBImpl.inl" diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 731830fb3896713123ebdad36edc802d89f95828..95611f9823c523b8dc2d1e068f26a894593f23a4 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -7,6 +7,7 @@ #include "IDGenerator.h" #include "Utils.h" #include "MetaConsts.h" +#include "Factories.h" #include "metrics/Metrics.h" #include @@ -29,26 +30,30 @@ using namespace sqlite_orm; inline auto StoragePrototype(const std::string &path) { return make_storage(path, make_table("Table", - make_column("id", &TableSchema::id, primary_key()), - make_column("table_id", &TableSchema::table_id, unique()), - make_column("dimension", &TableSchema::dimension), - make_column("created_on", &TableSchema::created_on), - make_column("files_cnt", &TableSchema::files_cnt, default_value(0))), + make_column("id", &TableSchema::id_, primary_key()), + make_column("table_id", &TableSchema::table_id_, unique()), + make_column("dimension", &TableSchema::dimension_), + make_column("created_on", &TableSchema::created_on_), + make_column("files_cnt", &TableSchema::files_cnt_, default_value(0)), + make_column("engine_type", &TableSchema::engine_type_), + make_column("store_raw_data", &TableSchema::store_raw_data_)), make_table("TableFile", - make_column("id", &TableFileSchema::id, primary_key()), - make_column("table_id", &TableFileSchema::table_id), - make_column("file_id", &TableFileSchema::file_id), - make_column("file_type", &TableFileSchema::file_type), - make_column("size", &TableFileSchema::size, default_value(0)), - make_column("updated_time", &TableFileSchema::updated_time), - make_column("created_on", &TableFileSchema::created_on), - make_column("date", &TableFileSchema::date)) + make_column("id", &TableFileSchema::id_, primary_key()), + make_column("table_id", &TableFileSchema::table_id_), + make_column("engine_type", &TableFileSchema::engine_type_), + make_column("file_id", &TableFileSchema::file_id_), + make_column("file_type", &TableFileSchema::file_type_), + make_column("size", &TableFileSchema::size_, default_value(0)), + make_column("updated_time", &TableFileSchema::updated_time_), + make_column("created_on", &TableFileSchema::created_on_), + make_column("date", &TableFileSchema::date_)) ); } using ConnectorT = decltype(StoragePrototype("")); static std::unique_ptr ConnectorPtr; +using ConditionT = decltype(c(&TableFileSchema::id_) == 1UL); std::string DBMetaImpl::GetTablePath(const std::string &table_id) { return options_.path + "/tables/" + table_id; @@ -61,13 +66,13 @@ std::string DBMetaImpl::GetTableDatePartitionPath(const std::string &table_id, D } void DBMetaImpl::GetTableFilePath(TableFileSchema &group_file) { - if (group_file.date == EmptyDate) { - group_file.date = Meta::GetDate(); + if (group_file.date_ == EmptyDate) { + group_file.date_ = Meta::GetDate(); } std::stringstream ss; - ss << GetTableDatePartitionPath(group_file.table_id, group_file.date) - << "/" << group_file.file_id; - group_file.location = ss.str(); + ss << GetTableDatePartitionPath(group_file.table_id_, group_file.date_) + << "/" << group_file.file_id_; + group_file.location_ = ss.str(); } Status DBMetaImpl::NextTableId(std::string &table_id) { @@ -119,7 +124,7 @@ Status DBMetaImpl::DropPartitionsByDates(const std::string &table_id, } TableSchema table_schema; - table_schema.table_id = table_id; + table_schema.table_id_ = table_id; auto status = DescribeTable(table_schema); if (!status.ok()) { return status; @@ -136,11 +141,11 @@ Status DBMetaImpl::DropPartitionsByDates(const std::string &table_id, try { ConnectorPtr->update_all( set( - c(&TableFileSchema::file_type) = (int) TableFileSchema::TO_DELETE + c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE ), where( - c(&TableFileSchema::table_id) == table_id and - in(&TableFileSchema::date, dates) + c(&TableFileSchema::table_id_) == table_id and + in(&TableFileSchema::date_, dates) )); } catch (std::exception &e) { LOG(DEBUG) << e.what(); @@ -151,17 +156,17 @@ Status DBMetaImpl::DropPartitionsByDates(const std::string &table_id, Status DBMetaImpl::CreateTable(TableSchema &table_schema) { server::Metrics::GetInstance().MetaAccessTotalIncrement(); - if (table_schema.table_id == "") { - NextTableId(table_schema.table_id); + if (table_schema.table_id_ == "") { + NextTableId(table_schema.table_id_); } - table_schema.files_cnt = 0; - table_schema.id = -1; - table_schema.created_on = utils::GetMicroSecTimeStamp(); + table_schema.files_cnt_ = 0; + table_schema.id_ = -1; + table_schema.created_on_ = utils::GetMicroSecTimeStamp(); auto start_time = METRICS_NOW_TIME; { try { auto id = ConnectorPtr->insert(table_schema); - table_schema.id = id; + table_schema.id_ = id; } catch (...) { return Status::DBTransactionError("Add Table Error"); } @@ -170,7 +175,7 @@ Status DBMetaImpl::CreateTable(TableSchema &table_schema) { auto total_time = METRICS_MICROSECONDS(start_time, end_time); server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); - auto group_path = GetTablePath(table_schema.table_id); + auto group_path = GetTablePath(table_schema.table_id_); if (!boost::filesystem::is_directory(group_path)) { auto ret = boost::filesystem::create_directories(group_path); @@ -187,21 +192,25 @@ Status DBMetaImpl::DescribeTable(TableSchema &table_schema) { try { server::Metrics::GetInstance().MetaAccessTotalIncrement(); auto start_time = METRICS_NOW_TIME; - auto groups = ConnectorPtr->select(columns(&TableSchema::id, - &TableSchema::table_id, - &TableSchema::files_cnt, - &TableSchema::dimension), - where(c(&TableSchema::table_id) == table_schema.table_id)); + auto groups = ConnectorPtr->select(columns(&TableSchema::id_, + &TableSchema::table_id_, + &TableSchema::files_cnt_, + &TableSchema::dimension_, + &TableSchema::engine_type_, + &TableSchema::store_raw_data_), + where(c(&TableSchema::table_id_) == table_schema.table_id_)); auto end_time = METRICS_NOW_TIME; auto total_time = METRICS_MICROSECONDS(start_time, end_time); server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); assert(groups.size() <= 1); if (groups.size() == 1) { - table_schema.id = std::get<0>(groups[0]); - table_schema.files_cnt = std::get<2>(groups[0]); - table_schema.dimension = std::get<3>(groups[0]); + table_schema.id_ = std::get<0>(groups[0]); + table_schema.files_cnt_ = std::get<2>(groups[0]); + table_schema.dimension_ = std::get<3>(groups[0]); + table_schema.engine_type_ = std::get<4>(groups[0]); + table_schema.store_raw_data_ = std::get<5>(groups[0]); } else { - return Status::NotFound("Table " + table_schema.table_id + " not found"); + return Status::NotFound("Table " + table_schema.table_id_ + " not found"); } } catch (std::exception &e) { LOG(DEBUG) << e.what(); @@ -216,8 +225,8 @@ Status DBMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { server::Metrics::GetInstance().MetaAccessTotalIncrement(); auto start_time = METRICS_NOW_TIME; - auto tables = ConnectorPtr->select(columns(&TableSchema::id), - where(c(&TableSchema::table_id) == table_id)); + auto tables = ConnectorPtr->select(columns(&TableSchema::id_), + where(c(&TableSchema::table_id_) == table_id)); auto end_time = METRICS_NOW_TIME; auto total_time = METRICS_MICROSECONDS(start_time, end_time); server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); @@ -235,22 +244,23 @@ Status DBMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { } Status DBMetaImpl::CreateTableFile(TableFileSchema &file_schema) { - if (file_schema.date == EmptyDate) { - file_schema.date = Meta::GetDate(); + if (file_schema.date_ == EmptyDate) { + file_schema.date_ = Meta::GetDate(); } TableSchema table_schema; - table_schema.table_id = file_schema.table_id; + table_schema.table_id_ = file_schema.table_id_; auto status = DescribeTable(table_schema); if (!status.ok()) { return status; } - NextFileId(file_schema.file_id); - file_schema.file_type = TableFileSchema::NEW; - file_schema.dimension = table_schema.dimension; - file_schema.size = 0; - file_schema.created_on = utils::GetMicroSecTimeStamp(); - file_schema.updated_time = file_schema.created_on; + NextFileId(file_schema.file_id_); + file_schema.file_type_ = TableFileSchema::NEW; + file_schema.dimension_ = table_schema.dimension_; + file_schema.size_ = 0; + file_schema.created_on_ = utils::GetMicroSecTimeStamp(); + file_schema.updated_time_ = file_schema.created_on_; + file_schema.engine_type_ = table_schema.engine_type_; GetTableFilePath(file_schema); { @@ -258,7 +268,7 @@ Status DBMetaImpl::CreateTableFile(TableFileSchema &file_schema) { server::Metrics::GetInstance().MetaAccessTotalIncrement(); auto start_time = METRICS_NOW_TIME; auto id = ConnectorPtr->insert(file_schema); - file_schema.id = id; + file_schema.id_ = id; auto end_time = METRICS_NOW_TIME; auto total_time = METRICS_MICROSECONDS(start_time, end_time); server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); @@ -267,7 +277,7 @@ Status DBMetaImpl::CreateTableFile(TableFileSchema &file_schema) { } } - auto partition_path = GetTableDatePartitionPath(file_schema.table_id, file_schema.date); + auto partition_path = GetTableDatePartitionPath(file_schema.table_id_, file_schema.date_); if (!boost::filesystem::is_directory(partition_path)) { auto ret = boost::filesystem::create_directory(partition_path); @@ -286,13 +296,14 @@ Status DBMetaImpl::FilesToIndex(TableFilesSchema &files) { try { server::Metrics::GetInstance().MetaAccessTotalIncrement(); auto start_time = METRICS_NOW_TIME; - auto selected = ConnectorPtr->select(columns(&TableFileSchema::id, - &TableFileSchema::table_id, - &TableFileSchema::file_id, - &TableFileSchema::file_type, - &TableFileSchema::size, - &TableFileSchema::date), - where(c(&TableFileSchema::file_type) + auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, + &TableFileSchema::table_id_, + &TableFileSchema::file_id_, + &TableFileSchema::file_type_, + &TableFileSchema::size_, + &TableFileSchema::date_, + &TableFileSchema::engine_type_), + where(c(&TableFileSchema::file_type_) == (int) TableFileSchema::TO_INDEX)); auto end_time = METRICS_NOW_TIME; auto total_time = METRICS_MICROSECONDS(start_time, end_time); @@ -302,24 +313,26 @@ Status DBMetaImpl::FilesToIndex(TableFilesSchema &files) { TableFileSchema table_file; for (auto &file : selected) { - table_file.id = std::get<0>(file); - table_file.table_id = std::get<1>(file); - table_file.file_id = std::get<2>(file); - table_file.file_type = std::get<3>(file); - table_file.size = std::get<4>(file); - table_file.date = std::get<5>(file); + table_file.id_ = std::get<0>(file); + table_file.table_id_ = std::get<1>(file); + table_file.file_id_ = std::get<2>(file); + table_file.file_type_ = std::get<3>(file); + table_file.size_ = std::get<4>(file); + table_file.date_ = std::get<5>(file); + table_file.engine_type_ = std::get<6>(file); + GetTableFilePath(table_file); - auto groupItr = groups.find(table_file.table_id); + auto groupItr = groups.find(table_file.table_id_); if (groupItr == groups.end()) { TableSchema table_schema; - table_schema.table_id = table_file.table_id; + table_schema.table_id_ = table_file.table_id_; auto status = DescribeTable(table_schema); if (!status.ok()) { return status; } - groups[table_file.table_id] = table_schema; + groups[table_file.table_id_] = table_schema; } - table_file.dimension = groups[table_file.table_id].dimension; + table_file.dimension_ = groups[table_file.table_id_].dimension_; files.push_back(table_file); } } catch (std::exception &e) { @@ -334,51 +347,95 @@ Status DBMetaImpl::FilesToSearch(const std::string &table_id, const DatesT &partition, DatePartionedTableFilesSchema &files) { files.clear(); - DatesT today = {Meta::GetDate()}; - const DatesT &dates = (partition.empty() == true) ? today : partition; try { server::Metrics::GetInstance().MetaAccessTotalIncrement(); auto start_time = METRICS_NOW_TIME; - auto selected = ConnectorPtr->select(columns(&TableFileSchema::id, - &TableFileSchema::table_id, - &TableFileSchema::file_id, - &TableFileSchema::file_type, - &TableFileSchema::size, - &TableFileSchema::date), - where(c(&TableFileSchema::table_id) == table_id and - in(&TableFileSchema::date, dates) and - (c(&TableFileSchema::file_type) == (int) TableFileSchema::RAW or - c(&TableFileSchema::file_type) - == (int) TableFileSchema::TO_INDEX or - c(&TableFileSchema::file_type) - == (int) TableFileSchema::INDEX))); - auto end_time = METRICS_NOW_TIME; - auto total_time = METRICS_MICROSECONDS(start_time, end_time); - server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); - TableSchema table_schema; - table_schema.table_id = table_id; - auto status = DescribeTable(table_schema); - if (!status.ok()) { - return status; - } + if (partition.empty()) { + auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, + &TableFileSchema::table_id_, + &TableFileSchema::file_id_, + &TableFileSchema::file_type_, + &TableFileSchema::size_, + &TableFileSchema::date_, + &TableFileSchema::engine_type_), + where(c(&TableFileSchema::table_id_) == table_id and + (c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW or + c(&TableFileSchema::file_type_) + == (int) TableFileSchema::TO_INDEX or + c(&TableFileSchema::file_type_) + == (int) TableFileSchema::INDEX))); + auto end_time = METRICS_NOW_TIME; + auto total_time = METRICS_MICROSECONDS(start_time, end_time); + server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); + TableSchema table_schema; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; + } - TableFileSchema table_file; + TableFileSchema table_file; + + for (auto &file : selected) { + table_file.id_ = std::get<0>(file); + table_file.table_id_ = std::get<1>(file); + table_file.file_id_ = std::get<2>(file); + table_file.file_type_ = std::get<3>(file); + table_file.size_ = std::get<4>(file); + table_file.date_ = std::get<5>(file); + table_file.engine_type_ = std::get<6>(file); + table_file.dimension_ = table_schema.dimension_; + GetTableFilePath(table_file); + auto dateItr = files.find(table_file.date_); + if (dateItr == files.end()) { + files[table_file.date_] = TableFilesSchema(); + } + files[table_file.date_].push_back(table_file); + } + } + else { + auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, + &TableFileSchema::table_id_, + &TableFileSchema::file_id_, + &TableFileSchema::file_type_, + &TableFileSchema::size_, + &TableFileSchema::date_), + where(c(&TableFileSchema::table_id_) == table_id and + in(&TableFileSchema::date_, partition) and + (c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW or + c(&TableFileSchema::file_type_) + == (int) TableFileSchema::TO_INDEX or + c(&TableFileSchema::file_type_) + == (int) TableFileSchema::INDEX))); + auto end_time = METRICS_NOW_TIME; + auto total_time = METRICS_MICROSECONDS(start_time, end_time); + server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); + TableSchema table_schema; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; + } - for (auto &file : selected) { - table_file.id = std::get<0>(file); - table_file.table_id = std::get<1>(file); - table_file.file_id = std::get<2>(file); - table_file.file_type = std::get<3>(file); - table_file.size = std::get<4>(file); - table_file.date = std::get<5>(file); - table_file.dimension = table_schema.dimension; - GetTableFilePath(table_file); - auto dateItr = files.find(table_file.date); - if (dateItr == files.end()) { - files[table_file.date] = TableFilesSchema(); + TableFileSchema table_file; + + for (auto &file : selected) { + table_file.id_ = std::get<0>(file); + table_file.table_id_ = std::get<1>(file); + table_file.file_id_ = std::get<2>(file); + table_file.file_type_ = std::get<3>(file); + table_file.size_ = std::get<4>(file); + table_file.date_ = std::get<5>(file); + table_file.dimension_ = table_schema.dimension_; + GetTableFilePath(table_file); + auto dateItr = files.find(table_file.date_); + if (dateItr == files.end()) { + files[table_file.date_] = TableFilesSchema(); + } + files[table_file.date_].push_back(table_file); } - files[table_file.date].push_back(table_file); + } } catch (std::exception &e) { LOG(DEBUG) << e.what(); @@ -395,19 +452,19 @@ Status DBMetaImpl::FilesToMerge(const std::string &table_id, try { server::Metrics::GetInstance().MetaAccessTotalIncrement(); auto start_time = METRICS_NOW_TIME; - auto selected = ConnectorPtr->select(columns(&TableFileSchema::id, - &TableFileSchema::table_id, - &TableFileSchema::file_id, - &TableFileSchema::file_type, - &TableFileSchema::size, - &TableFileSchema::date), - where(c(&TableFileSchema::file_type) == (int) TableFileSchema::RAW and - c(&TableFileSchema::table_id) == table_id)); + auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, + &TableFileSchema::table_id_, + &TableFileSchema::file_id_, + &TableFileSchema::file_type_, + &TableFileSchema::size_, + &TableFileSchema::date_), + where(c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW and + c(&TableFileSchema::table_id_) == table_id)); auto end_time = METRICS_NOW_TIME; auto total_time = METRICS_MICROSECONDS(start_time, end_time); server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); TableSchema table_schema; - table_schema.table_id = table_id; + table_schema.table_id_ = table_id; auto status = DescribeTable(table_schema); if (!status.ok()) { @@ -416,19 +473,19 @@ Status DBMetaImpl::FilesToMerge(const std::string &table_id, TableFileSchema table_file; for (auto &file : selected) { - table_file.id = std::get<0>(file); - table_file.table_id = std::get<1>(file); - table_file.file_id = std::get<2>(file); - table_file.file_type = std::get<3>(file); - table_file.size = std::get<4>(file); - table_file.date = std::get<5>(file); - table_file.dimension = table_schema.dimension; + table_file.id_ = std::get<0>(file); + table_file.table_id_ = std::get<1>(file); + table_file.file_id_ = std::get<2>(file); + table_file.file_type_ = std::get<3>(file); + table_file.size_ = std::get<4>(file); + table_file.date_ = std::get<5>(file); + table_file.dimension_ = table_schema.dimension_; GetTableFilePath(table_file); - auto dateItr = files.find(table_file.date); + auto dateItr = files.find(table_file.date_); if (dateItr == files.end()) { - files[table_file.date] = TableFilesSchema(); + files[table_file.date_] = TableFilesSchema(); } - files[table_file.date].push_back(table_file); + files[table_file.date_].push_back(table_file); } } catch (std::exception &e) { LOG(DEBUG) << e.what(); @@ -441,26 +498,26 @@ Status DBMetaImpl::FilesToMerge(const std::string &table_id, Status DBMetaImpl::GetTableFile(TableFileSchema &file_schema) { try { - auto files = ConnectorPtr->select(columns(&TableFileSchema::id, - &TableFileSchema::table_id, - &TableFileSchema::file_id, - &TableFileSchema::file_type, - &TableFileSchema::size, - &TableFileSchema::date), - where(c(&TableFileSchema::file_id) == file_schema.file_id and - c(&TableFileSchema::table_id) == file_schema.table_id + auto files = ConnectorPtr->select(columns(&TableFileSchema::id_, + &TableFileSchema::table_id_, + &TableFileSchema::file_id_, + &TableFileSchema::file_type_, + &TableFileSchema::size_, + &TableFileSchema::date_), + where(c(&TableFileSchema::file_id_) == file_schema.file_id_ and + c(&TableFileSchema::table_id_) == file_schema.table_id_ )); assert(files.size() <= 1); if (files.size() == 1) { - file_schema.id = std::get<0>(files[0]); - file_schema.table_id = std::get<1>(files[0]); - file_schema.file_id = std::get<2>(files[0]); - file_schema.file_type = std::get<3>(files[0]); - file_schema.size = std::get<4>(files[0]); - file_schema.date = std::get<5>(files[0]); + file_schema.id_ = std::get<0>(files[0]); + file_schema.table_id_ = std::get<1>(files[0]); + file_schema.file_id_ = std::get<2>(files[0]); + file_schema.file_type_ = std::get<3>(files[0]); + file_schema.size_ = std::get<4>(files[0]); + file_schema.date_ = std::get<5>(files[0]); } else { - return Status::NotFound("Table:" + file_schema.table_id + - " File:" + file_schema.file_id + " not found"); + return Status::NotFound("Table:" + file_schema.table_id_ + + " File:" + file_schema.file_id_ + " not found"); } } catch (std::exception &e) { LOG(DEBUG) << e.what(); @@ -486,11 +543,11 @@ Status DBMetaImpl::Archive() { try { ConnectorPtr->update_all( set( - c(&TableFileSchema::file_type) = (int) TableFileSchema::TO_DELETE + c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE ), where( - c(&TableFileSchema::created_on) < (long) (now - usecs) and - c(&TableFileSchema::file_type) != (int) TableFileSchema::TO_DELETE + c(&TableFileSchema::created_on_) < (long) (now - usecs) and + c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE )); } catch (std::exception &e) { LOG(DEBUG) << e.what(); @@ -512,9 +569,9 @@ Status DBMetaImpl::Archive() { Status DBMetaImpl::Size(long &result) { result = 0; try { - auto selected = ConnectorPtr->select(columns(sum(&TableFileSchema::size)), + auto selected = ConnectorPtr->select(columns(sum(&TableFileSchema::size_)), where( - c(&TableFileSchema::file_type) != (int) TableFileSchema::TO_DELETE + c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE )); for (auto &sub_query : selected) { @@ -537,11 +594,11 @@ Status DBMetaImpl::DiscardFiles(long to_discard_size) { return Status::OK(); } try { - auto selected = ConnectorPtr->select(columns(&TableFileSchema::id, - &TableFileSchema::size), - where(c(&TableFileSchema::file_type) + auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, + &TableFileSchema::size_), + where(c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE), - order_by(&TableFileSchema::id), + order_by(&TableFileSchema::id_), limit(10)); std::vector ids; @@ -549,11 +606,11 @@ Status DBMetaImpl::DiscardFiles(long to_discard_size) { for (auto &file : selected) { if (to_discard_size <= 0) break; - table_file.id = std::get<0>(file); - table_file.size = std::get<1>(file); - ids.push_back(table_file.id); - LOG(DEBUG) << "Discard table_file.id=" << table_file.file_id << " table_file.size=" << table_file.size; - to_discard_size -= table_file.size; + table_file.id_ = std::get<0>(file); + table_file.size_ = std::get<1>(file); + ids.push_back(table_file.id_); + LOG(DEBUG) << "Discard table_file.id=" << table_file.file_id_ << " table_file.size=" << table_file.size_; + to_discard_size -= table_file.size_; } if (ids.size() == 0) { @@ -562,10 +619,10 @@ Status DBMetaImpl::DiscardFiles(long to_discard_size) { ConnectorPtr->update_all( set( - c(&TableFileSchema::file_type) = (int) TableFileSchema::TO_DELETE + c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE ), where( - in(&TableFileSchema::id, ids) + in(&TableFileSchema::id_, ids) )); } catch (std::exception &e) { @@ -578,7 +635,7 @@ Status DBMetaImpl::DiscardFiles(long to_discard_size) { } Status DBMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { - file_schema.updated_time = utils::GetMicroSecTimeStamp(); + file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); try { server::Metrics::GetInstance().MetaAccessTotalIncrement(); auto start_time = METRICS_NOW_TIME; @@ -588,7 +645,7 @@ Status DBMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); } catch (std::exception &e) { LOG(DEBUG) << e.what(); - LOG(DEBUG) << "table_id= " << file_schema.table_id << " file_id=" << file_schema.file_id; + LOG(DEBUG) << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; throw e; } return Status::OK(); @@ -600,7 +657,7 @@ Status DBMetaImpl::UpdateTableFiles(TableFilesSchema &files) { auto start_time = METRICS_NOW_TIME; auto commited = ConnectorPtr->transaction([&]() mutable { for (auto &file : files) { - file.updated_time = utils::GetMicroSecTimeStamp(); + file.updated_time_ = utils::GetMicroSecTimeStamp(); ConnectorPtr->update(file); } auto end_time = METRICS_NOW_TIME; @@ -621,33 +678,33 @@ Status DBMetaImpl::UpdateTableFiles(TableFilesSchema &files) { Status DBMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { auto now = utils::GetMicroSecTimeStamp(); try { - auto selected = ConnectorPtr->select(columns(&TableFileSchema::id, - &TableFileSchema::table_id, - &TableFileSchema::file_id, - &TableFileSchema::file_type, - &TableFileSchema::size, - &TableFileSchema::date), + auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, + &TableFileSchema::table_id_, + &TableFileSchema::file_id_, + &TableFileSchema::file_type_, + &TableFileSchema::size_, + &TableFileSchema::date_), where( - c(&TableFileSchema::file_type) == (int) TableFileSchema::TO_DELETE + c(&TableFileSchema::file_type_) == (int) TableFileSchema::TO_DELETE and - c(&TableFileSchema::updated_time) + c(&TableFileSchema::updated_time_) > now - seconds * US_PS)); TableFilesSchema updated; TableFileSchema table_file; for (auto &file : selected) { - table_file.id = std::get<0>(file); - table_file.table_id = std::get<1>(file); - table_file.file_id = std::get<2>(file); - table_file.file_type = std::get<3>(file); - table_file.size = std::get<4>(file); - table_file.date = std::get<5>(file); + table_file.id_ = std::get<0>(file); + table_file.table_id_ = std::get<1>(file); + table_file.file_id_ = std::get<2>(file); + table_file.file_type_ = std::get<3>(file); + table_file.size_ = std::get<4>(file); + table_file.date_ = std::get<5>(file); GetTableFilePath(table_file); - if (table_file.file_type == TableFileSchema::TO_DELETE) { - boost::filesystem::remove(table_file.location); + if (table_file.file_type_ == TableFileSchema::TO_DELETE) { + boost::filesystem::remove(table_file.location_); } - ConnectorPtr->remove(table_file.id); + ConnectorPtr->remove(table_file.id_); /* LOG(DEBUG) << "Removing deleted id=" << table_file.id << " location=" << table_file.location << std::endl; */ } } catch (std::exception &e) { @@ -660,33 +717,33 @@ Status DBMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { Status DBMetaImpl::CleanUp() { try { - auto selected = ConnectorPtr->select(columns(&TableFileSchema::id, - &TableFileSchema::table_id, - &TableFileSchema::file_id, - &TableFileSchema::file_type, - &TableFileSchema::size, - &TableFileSchema::date), + auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, + &TableFileSchema::table_id_, + &TableFileSchema::file_id_, + &TableFileSchema::file_type_, + &TableFileSchema::size_, + &TableFileSchema::date_), where( - c(&TableFileSchema::file_type) == (int) TableFileSchema::TO_DELETE + c(&TableFileSchema::file_type_) == (int) TableFileSchema::TO_DELETE or - c(&TableFileSchema::file_type) + c(&TableFileSchema::file_type_) == (int) TableFileSchema::NEW)); TableFilesSchema updated; TableFileSchema table_file; for (auto &file : selected) { - table_file.id = std::get<0>(file); - table_file.table_id = std::get<1>(file); - table_file.file_id = std::get<2>(file); - table_file.file_type = std::get<3>(file); - table_file.size = std::get<4>(file); - table_file.date = std::get<5>(file); + table_file.id_ = std::get<0>(file); + table_file.table_id_ = std::get<1>(file); + table_file.file_id_ = std::get<2>(file); + table_file.file_type_ = std::get<3>(file); + table_file.size_ = std::get<4>(file); + table_file.date_ = std::get<5>(file); GetTableFilePath(table_file); - if (table_file.file_type == TableFileSchema::TO_DELETE) { - boost::filesystem::remove(table_file.location); + if (table_file.file_type_ == TableFileSchema::TO_DELETE) { + boost::filesystem::remove(table_file.location_); } - ConnectorPtr->remove(table_file.id); + ConnectorPtr->remove(table_file.id_); /* LOG(DEBUG) << "Removing id=" << table_file.id << " location=" << table_file.location << std::endl; */ } } catch (std::exception &e) { @@ -703,19 +760,19 @@ Status DBMetaImpl::Count(const std::string &table_id, long &result) { server::Metrics::GetInstance().MetaAccessTotalIncrement(); auto start_time = METRICS_NOW_TIME; - auto selected = ConnectorPtr->select(columns(&TableFileSchema::size, - &TableFileSchema::date), - where((c(&TableFileSchema::file_type) == (int) TableFileSchema::RAW or - c(&TableFileSchema::file_type) == (int) TableFileSchema::TO_INDEX + auto selected = ConnectorPtr->select(columns(&TableFileSchema::size_, + &TableFileSchema::date_), + where((c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW or + c(&TableFileSchema::file_type_) == (int) TableFileSchema::TO_INDEX or - c(&TableFileSchema::file_type) == (int) TableFileSchema::INDEX) + c(&TableFileSchema::file_type_) == (int) TableFileSchema::INDEX) and - c(&TableFileSchema::table_id) == table_id)); + c(&TableFileSchema::table_id_) == table_id)); auto end_time = METRICS_NOW_TIME; auto total_time = METRICS_MICROSECONDS(start_time, end_time); server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); TableSchema table_schema; - table_schema.table_id = table_id; + table_schema.table_id_ = table_id; auto status = DescribeTable(table_schema); if (!status.ok()) { @@ -727,7 +784,7 @@ Status DBMetaImpl::Count(const std::string &table_id, long &result) { result += std::get<0>(file); } - result /= table_schema.dimension; + result /= table_schema.dimension_; } catch (std::exception &e) { LOG(DEBUG) << e.what(); diff --git a/cpp/src/db/EngineFactory.cpp b/cpp/src/db/EngineFactory.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a9b0bc744d4d21558195e0a1c6b56fea3baa20b5 --- /dev/null +++ b/cpp/src/db/EngineFactory.cpp @@ -0,0 +1,31 @@ +/******************************************************************************* + * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved + * Unauthorized copying of this file, via any medium is strictly prohibited. + * Proprietary and confidential. + ******************************************************************************/ +#include "EngineFactory.h" +#include "FaissExecutionEngine.h" +#include "Log.h" + +namespace zilliz { +namespace vecwise { +namespace engine { + +ExecutionEnginePtr +EngineFactory::Build(uint16_t dimension, + const std::string& location, + EngineType type) { + switch(type) { + case EngineType::FAISS_IDMAP: + return ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IDMap", "IDMap,Flat")); + case EngineType::FAISS_IVFFLAT: + return ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IVF", "IDMap,Flat")); + default: + ENGINE_LOG_ERROR << "Unsupportted engine type"; + return nullptr; + } +} + +} +} +} \ No newline at end of file diff --git a/cpp/src/db/Traits.h b/cpp/src/db/EngineFactory.h similarity index 61% rename from cpp/src/db/Traits.h rename to cpp/src/db/EngineFactory.h index 1740bc87ed6e821f36b0add1a282ba3a62b5b591..fe984d70cd52b80c36ecfeecb15669888c132f15 100644 --- a/cpp/src/db/Traits.h +++ b/cpp/src/db/EngineFactory.h @@ -5,22 +5,20 @@ ******************************************************************************/ #pragma once +#include "Status.h" +#include "ExecutionEngine.h" namespace zilliz { namespace vecwise { namespace engine { -struct IVFIndexTrait { - static const char* BuildIndexType; - static const char* RawIndexType; +class EngineFactory { +public: + static ExecutionEnginePtr Build(uint16_t dimension, + const std::string& location, + EngineType type); }; -struct IDMapIndexTrait { - static const char* BuildIndexType; - static const char* RawIndexType; -}; - - -} // namespace engine -} // namespace vecwise -} // namespace zilliz +} +} +} diff --git a/cpp/src/db/ExecutionEngine.cpp b/cpp/src/db/ExecutionEngine.cpp index 6eb04bb788e17dc403c49975c760a369efb9b8d9..54349b7b69ff436b1677560f50a378f25eff76d5 100644 --- a/cpp/src/db/ExecutionEngine.cpp +++ b/cpp/src/db/ExecutionEngine.cpp @@ -11,8 +11,7 @@ namespace zilliz { namespace vecwise { namespace engine { -template -Status ExecutionEngine::AddWithIds(const std::vector& vectors, const std::vector& vector_ids) { +Status ExecutionEngine::AddWithIds(const std::vector& vectors, const std::vector& vector_ids) { long n1 = (long)vectors.size(); long n2 = (long)vector_ids.size(); if (n1 != n2) { @@ -22,60 +21,6 @@ Status ExecutionEngine::AddWithIds(const std::vector& vectors, c return AddWithIds(n1, vectors.data(), vector_ids.data()); } -template -Status ExecutionEngine::AddWithIds(long n, const float *xdata, const long *xids) { - return static_cast(this)->AddWithIds(n, xdata, xids); -} - -template -size_t ExecutionEngine::Count() const { - return static_cast(this)->Count(); -} - -template -size_t ExecutionEngine::Size() const { - return static_cast(this)->Size(); -} - -template -size_t ExecutionEngine::PhysicalSize() const { - return static_cast(this)->PhysicalSize(); -} - -template -Status ExecutionEngine::Serialize() { - return static_cast(this)->Serialize(); -} - -template -Status ExecutionEngine::Load() { - return static_cast(this)->Load(); -} - -template -Status ExecutionEngine::Merge(const std::string& location) { - return static_cast(this)->Merge(location); -} - -template -Status ExecutionEngine::Search(long n, - const float *data, - long k, - float *distances, - long *labels) const { - return static_cast(this)->Search(n, data, k, distances, labels); -} - -template -Status ExecutionEngine::Cache() { - return static_cast(this)->Cache(); -} - -template -std::shared_ptr ExecutionEngine::BuildIndex(const std::string& location) { - return static_cast(this)->BuildIndex(location); -} - } // namespace engine } // namespace vecwise diff --git a/cpp/src/db/ExecutionEngine.h b/cpp/src/db/ExecutionEngine.h index 6ad91959e8fe8c95a5f8e0d32ef2b098525b993b..ad4355786f2322968e5620c3f7d036b212598de3 100644 --- a/cpp/src/db/ExecutionEngine.h +++ b/cpp/src/db/ExecutionEngine.h @@ -14,38 +14,47 @@ namespace zilliz { namespace vecwise { namespace engine { -template +enum class EngineType { + INVALID = 0, + FAISS_IDMAP = 1, + FAISS_IVFFLAT, +}; + class ExecutionEngine { public: - Status AddWithIds(const std::vector& vectors, - const std::vector& vector_ids); + virtual Status AddWithIds(const std::vector& vectors, + const std::vector& vector_ids); - Status AddWithIds(long n, const float *xdata, const long *xids); + virtual Status AddWithIds(long n, const float *xdata, const long *xids) = 0; - size_t Count() const; + virtual size_t Count() const = 0; - size_t Size() const; + virtual size_t Size() const = 0; - size_t PhysicalSize() const; + virtual size_t Dimension() const = 0; - Status Serialize(); + virtual size_t PhysicalSize() const = 0; - Status Load(); + virtual Status Serialize() = 0; - Status Merge(const std::string& location); + virtual Status Load() = 0; - Status Search(long n, + virtual Status Merge(const std::string& location) = 0; + + virtual Status Search(long n, const float *data, long k, float *distances, - long *labels) const; + long *labels) const = 0; - std::shared_ptr BuildIndex(const std::string&); + virtual std::shared_ptr BuildIndex(const std::string&) = 0; - Status Cache(); + virtual Status Cache() = 0; }; +using ExecutionEnginePtr = std::shared_ptr; + } // namespace engine } // namespace vecwise diff --git a/cpp/src/db/Factories.cpp b/cpp/src/db/Factories.cpp index 191d3dbae5f04ca722143a71e5b274b2534389c9..bbaa17343b8738cfd306c19dd1d86f619671676c 100644 --- a/cpp/src/db/Factories.cpp +++ b/cpp/src/db/Factories.cpp @@ -5,8 +5,6 @@ //////////////////////////////////////////////////////////////////////////////// #include "Factories.h" #include "DBImpl.h" -#include "FaissExecutionEngine.h" -#include "Traits.h" #include #include @@ -45,28 +43,14 @@ std::shared_ptr DBMetaImplFactory::Build() { return std::shared_ptr(new meta::DBMetaImpl(options)); } -std::shared_ptr DBFactory::Build(const std::string& db_type) { +std::shared_ptr DBFactory::Build() { auto options = OptionsFactory::Build(); - auto db = DBFactory::Build(options, db_type); + auto db = DBFactory::Build(options); return std::shared_ptr(db); } -DB* DBFactory::Build(const Options& options, const std::string& db_type) { - std::stringstream ss(db_type); - std::string token; - std::vector tokens; - while (std::getline(ss, token, ',')) { - tokens.push_back(token); - } - - assert(tokens.size()==2); - assert(tokens[0]=="Faiss"); - if (tokens[1] == "IVF") { - return new DBImpl>(options); - } else if (tokens[1] == "IDMap") { - return new DBImpl>(options); - } - return nullptr; +DB* DBFactory::Build(const Options& options) { + return new DBImpl(options); } } // namespace engine diff --git a/cpp/src/db/Factories.h b/cpp/src/db/Factories.h index 0e9ab71bb12edb5719288875be52a23ab43a3e02..9bcd3ffb6776f28b97b8727aa9ffe9ad1c83a153 100644 --- a/cpp/src/db/Factories.h +++ b/cpp/src/db/Factories.h @@ -8,6 +8,7 @@ #include "DB.h" #include "DBMetaImpl.h" #include "Options.h" +#include "ExecutionEngine.h" #include #include @@ -29,8 +30,8 @@ struct DBMetaImplFactory { }; struct DBFactory { - static std::shared_ptr Build(const std::string& db_type = "Faiss,IVF"); - static DB* Build(const Options&, const std::string& db_type = "Faiss,IVF"); + static std::shared_ptr Build(); + static DB* Build(const Options&); }; } // namespace engine diff --git a/cpp/src/db/FaissExecutionEngine.inl b/cpp/src/db/FaissExecutionEngine.cpp similarity index 71% rename from cpp/src/db/FaissExecutionEngine.inl rename to cpp/src/db/FaissExecutionEngine.cpp index 0f22f41a7c2d139f953e5289db7c81cdfbb5c5f6..c676ac74b20a279ac37d78fd33790cd6f1907fb0 100644 --- a/cpp/src/db/FaissExecutionEngine.inl +++ b/cpp/src/db/FaissExecutionEngine.cpp @@ -3,8 +3,6 @@ * Unauthorized copying of this file, via any medium is strictly prohibited. * Proprietary and confidential. ******************************************************************************/ -#pragma once - #include "FaissExecutionEngine.h" #include @@ -23,47 +21,53 @@ namespace vecwise { namespace engine { -template -FaissExecutionEngine::FaissExecutionEngine(uint16_t dimension, const std::string& location) - : pIndex_(faiss::index_factory(dimension, IndexTrait::RawIndexType)), - location_(location) { +FaissExecutionEngine::FaissExecutionEngine(uint16_t dimension, + const std::string& location, + const std::string& build_index_type, + const std::string& raw_index_type) + : pIndex_(faiss::index_factory(dimension, raw_index_type.c_str())), + location_(location), + build_index_type_(build_index_type), + raw_index_type_(raw_index_type) { } -template -FaissExecutionEngine::FaissExecutionEngine(std::shared_ptr index, const std::string& location) +FaissExecutionEngine::FaissExecutionEngine(std::shared_ptr index, + const std::string& location, + const std::string& build_index_type, + const std::string& raw_index_type) : pIndex_(index), - location_(location) { + location_(location), + build_index_type_(build_index_type), + raw_index_type_(raw_index_type) { } -template -Status FaissExecutionEngine::AddWithIds(long n, const float *xdata, const long *xids) { +Status FaissExecutionEngine::AddWithIds(long n, const float *xdata, const long *xids) { pIndex_->add_with_ids(n, xdata, xids); return Status::OK(); } -template -size_t FaissExecutionEngine::Count() const { +size_t FaissExecutionEngine::Count() const { return (size_t)(pIndex_->ntotal); } -template -size_t FaissExecutionEngine::Size() const { +size_t FaissExecutionEngine::Size() const { return (size_t)(Count() * pIndex_->d)*sizeof(float); } -template -size_t FaissExecutionEngine::PhysicalSize() const { +size_t FaissExecutionEngine::Dimension() const { + return pIndex_->d; +} + +size_t FaissExecutionEngine::PhysicalSize() const { return (size_t)(Count() * pIndex_->d)*sizeof(float); } -template -Status FaissExecutionEngine::Serialize() { +Status FaissExecutionEngine::Serialize() { write_index(pIndex_.get(), location_.c_str()); return Status::OK(); } -template -Status FaissExecutionEngine::Load() { +Status FaissExecutionEngine::Load() { auto index = zilliz::vecwise::cache::CpuCacheMgr::GetInstance()->GetIndex(location_); bool to_cache = false; auto start_time = METRICS_NOW_TIME; @@ -90,8 +94,7 @@ Status FaissExecutionEngine::Load() { return Status::OK(); } -template -Status FaissExecutionEngine::Merge(const std::string& location) { +Status FaissExecutionEngine::Merge(const std::string& location) { if (location == location_) { return Status::Error("Cannot Merge Self"); } @@ -105,12 +108,11 @@ Status FaissExecutionEngine::Merge(const std::string& location) { return Status::OK(); } -template -typename FaissExecutionEngine::Ptr -FaissExecutionEngine::BuildIndex(const std::string& location) { +ExecutionEnginePtr +FaissExecutionEngine::BuildIndex(const std::string& location) { auto opd = std::make_shared(); opd->d = pIndex_->d; - opd->index_type = IndexTrait::BuildIndexType; + opd->index_type = build_index_type_; IndexBuilderPtr pBuilder = GetIndexBuilder(opd); auto from_index = dynamic_cast(pIndex_.get()); @@ -119,13 +121,12 @@ FaissExecutionEngine::BuildIndex(const std::string& location) { dynamic_cast(from_index->index)->xb.data(), from_index->id_map.data()); - Ptr new_ee(new FaissExecutionEngine(index->data(), location)); + ExecutionEnginePtr new_ee(new FaissExecutionEngine(index->data(), location, build_index_type_, raw_index_type_)); new_ee->Serialize(); return new_ee; } -template -Status FaissExecutionEngine::Search(long n, +Status FaissExecutionEngine::Search(long n, const float *data, long k, float *distances, @@ -138,8 +139,7 @@ Status FaissExecutionEngine::Search(long n, return Status::OK(); } -template -Status FaissExecutionEngine::Cache() { +Status FaissExecutionEngine::Cache() { zilliz::vecwise::cache::CpuCacheMgr::GetInstance( )->InsertItem(location_, std::make_shared(pIndex_)); diff --git a/cpp/src/db/FaissExecutionEngine.h b/cpp/src/db/FaissExecutionEngine.h index 2915acc0ebc7325dc790f03441bc6b3e30c3be2f..e41fe064569d044c0cb270c07cf20f1a17f24870 100644 --- a/cpp/src/db/FaissExecutionEngine.h +++ b/cpp/src/db/FaissExecutionEngine.h @@ -19,50 +19,54 @@ namespace vecwise { namespace engine { -template -class FaissExecutionEngine : public ExecutionEngine> { +class FaissExecutionEngine : public ExecutionEngine { public: - using Ptr = std::shared_ptr>; - FaissExecutionEngine(uint16_t dimension, const std::string& location); - FaissExecutionEngine(std::shared_ptr index, const std::string& location); + FaissExecutionEngine(uint16_t dimension, + const std::string& location, + const std::string& build_index_type, + const std::string& raw_index_type); - Status AddWithIds(const std::vector& vectors, - const std::vector& vector_ids); + FaissExecutionEngine(std::shared_ptr index, + const std::string& location, + const std::string& build_index_type, + const std::string& raw_index_type); - Status AddWithIds(long n, const float *xdata, const long *xids); + Status AddWithIds(long n, const float *xdata, const long *xids) override; - size_t Count() const; + size_t Count() const override; - size_t Size() const; + size_t Size() const override; - size_t PhysicalSize() const; + size_t Dimension() const override; - Status Serialize(); + size_t PhysicalSize() const override; - Status Load(); + Status Serialize() override; - Status Merge(const std::string& location); + Status Load() override; + + Status Merge(const std::string& location) override; Status Search(long n, const float *data, long k, float *distances, - long *labels) const; + long *labels) const override; - Ptr BuildIndex(const std::string&); + ExecutionEnginePtr BuildIndex(const std::string&) override; - Status Cache(); + Status Cache() override; protected: - std::shared_ptr pIndex_; std::string location_; + + std::string build_index_type_; + std::string raw_index_type_; }; } // namespace engine } // namespace vecwise } // namespace zilliz - -#include "FaissExecutionEngine.inl" diff --git a/cpp/src/db/Log.h b/cpp/src/db/Log.h new file mode 100644 index 0000000000000000000000000000000000000000..1e938591a0697105f3a257bbc095159295744512 --- /dev/null +++ b/cpp/src/db/Log.h @@ -0,0 +1,26 @@ +/******************************************************************************* + * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved + * Unauthorized copying of this file, via any medium is strictly prohibited. + * Proprietary and confidential. + ******************************************************************************/ +#pragma once + +#include + +namespace zilliz { +namespace vecwise { +namespace engine { + +#define ENGINE_DOMAIN_NAME "[ENGINE] " +#define ENGINE_ERROR_TEXT "ENGINE Error:" + +#define ENGINE_LOG_TRACE LOG(TRACE) << ENGINE_DOMAIN_NAME +#define ENGINE_LOG_DEBUG LOG(DEBUG) << ENGINE_DOMAIN_NAME +#define ENGINE_LOG_INFO LOG(INFO) << ENGINE_DOMAIN_NAME +#define ENGINE_LOG_WARNING LOG(WARNING) << ENGINE_DOMAIN_NAME +#define ENGINE_LOG_ERROR LOG(ERROR) << ENGINE_DOMAIN_NAME +#define ENGINE_LOG_FATAL LOG(FATAL) << ENGINE_DOMAIN_NAME + +} // namespace sql +} // namespace zilliz +} // namespace server diff --git a/cpp/src/db/MemManager.inl b/cpp/src/db/MemManager.cpp similarity index 67% rename from cpp/src/db/MemManager.inl rename to cpp/src/db/MemManager.cpp index b4ffc1b8db27db744721ea4cfb3eed21f0174248..a7d639d655aee46d7e718709559e24ff4fda872c 100644 --- a/cpp/src/db/MemManager.inl +++ b/cpp/src/db/MemManager.cpp @@ -3,11 +3,10 @@ * Unauthorized copying of this file, via any medium is strictly prohibited. * Proprietary and confidential. ******************************************************************************/ -#pragma once - #include "MemManager.h" #include "Meta.h" #include "MetaConsts.h" +#include "EngineFactory.h" #include "metrics/Metrics.h" #include @@ -19,18 +18,17 @@ namespace zilliz { namespace vecwise { namespace engine { -template -MemVectors::MemVectors(const std::shared_ptr& meta_ptr, +MemVectors::MemVectors(const std::shared_ptr& meta_ptr, const meta::TableFileSchema& schema, const Options& options) : pMeta_(meta_ptr), options_(options), schema_(schema), pIdGenerator_(new SimpleIDGenerator()), - pEE_(new EngineT(schema_.dimension, schema_.location)) { + pEE_(EngineFactory::Build(schema_.dimension_, schema_.location_, (EngineType)schema_.engine_type_)) { } -template -void MemVectors::Add(size_t n_, const float* vectors_, IDNumbers& vector_ids_) { + +void MemVectors::Add(size_t n_, const float* vectors_, IDNumbers& vector_ids_) { auto start_time = METRICS_NOW_TIME; pIdGenerator_->GetNextIDNumbers(n_, vector_ids_); pEE_->AddWithIds(n_, vectors_, vector_ids_.data()); @@ -39,43 +37,39 @@ void MemVectors::Add(size_t n_, const float* vectors_, IDNumbers& vecto server::Metrics::GetInstance().AddVectorsPerSecondGaugeSet(static_cast(n_), static_cast(schema_.dimension), total_time); } -template -size_t MemVectors::Total() const { +size_t MemVectors::Total() const { return pEE_->Count(); } -template -size_t MemVectors::ApproximateSize() const { +size_t MemVectors::ApproximateSize() const { return pEE_->Size(); } -template -Status MemVectors::Serialize(std::string& table_id) { - table_id = schema_.table_id; +Status MemVectors::Serialize(std::string& table_id) { + table_id = schema_.table_id_; auto size = ApproximateSize(); auto start_time = METRICS_NOW_TIME; pEE_->Serialize(); auto end_time = METRICS_NOW_TIME; auto total_time = METRICS_MICROSECONDS(start_time, end_time); - schema_.size = size; + schema_.size_ = size; server::Metrics::GetInstance().DiskStoreIOSpeedGaugeSet(size/total_time); - schema_.file_type = (size >= options_.index_trigger_size) ? + schema_.file_type_ = (size >= options_.index_trigger_size) ? meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; auto status = pMeta_->UpdateTableFile(schema_); - LOG(DEBUG) << "New " << ((schema_.file_type == meta::TableFileSchema::RAW) ? "raw" : "to_index") - << " file " << schema_.file_id << " of size " << pEE_->Size() / meta::M << " M"; + LOG(DEBUG) << "New " << ((schema_.file_type_ == meta::TableFileSchema::RAW) ? "raw" : "to_index") + << " file " << schema_.file_id_ << " of size " << pEE_->Size() / meta::M << " M"; pEE_->Cache(); return status; } -template -MemVectors::~MemVectors() { +MemVectors::~MemVectors() { if (pIdGenerator_ != nullptr) { delete pIdGenerator_; pIdGenerator_ = nullptr; @@ -85,9 +79,7 @@ MemVectors::~MemVectors() { /* * MemManager */ - -template -typename MemManager::MemVectorsPtr MemManager::GetMemByTable( +MemManager::MemVectorsPtr MemManager::GetMemByTable( const std::string& table_id) { auto memIt = memMap_.find(table_id); if (memIt != memMap_.end()) { @@ -95,18 +87,17 @@ typename MemManager::MemVectorsPtr MemManager::GetMemByTable( } meta::TableFileSchema table_file; - table_file.table_id = table_id; + table_file.table_id_ = table_id; auto status = pMeta_->CreateTableFile(table_file); if (!status.ok()) { return nullptr; } - memMap_[table_id] = MemVectorsPtr(new MemVectors(pMeta_, table_file, options_)); + memMap_[table_id] = MemVectorsPtr(new MemVectors(pMeta_, table_file, options_)); return memMap_[table_id]; } -template -Status MemManager::InsertVectors(const std::string& table_id_, +Status MemManager::InsertVectors(const std::string& table_id_, size_t n_, const float* vectors_, IDNumbers& vector_ids_) { @@ -115,8 +106,7 @@ Status MemManager::InsertVectors(const std::string& table_id_, return InsertVectorsNoLock(table_id_, n_, vectors_, vector_ids_); } -template -Status MemManager::InsertVectorsNoLock(const std::string& table_id, +Status MemManager::InsertVectorsNoLock(const std::string& table_id, size_t n, const float* vectors, IDNumbers& vector_ids) { @@ -129,8 +119,7 @@ Status MemManager::InsertVectorsNoLock(const std::string& table_id, return Status::OK(); } -template -Status MemManager::ToImmutable() { +Status MemManager::ToImmutable() { std::unique_lock lock(mutex_); for (auto& kv: memMap_) { immMems_.push_back(kv.second); @@ -140,8 +129,7 @@ Status MemManager::ToImmutable() { return Status::OK(); } -template -Status MemManager::Serialize(std::vector& table_ids) { +Status MemManager::Serialize(std::vector& table_ids) { ToImmutable(); std::unique_lock lock(serialization_mtx_); std::string table_id; diff --git a/cpp/src/db/MemManager.h b/cpp/src/db/MemManager.h index 35272f4211d32e4340ad8252e9a0f963e21c1432..1363bf971ad17dbd985e0874637f704bfab84f58 100644 --- a/cpp/src/db/MemManager.h +++ b/cpp/src/db/MemManager.h @@ -5,6 +5,7 @@ ******************************************************************************/ #pragma once +#include "ExecutionEngine.h" #include "IDGenerator.h" #include "Status.h" #include "Meta.h" @@ -23,12 +24,10 @@ namespace meta { class Meta; } -template class MemVectors { public: - using EnginePtr = typename EngineT::Ptr; using MetaPtr = meta::Meta::Ptr; - using Ptr = std::shared_ptr>; + using Ptr = std::shared_ptr; explicit MemVectors(const std::shared_ptr&, const meta::TableFileSchema&, const Options&); @@ -43,7 +42,7 @@ public: ~MemVectors(); - const std::string& Location() const { return schema_.location; } + const std::string& Location() const { return schema_.location_; } private: MemVectors() = delete; @@ -54,18 +53,17 @@ private: Options options_; meta::TableFileSchema schema_; IDGenerator* pIdGenerator_; - EnginePtr pEE_; + ExecutionEnginePtr pEE_; }; // MemVectors -template class MemManager { public: using MetaPtr = meta::Meta::Ptr; - using MemVectorsPtr = typename MemVectors::Ptr; - using Ptr = std::shared_ptr>; + using MemVectorsPtr = typename MemVectors::Ptr; + using Ptr = std::shared_ptr; MemManager(const std::shared_ptr& meta, const Options& options) : pMeta_(meta), options_(options) {} @@ -96,4 +94,3 @@ private: } // namespace engine } // namespace vecwise } // namespace zilliz -#include "MemManager.inl" diff --git a/cpp/src/db/MetaTypes.h b/cpp/src/db/MetaTypes.h index b2fe7833236df885e28d16e4c7d1ac39f7298738..64f653611fe387cbf7232609cc0735c7d683ba7d 100644 --- a/cpp/src/db/MetaTypes.h +++ b/cpp/src/db/MetaTypes.h @@ -5,6 +5,8 @@ ******************************************************************************/ #pragma once +#include "ExecutionEngine.h" + #include #include #include @@ -19,12 +21,14 @@ const DateT EmptyDate = -1; typedef std::vector DatesT; struct TableSchema { - size_t id; - std::string table_id; - size_t files_cnt = 0; - uint16_t dimension; - std::string location; - long created_on; + size_t id_; + std::string table_id_; + size_t files_cnt_ = 0; + uint16_t dimension_; + std::string location_; + long created_on_; + int engine_type_ = (int)EngineType::FAISS_IDMAP; + bool store_raw_data_ = false; }; // TableSchema struct TableFileSchema { @@ -36,16 +40,17 @@ struct TableFileSchema { TO_DELETE, } FILE_TYPE; - size_t id; - std::string table_id; - std::string file_id; - int file_type = NEW; - size_t size; - DateT date = EmptyDate; - uint16_t dimension; - std::string location; - long updated_time; - long created_on; + size_t id_; + std::string table_id_; + int engine_type_ = (int)EngineType::FAISS_IDMAP; + std::string file_id_; + int file_type_ = NEW; + size_t size_; + DateT date_ = EmptyDate; + uint16_t dimension_; + std::string location_; + long updated_time_; + long created_on_; }; // TableFileSchema typedef std::vector TableFilesSchema; diff --git a/cpp/src/db/Traits.cpp b/cpp/src/db/Traits.cpp deleted file mode 100644 index a30b9fc036cfb4c7e02ee6e9b79799577d07ec5d..0000000000000000000000000000000000000000 --- a/cpp/src/db/Traits.cpp +++ /dev/null @@ -1,20 +0,0 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ -#include "Traits.h" - -namespace zilliz { -namespace vecwise { -namespace engine { - -const char* IVFIndexTrait::BuildIndexType = "IVF"; -const char* IVFIndexTrait::RawIndexType = "IDMap,Flat"; - -const char* IDMapIndexTrait::BuildIndexType = "IDMap"; -const char* IDMapIndexTrait::RawIndexType = "IDMap,Flat"; - -} // namespace engine -} // namespace vecwise -} // namespace zilliz diff --git a/cpp/src/db/Types.h b/cpp/src/db/Types.h index f9a432fd94e3ee1decb50b7999b2ca0cec61f5bb..73ecc81fa8da14cb3b1c9e3846de2aaba90c844c 100644 --- a/cpp/src/db/Types.h +++ b/cpp/src/db/Types.h @@ -15,7 +15,7 @@ typedef long IDNumber; typedef IDNumber* IDNumberPtr; typedef std::vector IDNumbers; -typedef std::vector QueryResult; +typedef std::vector> QueryResult; typedef std::vector QueryResults; diff --git a/cpp/src/db/scheduler/ScheduleStrategy.cpp b/cpp/src/db/scheduler/ScheduleStrategy.cpp index 2b344b33ef59edd8173c4c497b6ee0e42b3be2bb..1e8b5415b8abf5f3b23cccac27342c486d5e214c 100644 --- a/cpp/src/db/scheduler/ScheduleStrategy.cpp +++ b/cpp/src/db/scheduler/ScheduleStrategy.cpp @@ -24,21 +24,21 @@ public: SearchContext::Id2IndexMap index_files = search_context->GetIndexMap(); //some index loader alread exists for(auto& loader : loader_list) { - if(index_files.find(loader->file_->id) != index_files.end()){ + if(index_files.find(loader->file_->id_) != index_files.end()){ SERVER_LOG_INFO << "Append SearchContext to exist IndexLoaderContext"; - index_files.erase(loader->file_->id); + index_files.erase(loader->file_->id_); loader->search_contexts_.push_back(search_context); } } //index_files still contains some index files, create new loader for(auto& pair : index_files) { - SERVER_LOG_INFO << "Create new IndexLoaderContext for: " << pair.second->location; + SERVER_LOG_INFO << "Create new IndexLoaderContext for: " << pair.second->location_; IndexLoaderContextPtr new_loader = std::make_shared(); new_loader->search_contexts_.push_back(search_context); new_loader->file_ = pair.second; - auto index = zilliz::vecwise::cache::CpuCacheMgr::GetInstance()->GetIndex(pair.second->location); + auto index = zilliz::vecwise::cache::CpuCacheMgr::GetInstance()->GetIndex(pair.second->location_); if(index != nullptr) { //if the index file has been in memory, increase its priority loader_list.push_front(new_loader); diff --git a/cpp/src/db/scheduler/SearchContext.cpp b/cpp/src/db/scheduler/SearchContext.cpp index 7cde55f31ad367484c88e29766e34f47bffcf58e..5ce8d37fbe74cec07e43e0bfe3e67eb5a770a1ca 100644 --- a/cpp/src/db/scheduler/SearchContext.cpp +++ b/cpp/src/db/scheduler/SearchContext.cpp @@ -26,13 +26,13 @@ SearchContext::SearchContext(uint64_t topk, uint64_t nq, const float* vectors) bool SearchContext::AddIndexFile(TableFileSchemaPtr& index_file) { std::unique_lock lock(mtx_); - if(index_file == nullptr || map_index_files_.find(index_file->id) != map_index_files_.end()) { + if(index_file == nullptr || map_index_files_.find(index_file->id_) != map_index_files_.end()) { return false; } - SERVER_LOG_INFO << "SearchContext " << identity_ << " add index file: " << index_file->id; + SERVER_LOG_INFO << "SearchContext " << identity_ << " add index file: " << index_file->id_; - map_index_files_[index_file->id] = index_file; + map_index_files_[index_file->id_] = index_file; return true; } diff --git a/cpp/src/db/scheduler/SearchContext.h b/cpp/src/db/scheduler/SearchContext.h index ae7327fd6829eee012b123296521044f0957f94b..b212ea34d99d15fed7ea5d0dd8e2134c61372d26 100644 --- a/cpp/src/db/scheduler/SearchContext.h +++ b/cpp/src/db/scheduler/SearchContext.h @@ -31,8 +31,8 @@ public: using Id2IndexMap = std::unordered_map; const Id2IndexMap& GetIndexMap() const { return map_index_files_; } - using Score2IdMap = std::map; - using ResultSet = std::vector; + using Id2ScoreMap = std::vector>; + using ResultSet = std::vector; const ResultSet& GetResult() const { return result_; } ResultSet& GetResult() { return result_; } diff --git a/cpp/src/db/scheduler/SearchScheduler.cpp b/cpp/src/db/scheduler/SearchScheduler.cpp index 4d3a12fdf81b670de21c4fa0640194b721991668..c18f95d04d218067c1b82a16081227dc42b06460 100644 --- a/cpp/src/db/scheduler/SearchScheduler.cpp +++ b/cpp/src/db/scheduler/SearchScheduler.cpp @@ -10,11 +10,50 @@ #include "utils/Log.h" #include "utils/TimeRecorder.h" #include "metrics/Metrics.h" +#include "db/EngineFactory.h" namespace zilliz { namespace vecwise { namespace engine { +namespace { +void CollectFileMetrics(int file_type, size_t file_size) { + switch(file_type) { + case meta::TableFileSchema::RAW: + case meta::TableFileSchema::TO_INDEX: { + server::Metrics::GetInstance().RawFileSizeHistogramObserve(file_size); + server::Metrics::GetInstance().RawFileSizeTotalIncrement(file_size); + server::Metrics::GetInstance().RawFileSizeGaugeSet(file_size); + break; + } + default: { + server::Metrics::GetInstance().IndexFileSizeHistogramObserve(file_size); + server::Metrics::GetInstance().IndexFileSizeTotalIncrement(file_size); + server::Metrics::GetInstance().IndexFileSizeGaugeSet(file_size); + break; + } + } +} + +void CollectDurationMetrics(int index_type, double total_time) { + switch(index_type) { + case meta::TableFileSchema::RAW: { + server::Metrics::GetInstance().SearchRawDataDurationSecondsHistogramObserve(total_time); + break; + } + case meta::TableFileSchema::TO_INDEX: { + server::Metrics::GetInstance().SearchRawDataDurationSecondsHistogramObserve(total_time); + break; + } + default: { + server::Metrics::GetInstance().SearchIndexDataDurationSecondsHistogramObserve(total_time); + break; + } + } +} + +} + SearchScheduler::SearchScheduler() : thread_pool_(2), stopped_(true) { @@ -75,45 +114,27 @@ SearchScheduler::IndexLoadWorker() { break;//exit } - SERVER_LOG_INFO << "Loading index(" << context->file_->id << ") from location: " << context->file_->location; + SERVER_LOG_INFO << "Loading index(" << context->file_->id_ << ") from location: " << context->file_->location_; server::TimeRecorder rc("Load index"); - //load index - IndexEnginePtr index_ptr = std::make_shared(context->file_->dimension, context->file_->location); + //step 1: load index + ExecutionEnginePtr index_ptr = EngineFactory::Build(context->file_->dimension_, + context->file_->location_, + (EngineType)context->file_->engine_type_); index_ptr->Load(); rc.Record("load index file to memory"); size_t file_size = index_ptr->PhysicalSize(); - LOG(DEBUG) << "Index file type " << context->file_->file_type << " Of Size: " + LOG(DEBUG) << "Index file type " << context->file_->file_type_ << " Of Size: " << file_size/(1024*1024) << " M"; - //metric - switch(context->file_->file_type) { - case meta::TableFileSchema::RAW: { - server::Metrics::GetInstance().RawFileSizeHistogramObserve(file_size); - server::Metrics::GetInstance().RawFileSizeTotalIncrement(file_size); - server::Metrics::GetInstance().RawFileSizeGaugeSet(file_size); - break; - } - case meta::TableFileSchema::TO_INDEX: { - server::Metrics::GetInstance().RawFileSizeHistogramObserve(file_size); - server::Metrics::GetInstance().RawFileSizeTotalIncrement(file_size); - server::Metrics::GetInstance().RawFileSizeGaugeSet(file_size); - break; - } - default: { - server::Metrics::GetInstance().IndexFileSizeHistogramObserve(file_size); - server::Metrics::GetInstance().IndexFileSizeTotalIncrement(file_size); - server::Metrics::GetInstance().IndexFileSizeGaugeSet(file_size); - break; - } - } + CollectFileMetrics(context->file_->file_type_, file_size); - //put search task to another queue - SearchTaskPtr task_ptr = std::make_shared(); - task_ptr->index_id_ = context->file_->id; - task_ptr->index_type_ = context->file_->file_type; + //step 2: put search task into another queue + SearchTaskPtr task_ptr = std::make_shared(); + task_ptr->index_id_ = context->file_->id_; + task_ptr->index_type_ = context->file_->file_type_; task_ptr->index_engine_ = index_ptr; task_ptr->search_contexts_.swap(context->search_contexts_); search_queue.Put(task_ptr); @@ -140,20 +161,7 @@ SearchScheduler::SearchWorker() { task_ptr->DoSearch(); auto end_time = METRICS_NOW_TIME; auto total_time = METRICS_MICROSECONDS(start_time, end_time); - switch(task_ptr->index_type_) { - case meta::TableFileSchema::RAW: { - server::Metrics::GetInstance().SearchRawDataDurationSecondsHistogramObserve(total_time); - break; - } - case meta::TableFileSchema::TO_INDEX: { - server::Metrics::GetInstance().SearchRawDataDurationSecondsHistogramObserve(total_time); - break; - } - default: { - server::Metrics::GetInstance().SearchIndexDataDurationSecondsHistogramObserve(total_time); - break; - } - } + CollectDurationMetrics(task_ptr->index_type_, total_time); } return true; diff --git a/cpp/src/db/scheduler/SearchTaskQueue.inl b/cpp/src/db/scheduler/SearchTaskQueue.cpp similarity index 56% rename from cpp/src/db/scheduler/SearchTaskQueue.inl rename to cpp/src/db/scheduler/SearchTaskQueue.cpp index 7816ba8878ce05dbd85efa63582acac07088d4a1..38db5fd7a7cf675e9bd4496f2e60c14c05f88748 100644 --- a/cpp/src/db/scheduler/SearchTaskQueue.inl +++ b/cpp/src/db/scheduler/SearchTaskQueue.cpp @@ -3,8 +3,6 @@ * Unauthorized copying of this file, via any medium is strictly prohibited. * Proprietary and confidential. ******************************************************************************/ -#pragma once - #include "SearchTaskQueue.h" #include "utils/Log.h" #include "utils/TimeRecorder.h" @@ -21,12 +19,29 @@ void ClusterResult(const std::vector &output_ids, SearchContext::ResultSet &result_set) { result_set.clear(); for (auto i = 0; i < nq; i++) { - SearchContext::Score2IdMap score2id; + SearchContext::Id2ScoreMap id_score; for (auto k = 0; k < topk; k++) { uint64_t index = i * nq + k; - score2id.insert(std::make_pair(output_distence[index], output_ids[index])); + id_score.push_back(std::make_pair(output_ids[index], output_distence[index])); } - result_set.emplace_back(score2id); + result_set.emplace_back(id_score); + } +} + +void MergeResult(SearchContext::Id2ScoreMap &score_src, + SearchContext::Id2ScoreMap &score_target, + uint64_t topk) { + for (auto& pair_src : score_src) { + for (auto iter = score_target.begin(); iter != score_target.end(); ++iter) { + if(pair_src.second > iter->second) { + score_target.insert(iter, pair_src); + } + } + } + + //remove unused items + while (score_target.size() > topk) { + score_target.pop_back(); } } @@ -44,18 +59,39 @@ void TopkResult(SearchContext::ResultSet &result_src, } for (size_t i = 0; i < result_src.size(); i++) { - SearchContext::Score2IdMap &score2id_src = result_src[i]; - SearchContext::Score2IdMap &score2id_target = result_target[i]; - for (auto iter = score2id_src.begin(); iter != score2id_src.end(); ++iter) { - score2id_target.insert(std::make_pair(iter->first, iter->second)); + SearchContext::Id2ScoreMap &score_src = result_src[i]; + SearchContext::Id2ScoreMap &score_target = result_target[i]; + MergeResult(score_src, score_target, topk); + } +} + +void CalcScore(uint64_t vector_count, + const float *vectors_data, + uint64_t dimension, + const SearchContext::ResultSet &result_src, + SearchContext::ResultSet &result_target) { + result_target.clear(); + if(result_src.empty()){ + return; + } + + int vec_index = 0; + for(auto& result : result_src) { + const float * vec_data = vectors_data + vec_index*dimension; + double vec_len = 0; + for(uint64_t i = 0; i < dimension; i++) { + vec_len += vec_data[i]*vec_data[i]; } + vec_index++; - //remove unused items - while (score2id_target.size() > topk) { - score2id_target.erase(score2id_target.rbegin()->first); + SearchContext::Id2ScoreMap score_array; + for(auto& pair : result) { + score_array.push_back(std::make_pair(pair.first, (1 - pair.second/vec_len)*100.0)); } + result_target.emplace_back(score_array); } } + } @@ -70,8 +106,7 @@ SearchTaskQueue::GetInstance() { return instance; } -template -bool SearchTask::DoSearch() { +bool SearchTask::DoSearch() { if(index_engine_ == nullptr) { return false; } @@ -81,10 +116,12 @@ bool SearchTask::DoSearch() { std::vector output_ids; std::vector output_distence; for(auto& context : search_contexts_) { + //step 1: allocate memory auto inner_k = index_engine_->Count() < context->topk() ? index_engine_->Count() : context->topk(); output_ids.resize(inner_k*context->nq()); output_distence.resize(inner_k*context->nq()); + //step 2: search try { index_engine_->Search(context->nq(), context->vectors(), inner_k, output_distence.data(), output_ids.data()); @@ -96,11 +133,21 @@ bool SearchTask::DoSearch() { rc.Record("do search"); + //step 3: cluster result SearchContext::ResultSet result_set; ClusterResult(output_ids, output_distence, context->nq(), inner_k, result_set); rc.Record("cluster result"); + + //step 4: pick up topk result TopkResult(result_set, inner_k, context->GetResult()); rc.Record("reduce topk"); + + //step 5: calculate score between 0 ~ 100 + CalcScore(context->nq(), context->vectors(), index_engine_->Dimension(), context->GetResult(), result_set); + context->GetResult().swap(result_set); + rc.Record("calculate score"); + + //step 6: notify to send result to client context->IndexSearchDone(index_id_); } diff --git a/cpp/src/db/scheduler/SearchTaskQueue.h b/cpp/src/db/scheduler/SearchTaskQueue.h index 3b58294811e1de0dc8bd5e9247872373940a9cb1..bd8e9d7f245859d16d9720ce92ed29b772f48349 100644 --- a/cpp/src/db/scheduler/SearchTaskQueue.h +++ b/cpp/src/db/scheduler/SearchTaskQueue.h @@ -7,8 +7,7 @@ #include "SearchContext.h" #include "utils/BlockingQueue.h" -#include "../FaissExecutionEngine.h" -#include "../Traits.h" +#include "db/ExecutionEngine.h" #include @@ -16,16 +15,6 @@ namespace zilliz { namespace vecwise { namespace engine { -#ifdef GPU_VERSION -using IndexTraitClass = IVFIndexTrait; -#else -using IndexTraitClass = IDMapIndexTrait; -#endif - -using IndexClass = FaissExecutionEngine; -using IndexEnginePtr = std::shared_ptr; - -template class SearchTask { public: bool DoSearch(); @@ -33,12 +22,11 @@ public: public: size_t index_id_ = 0; int index_type_ = 0; //for metrics - IndexEnginePtr index_engine_; + ExecutionEnginePtr index_engine_; std::vector search_contexts_; }; -using SearchTaskClass = SearchTask; -using SearchTaskPtr = std::shared_ptr; +using SearchTaskPtr = std::shared_ptr; class SearchTaskQueue : public server::BlockingQueue { private: @@ -58,6 +46,4 @@ private: } } -} - -#include "SearchTaskQueue.inl" \ No newline at end of file +} \ No newline at end of file diff --git a/cpp/src/main.cpp b/cpp/src/main.cpp index 08ecb8c194342dc154a93dd48ce16561edb458b0..32f7c515e34c7ada899c8936efcba9174758fb4f 100644 --- a/cpp/src/main.cpp +++ b/cpp/src/main.cpp @@ -4,6 +4,7 @@ // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// #include "server/Server.h" +#include "version.h" #include #include @@ -25,8 +26,8 @@ using namespace zilliz::vecwise; int main(int argc, char *argv[]) { - printf("Vecwise engine server start...\n"); -// zilliz::lib::gpu::InitMemoryAllocator(); + printf("Megasearch %s version: v%s built at %s\n", BUILD_TYPE, MEGASEARCH_VERSION, BUILD_TIME); + printf("Megasearch server start...\n"); signal(SIGINT, server::SignalUtil::HandleSignal); signal(SIGSEGV, server::SignalUtil::HandleSignal); diff --git a/cpp/src/sdk/examples/simple/src/ClientTest.cpp b/cpp/src/sdk/examples/simple/src/ClientTest.cpp index d094e272f1d890a708926916e376638f9c9ae634..3eccb6bfa6f70c0336570ce5b3225db7390713da 100644 --- a/cpp/src/sdk/examples/simple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/simple/src/ClientTest.cpp @@ -16,9 +16,6 @@ namespace { std::string GetTableName(); static const std::string TABLE_NAME = GetTableName(); - static const std::string VECTOR_COLUMN_NAME = "face_vector"; - static const std::string ID_COLUMN_NAME = "aid"; - static const std::string CITY_COLUMN_NAME = "city"; static constexpr int64_t TABLE_DIMENSION = 512; static constexpr int64_t TOTAL_ROW_COUNT = 100000; static constexpr int64_t TOP_K = 10; @@ -29,9 +26,9 @@ namespace { void PrintTableSchema(const megasearch::TableSchema& tb_schema) { BLOCK_SPLITER std::cout << "Table name: " << tb_schema.table_name << std::endl; - std::cout << "Table vectors: " << tb_schema.vector_column_array.size() << std::endl; - std::cout << "Table attributes: " << tb_schema.attribute_column_array.size() << std::endl; - std::cout << "Table partitions: " << tb_schema.partition_column_name_array.size() << std::endl; + std::cout << "Table index type: " << (int)tb_schema.index_type << std::endl; + std::cout << "Table dimension: " << tb_schema.dimension << std::endl; + std::cout << "Table store raw data: " << tb_schema.store_raw_vector << std::endl; BLOCK_SPLITER } @@ -58,9 +55,6 @@ namespace { << " search result:" << std::endl; for(auto& item : result.query_result_arrays) { std::cout << "\t" << std::to_string(item.id) << "\tscore:" << std::to_string(item.score); - for(auto& attribute : item.column_map) { - std::cout << "\t" << attribute.first << ":" << attribute.second; - } std::cout << std::endl; } } @@ -88,73 +82,30 @@ namespace { TableSchema BuildTableSchema() { TableSchema tb_schema; - VectorColumn col1; - col1.name = VECTOR_COLUMN_NAME; - col1.dimension = TABLE_DIMENSION; - col1.store_raw_vector = true; - tb_schema.vector_column_array.emplace_back(col1); - - Column col2 = {ColumnType::int8, ID_COLUMN_NAME}; - tb_schema.attribute_column_array.emplace_back(col2); - - Column col3 = {ColumnType::int16, CITY_COLUMN_NAME}; - tb_schema.attribute_column_array.emplace_back(col3); - tb_schema.table_name = TABLE_NAME; + tb_schema.index_type = IndexType::gpu_ivfflat; + tb_schema.dimension = TABLE_DIMENSION; + tb_schema.store_raw_vector = true; return tb_schema; } void BuildVectors(int64_t from, int64_t to, - std::vector* vector_record_array, - std::vector* query_record_array) { + std::vector& vector_record_array) { if(to <= from){ return; } - if(vector_record_array) { - vector_record_array->clear(); - } - if(query_record_array) { - query_record_array->clear(); - } - - static const std::map CITY_MAP = { - {0, "Beijing"}, - {1, "Shanhai"}, - {2, "Hangzhou"}, - {3, "Guangzhou"}, - {4, "Shenzheng"}, - {5, "Wuhan"}, - {6, "Chengdu"}, - {7, "Chongqin"}, - {8, "Tianjing"}, - {9, "Hongkong"}, - }; + vector_record_array.clear(); for (int64_t k = from; k < to; k++) { - - std::vector f_p; - f_p.resize(TABLE_DIMENSION); + RowRecord record; + record.data.resize(TABLE_DIMENSION); for(int64_t i = 0; i < TABLE_DIMENSION; i++) { - f_p[i] = (float)(i + k); - } - - if(vector_record_array) { - RowRecord record; - record.vector_map.insert(std::make_pair(VECTOR_COLUMN_NAME, f_p)); - record.attribute_map[ID_COLUMN_NAME] = std::to_string(k); - record.attribute_map[CITY_COLUMN_NAME] = CITY_MAP.at(k%CITY_MAP.size()); - vector_record_array->emplace_back(record); + record.data[i] = (float)(i + k); } - if(query_record_array) { - QueryRecord record; - record.vector_map.insert(std::make_pair(VECTOR_COLUMN_NAME, f_p)); - record.selected_column_array.push_back(ID_COLUMN_NAME); - record.selected_column_array.push_back(CITY_COLUMN_NAME); - query_record_array->emplace_back(record); - } + vector_record_array.emplace_back(record); } } } @@ -205,7 +156,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { {//add vectors std::vector record_array; - BuildVectors(0, TOTAL_ROW_COUNT, &record_array, nullptr); + BuildVectors(0, TOTAL_ROW_COUNT, record_array); std::vector record_ids; Status stat = conn->AddVector(TABLE_NAME, record_array, record_ids); std::cout << "AddVector function call status: " << stat.ToString() << std::endl; @@ -215,11 +166,12 @@ ClientTest::Test(const std::string& address, const std::string& port) { {//search vectors std::cout << "Waiting data persist. Sleep 10 seconds ..." << std::endl; sleep(10); - std::vector record_array; - BuildVectors(SEARCH_TARGET, SEARCH_TARGET + 10, nullptr, &record_array); + std::vector record_array; + BuildVectors(SEARCH_TARGET, SEARCH_TARGET + 10, record_array); + std::vector query_range_array; std::vector topk_query_result_array; - Status stat = conn->SearchVector(TABLE_NAME, record_array, topk_query_result_array, TOP_K); + Status stat = conn->SearchVector(TABLE_NAME, record_array, query_range_array, TOP_K, topk_query_result_array); std::cout << "SearchVector function call status: " << stat.ToString() << std::endl; PrintSearchResult(topk_query_result_array); } diff --git a/cpp/src/sdk/include/MegaSearch.h b/cpp/src/sdk/include/MegaSearch.h index d00692761462f5077fcb85cf43fed9e1bec48f54..7cbfdb1ed9da8e9fc59f6d420b01741dd296f92f 100644 --- a/cpp/src/sdk/include/MegaSearch.h +++ b/cpp/src/sdk/include/MegaSearch.h @@ -4,7 +4,6 @@ #include #include -#include #include /** \brief MegaSearch SDK namespace @@ -12,129 +11,70 @@ namespace megasearch { -/** - * @brief Column Type - */ -enum class ColumnType { - invalid, - int8, - int16, - int32, - int64, - float32, - float64, - date, - vector -}; - /** * @brief Index Type */ enum class IndexType { - raw, - ivfflat + invalid = 0, + cpu_idmap, + gpu_ivfflat, }; /** * @brief Connect API parameter */ struct ConnectParam { - std::string ip_address; ///< Server IP address - std::string port; ///< Server PORT -}; - -/** - * @brief Table column description - */ -struct Column { - ColumnType type = ColumnType::invalid; ///< Column Type: enum ColumnType - std::string name; ///< Column name -}; - -/** - * @brief Table vector column description - */ -struct VectorColumn : public Column { - VectorColumn() { type = ColumnType::vector; } - int64_t dimension = 0; ///< Vector dimension - IndexType index_type = IndexType::raw; ///< Index type - bool store_raw_vector = false; ///< Is vector self stored in the table + std::string ip_address; ///< Server IP address + std::string port; ///< Server PORT }; /** * @brief Table Schema */ struct TableSchema { - std::string table_name; ///< Table name - std::vector vector_column_array; ///< Vector column description - std::vector attribute_column_array; ///< Columns description - std::vector partition_column_name_array; ///< Partition column name + std::string table_name; ///< Table name + IndexType index_type = IndexType::invalid; ///< Index type + int64_t dimension = 0; ///< Vector dimension, must be a positive value + bool store_raw_vector = false; ///< Is vector raw data stored in the table }; /** * @brief Range information + * for DATE partition, the format is like: 'year-month-day' */ struct Range { - std::string start_value; ///< Range start - std::string end_value; ///< Range stop -}; - -/** - * @brief Create table partition parameters - */ -struct CreateTablePartitionParam { - std::string table_name; ///< Table name, vector/float32/float64 type column is not allowed for partition - std::string partition_name; ///< Partition name, created partition name - std::map range_map; ///< Column name to PartitionRange map -}; - - -/** - * @brief Delete table partition parameters - */ -struct DeleteTablePartitionParam { - std::string table_name; ///< Table name - std::vector partition_name_array; ///< Partition name array + std::string start_value; ///< Range start + std::string end_value; ///< Range stop }; /** * @brief Record inserted */ struct RowRecord { - std::map> vector_map; ///< Vector columns - std::map attribute_map; ///< Other attribute columns -}; - -/** - * @brief Query record - */ -struct QueryRecord { - std::map> vector_map; ///< Query vectors - std::vector selected_column_array; ///< Output column array - std::map> partition_filter_column_map; ///< Range used to select partitions + std::vector data; ///< Vector raw data }; /** * @brief Query result */ struct QueryResult { - int64_t id; ///< Output result - double score; ///< Vector similarity score: 0 ~ 100 - std::map column_map; ///< Other column + int64_t id; ///< Output result + double score; ///< Vector similarity score: 0 ~ 100 }; /** * @brief TopK query result */ struct TopKQueryResult { - std::vector query_result_arrays; ///< TopK query result + std::vector query_result_arrays; ///< TopK query result }; + /** * @brief SDK main class */ class Connection { - public: +public: /** * @brief CreateConnection @@ -228,30 +168,6 @@ class Connection { virtual Status DeleteTable(const std::string &table_name) = 0; - /** - * @brief Create table partition - * - * This method is used to create table partition. - * - * @param param, use to provide partition information to be created. - * - * @return Indicate if table partition is created successfully. - */ - virtual Status CreateTablePartition(const CreateTablePartitionParam ¶m) = 0; - - - /** - * @brief Delete table partition - * - * This method is used to delete table partition. - * - * @param param, use to provide partition information to be deleted. - * - * @return Indicate if table partition is delete successfully. - */ - virtual Status DeleteTablePartition(const DeleteTablePartitionParam ¶m) = 0; - - /** * @brief Add vector to table * @@ -264,8 +180,8 @@ class Connection { * @return Indicate if vector array are inserted successfully */ virtual Status AddVector(const std::string &table_name, - const std::vector &record_array, - std::vector &id_array) = 0; + const std::vector &record_array, + std::vector &id_array) = 0; /** @@ -275,15 +191,17 @@ class Connection { * * @param table_name, table_name is queried. * @param query_record_array, all vector are going to be queried. - * @param topk_query_result_array, result array. + * @param query_range_array, time ranges, if not specified, will search in whole table * @param topk, how many similarity vectors will be searched. + * @param topk_query_result_array, result array. * * @return Indicate if query is successful. */ virtual Status SearchVector(const std::string &table_name, - const std::vector &query_record_array, - std::vector &topk_query_result_array, - int64_t topk) = 0; + const std::vector &query_record_array, + const std::vector &query_range_array, + int64_t topk, + std::vector &topk_query_result_array) = 0; /** * @brief Show table description @@ -297,6 +215,18 @@ class Connection { */ virtual Status DescribeTable(const std::string &table_name, TableSchema &table_schema) = 0; + /** + * @brief Get table row count + * + * This method is used to get table row count. + * + * @param table_name, table's name. + * @param row_count, table total row count. + * + * @return Indicate if this operation is successful. + */ + virtual Status GetTableRowCount(const std::string &table_name, int64_t &row_count) = 0; + /** * @brief Show all tables in database * diff --git a/cpp/src/sdk/include/Status.h b/cpp/src/sdk/include/Status.h index f9b01685a0d95b8f0a7e114bc157fe098fde153e..3a93adfa6285873cc04988e641d612512826ef36 100644 --- a/cpp/src/sdk/include/Status.h +++ b/cpp/src/sdk/include/Status.h @@ -12,10 +12,13 @@ namespace megasearch { */ enum class StatusCode { OK = 0, - Invalid = 1, - UnknownError = 2, - NotSupported = 3, - NotConnected = 4 + // system error section + UnknownError = 1, + NotSupported, + NotConnected, + + // function error section + InvalidAgument = 1000, }; /** @@ -171,7 +174,7 @@ class Status { */ template static Status Invalid(Args &&... args) { - return Status(StatusCode::Invalid, + return Status(StatusCode::InvalidAgument, MessageBuilder(std::forward(args)...)); } @@ -221,7 +224,7 @@ class Status { * @return, if the status indicates invalid. * */ - bool IsInvalid() const { return code() == StatusCode::Invalid; } + bool IsInvalid() const { return code() == StatusCode::InvalidAgument; } /** * @brief IsUnknownError diff --git a/cpp/src/sdk/src/client/ClientProxy.cpp b/cpp/src/sdk/src/client/ClientProxy.cpp index 1e2348474203f028b1824a5949d1866d4a02d0b9..8805329c87cde8211530ae331a62463b861bc866 100644 --- a/cpp/src/sdk/src/client/ClientProxy.cpp +++ b/cpp/src/sdk/src/client/ClientProxy.cpp @@ -89,27 +89,9 @@ ClientProxy::CreateTable(const TableSchema ¶m) { try { thrift::TableSchema schema; schema.__set_table_name(param.table_name); - - std::vector vector_column_array; - for(auto& column : param.vector_column_array) { - thrift::VectorColumn col; - col.__set_dimension(column.dimension); - col.__set_index_type(ConvertUtil::IndexType2Str(column.index_type)); - col.__set_store_raw_vector(column.store_raw_vector); - vector_column_array.emplace_back(col); - } - schema.__set_vector_column_array(vector_column_array); - - std::vector attribute_column_array; - for(auto& column : param.attribute_column_array) { - thrift::Column col; - col.__set_name(col.name); - col.__set_type(col.type); - attribute_column_array.emplace_back(col); - } - schema.__set_attribute_column_array(attribute_column_array); - - schema.__set_partition_column_name_array(param.partition_column_name_array); + schema.__set_index_type((int)param.index_type); + schema.__set_dimension(param.dimension); + schema.__set_store_raw_vector(param.store_raw_vector); ClientPtr()->interface()->CreateTable(schema); @@ -120,55 +102,6 @@ ClientProxy::CreateTable(const TableSchema ¶m) { return Status::OK(); } -Status -ClientProxy::CreateTablePartition(const CreateTablePartitionParam ¶m) { - if(!IsConnected()) { - return Status(StatusCode::NotConnected, "not connected to server"); - } - - try { - thrift::CreateTablePartitionParam partition_param; - partition_param.__set_table_name(param.table_name); - partition_param.__set_partition_name(param.partition_name); - - std::map range_map; - for(auto& pair : param.range_map) { - thrift::Range range; - range.__set_start_value(pair.second.start_value); - range.__set_end_value(pair.second.end_value); - range_map.insert(std::make_pair(pair.first, range)); - } - partition_param.__set_range_map(range_map); - - ClientPtr()->interface()->CreateTablePartition(partition_param); - - } catch ( std::exception& ex) { - return Status(StatusCode::UnknownError, "failed to create table partition: " + std::string(ex.what())); - } - - return Status::OK(); -} - -Status -ClientProxy::DeleteTablePartition(const DeleteTablePartitionParam ¶m) { - if(!IsConnected()) { - return Status(StatusCode::NotConnected, "not connected to server"); - } - - try { - thrift::DeleteTablePartitionParam partition_param; - partition_param.__set_table_name(param.table_name); - partition_param.__set_partition_name_array(param.partition_name_array); - - ClientPtr()->interface()->DeleteTablePartition(partition_param); - - } catch ( std::exception& ex) { - return Status(StatusCode::UnknownError, "failed to delete table partition: " + std::string(ex.what())); - } - - return Status::OK(); -} - Status ClientProxy::DeleteTable(const std::string &table_name) { if(!IsConnected()) { @@ -197,17 +130,13 @@ ClientProxy::AddVector(const std::string &table_name, std::vector thrift_records; for(auto& record : record_array) { thrift::RowRecord thrift_record; - thrift_record.__set_attribute_map(record.attribute_map); - - for(auto& pair : record.vector_map) { - size_t dim = pair.second.size(); - std::string& thrift_vector = thrift_record.vector_map[pair.first]; - thrift_vector.resize(dim * sizeof(double)); - double *dbl = (double *) (const_cast(thrift_vector.data())); - for (size_t i = 0; i < dim; i++) { - dbl[i] = (double) (pair.second[i]); - } + + thrift_record.vector_data.resize(record.data.size() * sizeof(double)); + double *dbl = (double *) (const_cast(thrift_record.vector_data.data())); + for (size_t i = 0; i < record.data.size(); i++) { + dbl[i] = (double) (record.data[i]); } + thrift_records.emplace_back(thrift_record); } ClientPtr()->interface()->AddVector(id_array, table_name, thrift_records); @@ -221,33 +150,31 @@ ClientProxy::AddVector(const std::string &table_name, Status ClientProxy::SearchVector(const std::string &table_name, - const std::vector &query_record_array, - std::vector &topk_query_result_array, - int64_t topk) { + const std::vector &query_record_array, + const std::vector &query_range_array, + int64_t topk, + std::vector &topk_query_result_array) { if(!IsConnected()) { return Status(StatusCode::NotConnected, "not connected to server"); } try { - std::vector thrift_records; + std::vector thrift_records; for(auto& record : query_record_array) { - thrift::QueryRecord thrift_record; - thrift_record.__set_selected_column_array(record.selected_column_array); - - for(auto& pair : record.vector_map) { - size_t dim = pair.second.size(); - std::string& thrift_vector = thrift_record.vector_map[pair.first]; - thrift_vector.resize(dim * sizeof(double)); - double *dbl = (double *) (const_cast(thrift_vector.data())); - for (size_t i = 0; i < dim; i++) { - dbl[i] = (double) (pair.second[i]); - } + thrift::RowRecord thrift_record; + + thrift_record.vector_data.resize(record.data.size() * sizeof(double)); + double *dbl = (double *) (const_cast(thrift_record.vector_data.data())); + for (size_t i = 0; i < record.data.size(); i++) { + dbl[i] = (double) (record.data[i]); } + thrift_records.emplace_back(thrift_record); } + std::vector thrift_ranges; std::vector result_array; - ClientPtr()->interface()->SearchVector(result_array, table_name, thrift_records, topk); + ClientPtr()->interface()->SearchVector(result_array, table_name, thrift_records, thrift_ranges, topk); for(auto& thrift_topk_result : result_array) { TopKQueryResult result; @@ -255,7 +182,6 @@ ClientProxy::SearchVector(const std::string &table_name, for(auto& thrift_query_result : thrift_topk_result.query_result_arrays) { QueryResult query_result; query_result.id = thrift_query_result.id; - query_result.column_map = thrift_query_result.column_map; query_result.score = thrift_query_result.score; result.query_result_arrays.emplace_back(query_result); } @@ -281,27 +207,26 @@ ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_sch ClientPtr()->interface()->DescribeTable(thrift_schema, table_name); table_schema.table_name = thrift_schema.table_name; - table_schema.partition_column_name_array = thrift_schema.partition_column_name_array; + table_schema.index_type = (IndexType)thrift_schema.index_type; - for(auto& thrift_col : thrift_schema.attribute_column_array) { - Column col; - col.name = col.name; - col.type = col.type; - table_schema.attribute_column_array.emplace_back(col); - } + } catch ( std::exception& ex) { + return Status(StatusCode::UnknownError, "failed to describe table: " + std::string(ex.what())); + } - for(auto& thrift_col : thrift_schema.vector_column_array) { - VectorColumn col; - col.store_raw_vector = thrift_col.store_raw_vector; - col.index_type = ConvertUtil::Str2IndexType(thrift_col.index_type); - col.dimension = thrift_col.dimension; - col.name = thrift_col.base.name; - col.type = (ColumnType)thrift_col.base.type; - table_schema.vector_column_array.emplace_back(col); - } + return Status::OK(); +} + +Status +ClientProxy::GetTableRowCount(const std::string &table_name, int64_t &row_count) { + if(!IsConnected()) { + return Status(StatusCode::NotConnected, "not connected to server"); + } + + try { + row_count = ClientPtr()->interface()->GetTableRowCount(table_name); } catch ( std::exception& ex) { - return Status(StatusCode::UnknownError, "failed to describe table: " + std::string(ex.what())); + return Status(StatusCode::UnknownError, "failed to show tables: " + std::string(ex.what())); } return Status::OK(); diff --git a/cpp/src/sdk/src/client/ClientProxy.h b/cpp/src/sdk/src/client/ClientProxy.h index 5a96b915449ff8ae6fb3b10048859764a2911eef..71515739af09f0d15d31fc85e389573489021c1d 100644 --- a/cpp/src/sdk/src/client/ClientProxy.h +++ b/cpp/src/sdk/src/client/ClientProxy.h @@ -25,21 +25,20 @@ public: virtual Status DeleteTable(const std::string &table_name) override; - virtual Status CreateTablePartition(const CreateTablePartitionParam ¶m) override; - - virtual Status DeleteTablePartition(const DeleteTablePartitionParam ¶m) override; - virtual Status AddVector(const std::string &table_name, const std::vector &record_array, std::vector &id_array) override; virtual Status SearchVector(const std::string &table_name, - const std::vector &query_record_array, - std::vector &topk_query_result_array, - int64_t topk) override; + const std::vector &query_record_array, + const std::vector &query_range_array, + int64_t topk, + std::vector &topk_query_result_array) override; virtual Status DescribeTable(const std::string &table_name, TableSchema &table_schema) override; + virtual Status GetTableRowCount(const std::string &table_name, int64_t &row_count) override; + virtual Status ShowTables(std::vector &table_array) override; virtual std::string ClientVersion() const override; diff --git a/cpp/src/sdk/src/client/ThriftClient.cpp b/cpp/src/sdk/src/client/ThriftClient.cpp index 5ab000c200c5363493fa2f3ee8fce0182b11da76..179512387f28a5a13fa848433e0809416b60ba8f 100644 --- a/cpp/src/sdk/src/client/ThriftClient.cpp +++ b/cpp/src/sdk/src/client/ThriftClient.cpp @@ -58,7 +58,7 @@ ThriftClient::Connect(const std::string& address, int32_t port, const std::strin protocol_ptr.reset(new TCompactProtocol(transport_ptr)); } else { //CLIENT_LOG_ERROR << "Service protocol: " << protocol << " is not supported currently"; - return Status(StatusCode::Invalid, "unsupported protocol"); + return Status(StatusCode::InvalidAgument, "unsupported protocol"); } transport_ptr->open(); diff --git a/cpp/src/sdk/src/interface/ConnectionImpl.cpp b/cpp/src/sdk/src/interface/ConnectionImpl.cpp index 69612ad8b4748a9a667df41068fbd132a6efb0d1..053bff4d0595178a573d9408ec043a81998b2927 100644 --- a/cpp/src/sdk/src/interface/ConnectionImpl.cpp +++ b/cpp/src/sdk/src/interface/ConnectionImpl.cpp @@ -55,16 +55,6 @@ ConnectionImpl::CreateTable(const TableSchema ¶m) { return client_proxy_->CreateTable(param); } -Status -ConnectionImpl::CreateTablePartition(const CreateTablePartitionParam ¶m) { - return client_proxy_->CreateTablePartition(param); -} - -Status -ConnectionImpl::DeleteTablePartition(const DeleteTablePartitionParam ¶m) { - return client_proxy_->DeleteTablePartition(param); -} - Status ConnectionImpl::DeleteTable(const std::string &table_name) { return client_proxy_->DeleteTable(table_name); @@ -79,10 +69,11 @@ ConnectionImpl::AddVector(const std::string &table_name, Status ConnectionImpl::SearchVector(const std::string &table_name, - const std::vector &query_record_array, - std::vector &topk_query_result_array, - int64_t topk) { - return client_proxy_->SearchVector(table_name, query_record_array, topk_query_result_array, topk); + const std::vector &query_record_array, + const std::vector &query_range_array, + int64_t topk, + std::vector &topk_query_result_array) { + return client_proxy_->SearchVector(table_name, query_record_array, query_range_array, topk, topk_query_result_array); } Status @@ -90,6 +81,11 @@ ConnectionImpl::DescribeTable(const std::string &table_name, TableSchema &table_ return client_proxy_->DescribeTable(table_name, table_schema); } +Status +ConnectionImpl::GetTableRowCount(const std::string &table_name, int64_t &row_count) { + return client_proxy_->GetTableRowCount(table_name, row_count); +} + Status ConnectionImpl::ShowTables(std::vector &table_array) { return client_proxy_->ShowTables(table_array); diff --git a/cpp/src/sdk/src/interface/ConnectionImpl.h b/cpp/src/sdk/src/interface/ConnectionImpl.h index 0c67ea3f4259c53d99f7fd4a09197c8b53932693..a74703986c77ebd500cf986539c442d5017ef83d 100644 --- a/cpp/src/sdk/src/interface/ConnectionImpl.h +++ b/cpp/src/sdk/src/interface/ConnectionImpl.h @@ -27,21 +27,20 @@ public: virtual Status DeleteTable(const std::string &table_name) override; - virtual Status CreateTablePartition(const CreateTablePartitionParam ¶m) override; - - virtual Status DeleteTablePartition(const DeleteTablePartitionParam ¶m) override; - virtual Status AddVector(const std::string &table_name, const std::vector &record_array, std::vector &id_array) override; virtual Status SearchVector(const std::string &table_name, - const std::vector &query_record_array, - std::vector &topk_query_result_array, - int64_t topk) override; + const std::vector &query_record_array, + const std::vector &query_range_array, + int64_t topk, + std::vector &topk_query_result_array) override; virtual Status DescribeTable(const std::string &table_name, TableSchema &table_schema) override; + virtual Status GetTableRowCount(const std::string &table_name, int64_t &row_count) override; + virtual Status ShowTables(std::vector &table_array) override; virtual std::string ClientVersion() const override; diff --git a/cpp/src/sdk/src/interface/Status.cpp b/cpp/src/sdk/src/interface/Status.cpp index 6ac3b7ec9391038738b6d0b7d8117fce372ea0c5..1c47acfb4e00410df4f4e70e68b6d28c281a52cc 100644 --- a/cpp/src/sdk/src/interface/Status.cpp +++ b/cpp/src/sdk/src/interface/Status.cpp @@ -95,7 +95,7 @@ std::string Status::CodeAsString() const { switch (code()) { case StatusCode::OK: type = "OK"; break; - case StatusCode::Invalid: type = "Invalid"; + case StatusCode::InvalidAgument: type = "Invalid agument"; break; case StatusCode::UnknownError: type = "Unknown error"; break; diff --git a/cpp/src/sdk/src/util/ConvertUtil.cpp b/cpp/src/sdk/src/util/ConvertUtil.cpp index 577f5de446abdb37d1397e71e80dd30c00c48f7a..bf225074f48373822ceff9d87025cac5284afa24 100644 --- a/cpp/src/sdk/src/util/ConvertUtil.cpp +++ b/cpp/src/sdk/src/util/ConvertUtil.cpp @@ -15,13 +15,13 @@ static const std::string INDEX_IVFFLAT = "ivfflat"; std::string ConvertUtil::IndexType2Str(megasearch::IndexType index) { static const std::map s_index2str = { - {megasearch::IndexType::raw, INDEX_RAW}, - {megasearch::IndexType::ivfflat, INDEX_IVFFLAT} + {megasearch::IndexType::cpu_idmap, INDEX_RAW}, + {megasearch::IndexType::gpu_ivfflat, INDEX_IVFFLAT} }; const auto& iter = s_index2str.find(index); if(iter == s_index2str.end()) { - throw Exception(StatusCode::Invalid, "Invalid index type"); + throw Exception(StatusCode::InvalidAgument, "Invalid index type"); } return iter->second; @@ -29,13 +29,13 @@ std::string ConvertUtil::IndexType2Str(megasearch::IndexType index) { megasearch::IndexType ConvertUtil::Str2IndexType(const std::string& type) { static const std::map s_str2index = { - {INDEX_RAW, megasearch::IndexType::raw}, - {INDEX_IVFFLAT, megasearch::IndexType::ivfflat} + {INDEX_RAW, megasearch::IndexType::cpu_idmap}, + {INDEX_IVFFLAT, megasearch::IndexType::gpu_ivfflat} }; const auto& iter = s_str2index.find(type); if(iter == s_str2index.end()) { - throw Exception(StatusCode::Invalid, "Invalid index type"); + throw Exception(StatusCode::InvalidAgument, "Invalid index type"); } return iter->second; diff --git a/cpp/src/server/MegasearchHandler.cpp b/cpp/src/server/MegasearchHandler.cpp index 4e7c57560637c7d30b4ece0a59791719b4a3155b..af2a5a74378b500373865fc7123e5b9a74d05461 100644 --- a/cpp/src/server/MegasearchHandler.cpp +++ b/cpp/src/server/MegasearchHandler.cpp @@ -30,18 +30,6 @@ MegasearchServiceHandler::DeleteTable(const std::string &table_name) { MegasearchScheduler::ExecTask(task_ptr); } -void -MegasearchServiceHandler::CreateTablePartition(const thrift::CreateTablePartitionParam ¶m) { - BaseTaskPtr task_ptr = CreateTablePartitionTask::Create(param); - MegasearchScheduler::ExecTask(task_ptr); -} - -void -MegasearchServiceHandler::DeleteTablePartition(const thrift::DeleteTablePartitionParam ¶m) { - BaseTaskPtr task_ptr = DeleteTablePartitionTask::Create(param); - MegasearchScheduler::ExecTask(task_ptr); -} - void MegasearchServiceHandler::AddVector(std::vector &_return, const std::string &table_name, @@ -51,11 +39,12 @@ MegasearchServiceHandler::AddVector(std::vector &_return, } void -MegasearchServiceHandler::SearchVector(std::vector &_return, - const std::string &table_name, - const std::vector &query_record_array, - const int64_t topk) { - BaseTaskPtr task_ptr = SearchVectorTask::Create(table_name, query_record_array, topk, _return); +MegasearchServiceHandler::SearchVector(std::vector & _return, + const std::string& table_name, + const std::vector & query_record_array, + const std::vector & query_range_array, + const int64_t topk) { + BaseTaskPtr task_ptr = SearchVectorTask::Create(table_name, query_record_array, query_range_array, topk, _return); MegasearchScheduler::ExecTask(task_ptr); } @@ -65,6 +54,18 @@ MegasearchServiceHandler::DescribeTable(thrift::TableSchema &_return, const std: MegasearchScheduler::ExecTask(task_ptr); } +int64_t +MegasearchServiceHandler::GetTableRowCount(const std::string& table_name) { + int64_t row_count = 0; + { + BaseTaskPtr task_ptr = GetTableRowCountTask::Create(table_name, row_count); + MegasearchScheduler::ExecTask(task_ptr); + task_ptr->WaitToFinish(); + } + + return row_count; +} + void MegasearchServiceHandler::ShowTables(std::vector &_return) { BaseTaskPtr task_ptr = ShowTablesTask::Create(_return); diff --git a/cpp/src/server/MegasearchHandler.h b/cpp/src/server/MegasearchHandler.h index 10d90c14cfa106b1f3183e426e5202bd9e8091c7..6719f282af6ae59eb92e2ac5ddbc5c8607fe8119 100644 --- a/cpp/src/server/MegasearchHandler.h +++ b/cpp/src/server/MegasearchHandler.h @@ -19,15 +19,15 @@ public: MegasearchServiceHandler(); /** - * @brief Create table method - * - * This method is used to create table - * - * @param param, use to provide table information to be created. - * - * - * @param param - */ + * @brief Create table method + * + * This method is used to create table + * + * @param param, use to provide table information to be created. + * + * + * @param param + */ void CreateTable(const megasearch::thrift::TableSchema& param); /** @@ -42,30 +42,6 @@ public: */ void DeleteTable(const std::string& table_name); - /** - * @brief Create table partition - * - * This method is used to create table partition. - * - * @param param, use to provide partition information to be created. - * - * - * @param param - */ - void CreateTablePartition(const megasearch::thrift::CreateTablePartitionParam& param); - - /** - * @brief Delete table partition - * - * This method is used to delete table partition. - * - * @param param, use to provide partition information to be deleted. - * - * - * @param param - */ - void DeleteTablePartition(const megasearch::thrift::DeleteTablePartitionParam& param); - /** * @brief Add vector array to table * @@ -90,25 +66,28 @@ public: * * @param table_name, table_name is queried. * @param query_record_array, all vector are going to be queried. + * @param query_range_array, optional ranges for conditional search. If not specified, search whole table * @param topk, how many similarity vectors will be searched. * * @return query result array. * * @param table_name * @param query_record_array + * @param query_range_array * @param topk */ void SearchVector(std::vector & _return, const std::string& table_name, - const std::vector & query_record_array, + const std::vector & query_record_array, + const std::vector & query_range_array, const int64_t topk); /** - * @brief Show table information + * @brief Get table schema * - * This method is used to show table information. + * This method is used to get table schema. * - * @param table_name, which table is show. + * @param table_name, target table name. * * @return table schema * @@ -116,6 +95,19 @@ public: */ void DescribeTable(megasearch::thrift::TableSchema& _return, const std::string& table_name); + /** + * @brief Get table row count + * + * This method is used to get table row count. + * + * @param table_name, target table name. + * + * @return table row count + * + * @param table_name + */ + int64_t GetTableRowCount(const std::string& table_name); + /** * @brief List all tables in database * diff --git a/cpp/src/server/MegasearchScheduler.cpp b/cpp/src/server/MegasearchScheduler.cpp index c0a34b5432c0e22235fa468f4a9e25c40f3e4dd3..fb6c74d51e6115715e67e3460ff308e6c499f830 100644 --- a/cpp/src/server/MegasearchScheduler.cpp +++ b/cpp/src/server/MegasearchScheduler.cpp @@ -24,7 +24,7 @@ namespace { {SERVER_FILE_NOT_FOUND, thrift::ErrorCode::ILLEGAL_ARGUMENT}, {SERVER_NOT_IMPLEMENT, thrift::ErrorCode::ILLEGAL_ARGUMENT}, {SERVER_BLOCKING_QUEUE_EMPTY, thrift::ErrorCode::ILLEGAL_ARGUMENT}, - {SERVER_GROUP_NOT_EXIST, thrift::ErrorCode::TABLE_NOT_EXISTS}, + {SERVER_TABLE_NOT_EXIST, thrift::ErrorCode::TABLE_NOT_EXISTS}, {SERVER_INVALID_TIME_RANGE, thrift::ErrorCode::ILLEGAL_RANGE}, {SERVER_INVALID_VECTOR_DIMENSION, thrift::ErrorCode::ILLEGAL_DIMENSION}, }; @@ -40,7 +40,7 @@ namespace { {SERVER_FILE_NOT_FOUND, "file not found"}, {SERVER_NOT_IMPLEMENT, "not implemented"}, {SERVER_BLOCKING_QUEUE_EMPTY, "queue empty"}, - {SERVER_GROUP_NOT_EXIST, "group not exist"}, + {SERVER_TABLE_NOT_EXIST, "table not exist"}, {SERVER_INVALID_TIME_RANGE, "invalid time range"}, {SERVER_INVALID_VECTOR_DIMENSION, "invalid vector dimension"}, }; diff --git a/cpp/src/server/MegasearchTask.cpp b/cpp/src/server/MegasearchTask.cpp index 1ea2a235dc4f9f9678bc49d25442aa482e826fda..7c78b100468a3269400df7292af175180ac97202 100644 --- a/cpp/src/server/MegasearchTask.cpp +++ b/cpp/src/server/MegasearchTask.cpp @@ -5,15 +5,13 @@ ******************************************************************************/ #include "MegasearchTask.h" #include "ServerConfig.h" -#include "VecIdMapper.h" #include "utils/CommonUtil.h" #include "utils/Log.h" #include "utils/TimeRecorder.h" -#include "utils/ThreadPool.h" #include "db/DB.h" #include "db/Env.h" #include "db/Meta.h" - +#include "version.h" namespace zilliz { namespace vecwise { @@ -64,9 +62,18 @@ namespace { return db_wrapper.DB(); } - ThreadPool& GetThreadPool() { - static ThreadPool pool(6); - return pool; + engine::EngineType EngineType(int type) { + static std::map map_type = { + {0, engine::EngineType::INVALID}, + {1, engine::EngineType::FAISS_IDMAP}, + {2, engine::EngineType::FAISS_IVFFLAT}, + }; + + if(map_type.find(type) == map_type.end()) { + return engine::EngineType::INVALID; + } + + return map_type[type]; } } @@ -85,16 +92,20 @@ ServerError CreateTableTask::OnExecute() { TimeRecorder rc("CreateTableTask"); try { - if(schema_.vector_column_array.empty()) { + if(schema_.table_name.empty() || schema_.dimension == 0 || schema_.index_type == 0) { return SERVER_INVALID_ARGUMENT; } - IVecIdMapper::GetInstance()->AddGroup(schema_.table_name); + //step 1: construct table schema engine::meta::TableSchema table_info; - table_info.dimension = (uint16_t)schema_.vector_column_array[0].dimension; - table_info.table_id = schema_.table_name; + table_info.dimension_ = (uint16_t)schema_.dimension; + table_info.table_id_ = schema_.table_name; + table_info.engine_type_ = (int)EngineType(schema_.index_type); + table_info.store_raw_data_ = schema_.store_raw_vector; + + //step 2: create table engine::Status stat = DB()->CreateTable(table_info); - if(!stat.ok()) {//could exist + if(!stat.ok()) {//table could exist error_msg_ = "Engine failed: " + stat.ToString(); SERVER_LOG_ERROR << error_msg_; return SERVER_SUCCESS; @@ -129,10 +140,10 @@ ServerError DescribeTableTask::OnExecute() { try { engine::meta::TableSchema table_info; - table_info.table_id = table_name_; + table_info.table_id_ = table_name_; engine::Status stat = DB()->DescribeTable(table_info); if(!stat.ok()) { - error_code_ = SERVER_GROUP_NOT_EXIST; + error_code_ = SERVER_TABLE_NOT_EXIST; error_msg_ = "Engine failed: " + stat.ToString(); SERVER_LOG_ERROR << error_msg_; return error_code_; @@ -168,46 +179,6 @@ ServerError DeleteTableTask::OnExecute() { error_msg_ = "delete table not implemented"; SERVER_LOG_ERROR << error_msg_; - IVecIdMapper::GetInstance()->DeleteGroup(table_name_); - - return SERVER_NOT_IMPLEMENT; -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -CreateTablePartitionTask::CreateTablePartitionTask(const thrift::CreateTablePartitionParam ¶m) - : BaseTask(DDL_DML_TASK_GROUP), - param_(param) { - -} - -BaseTaskPtr CreateTablePartitionTask::Create(const thrift::CreateTablePartitionParam ¶m) { - return std::shared_ptr(new CreateTablePartitionTask(param)); -} - -ServerError CreateTablePartitionTask::OnExecute() { - error_code_ = SERVER_NOT_IMPLEMENT; - error_msg_ = "create table partition not implemented"; - SERVER_LOG_ERROR << error_msg_; - - return SERVER_NOT_IMPLEMENT; -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DeleteTablePartitionTask::DeleteTablePartitionTask(const thrift::DeleteTablePartitionParam ¶m) - : BaseTask(DDL_DML_TASK_GROUP), - param_(param) { - -} - -BaseTaskPtr DeleteTablePartitionTask::Create(const thrift::DeleteTablePartitionParam ¶m) { - return std::shared_ptr(new DeleteTablePartitionTask(param)); -} - -ServerError DeleteTablePartitionTask::OnExecute() { - error_code_ = SERVER_NOT_IMPLEMENT; - error_msg_ = "delete table partition not implemented"; - SERVER_LOG_ERROR << error_msg_; - return SERVER_NOT_IMPLEMENT; } @@ -223,7 +194,6 @@ BaseTaskPtr ShowTablesTask::Create(std::vector& tables) { } ServerError ShowTablesTask::OnExecute() { - IVecIdMapper::GetInstance()->AllGroups(tables_); return SERVER_SUCCESS; } @@ -253,31 +223,33 @@ ServerError AddVectorTask::OnExecute() { return SERVER_SUCCESS; } + //step 1: check table existence engine::meta::TableSchema table_info; - table_info.table_id = table_name_; + table_info.table_id_ = table_name_; engine::Status stat = DB()->DescribeTable(table_info); if(!stat.ok()) { - error_code_ = SERVER_GROUP_NOT_EXIST; + error_code_ = SERVER_TABLE_NOT_EXIST; error_msg_ = "Engine failed: " + stat.ToString(); SERVER_LOG_ERROR << error_msg_; return error_code_; } - rc.Record("get group info"); + rc.Record("check validation"); + //step 2: prepare float data uint64_t vec_count = (uint64_t)record_array_.size(); - uint64_t group_dim = table_info.dimension; + uint64_t group_dim = table_info.dimension_; std::vector vec_f; vec_f.resize(vec_count*group_dim);//allocate enough memory for(uint64_t i = 0; i < vec_count; i++) { const auto& record = record_array_[i]; - if(record.vector_map.empty()) { + if(record.vector_data.empty()) { error_code_ = SERVER_INVALID_ARGUMENT; error_msg_ = "No vector provided in record"; SERVER_LOG_ERROR << error_msg_; return error_code_; } - uint64_t vec_dim = record.vector_map.begin()->second.size()/sizeof(double);//how many double value? + uint64_t vec_dim = record.vector_data.size()/sizeof(double);//how many double value? if(vec_dim != group_dim) { SERVER_LOG_ERROR << "Invalid vector dimension: " << vec_dim << " vs. group dimension:" << group_dim; @@ -287,7 +259,7 @@ ServerError AddVectorTask::OnExecute() { } //convert double array to float array(thrift has no float type) - const double* d_p = reinterpret_cast(record.vector_map.begin()->second.data()); + const double* d_p = reinterpret_cast(record.vector_data.data()); for(uint64_t d = 0; d < vec_dim; d++) { vec_f[i*vec_dim + d] = (float)(d_p[d]); } @@ -295,6 +267,7 @@ ServerError AddVectorTask::OnExecute() { rc.Record("prepare vectors data"); + //step 3: insert vectors stat = DB()->InsertVectors(table_name_, vec_count, vec_f.data(), record_ids_); rc.Record("add vectors to engine"); if(!stat.ok()) { @@ -309,22 +282,8 @@ ServerError AddVectorTask::OnExecute() { return SERVER_UNEXPECTED_ERROR; } - //persist attributes - for(uint64_t i = 0; i < vec_count; i++) { - const auto &record = record_array_[i]; - - //any attributes? - if(record.attribute_map.empty()) { - continue; - } - - std::string nid = std::to_string(record_ids_[i]); - std::string attrib_str; - AttributeSerializer::Encode(record.attribute_map, attrib_str); - IVecIdMapper::GetInstance()->Put(nid, attrib_str, table_name_); - } - - rc.Record("persist vector attributes"); + rc.Record("do insert"); + rc.Elapse("totally cost"); } catch (std::exception& ex) { error_code_ = SERVER_UNEXPECTED_ERROR; @@ -338,28 +297,33 @@ ServerError AddVectorTask::OnExecute() { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// SearchVectorTask::SearchVectorTask(const std::string& table_name, + const std::vector & query_record_array, + const std::vector & query_range_array, const int64_t top_k, - const std::vector& record_array, std::vector& result_array) : BaseTask(DQL_TASK_GROUP), table_name_(table_name), + record_array_(query_record_array), + range_array_(query_range_array), top_k_(top_k), - record_array_(record_array), result_array_(result_array) { } BaseTaskPtr SearchVectorTask::Create(const std::string& table_name, - const std::vector& record_array, + const std::vector & query_record_array, + const std::vector & query_range_array, const int64_t top_k, std::vector& result_array) { - return std::shared_ptr(new SearchVectorTask(table_name, top_k, record_array, result_array)); + return std::shared_ptr(new SearchVectorTask(table_name, + query_record_array, query_range_array, top_k, result_array)); } ServerError SearchVectorTask::OnExecute() { try { TimeRecorder rc("SearchVectorTask"); + //step 1: check validation if(top_k_ <= 0 || record_array_.empty()) { error_code_ = SERVER_INVALID_ARGUMENT; error_msg_ = "Invalid topk value, or query record array is empty"; @@ -367,40 +331,44 @@ ServerError SearchVectorTask::OnExecute() { return error_code_; } + //step 2: check table existence engine::meta::TableSchema table_info; - table_info.table_id = table_name_; + table_info.table_id_ = table_name_; engine::Status stat = DB()->DescribeTable(table_info); if(!stat.ok()) { - error_code_ = SERVER_GROUP_NOT_EXIST; + error_code_ = SERVER_TABLE_NOT_EXIST; error_msg_ = "Engine failed: " + stat.ToString(); SERVER_LOG_ERROR << error_msg_; return error_code_; } + rc.Record("check validation"); + + //step 3: prepare float data std::vector vec_f; uint64_t record_count = (uint64_t)record_array_.size(); - vec_f.resize(record_count*table_info.dimension); + vec_f.resize(record_count*table_info.dimension_); for(uint64_t i = 0; i < record_array_.size(); i++) { const auto& record = record_array_[i]; - if (record.vector_map.empty()) { + if (record.vector_data.empty()) { error_code_ = SERVER_INVALID_ARGUMENT; error_msg_ = "Query record has no vector"; SERVER_LOG_ERROR << error_msg_; return error_code_; } - uint64_t vec_dim = record.vector_map.begin()->second.size() / sizeof(double);//how many double value? - if (vec_dim != table_info.dimension) { + uint64_t vec_dim = record.vector_data.size() / sizeof(double);//how many double value? + if (vec_dim != table_info.dimension_) { SERVER_LOG_ERROR << "Invalid vector dimension: " << vec_dim - << " vs. group dimension:" << table_info.dimension; + << " vs. group dimension:" << table_info.dimension_; error_code_ = SERVER_INVALID_VECTOR_DIMENSION; error_msg_ = "Engine failed: " + stat.ToString(); return error_code_; } //convert double array to float array(thrift has no float type) - const double* d_p = reinterpret_cast(record.vector_map.begin()->second.data()); + const double* d_p = reinterpret_cast(record.vector_data.data()); for(uint64_t d = 0; d < vec_dim; d++) { vec_f[i*vec_dim + d] = (float)(d_p[d]); } @@ -408,6 +376,8 @@ ServerError SearchVectorTask::OnExecute() { rc.Record("prepare vector data"); + + //step 4: search vectors std::vector dates; engine::QueryResults results; stat = DB()->Query(table_name_, (size_t)top_k_, record_count, vec_f.data(), dates, results); @@ -422,32 +392,18 @@ ServerError SearchVectorTask::OnExecute() { return SERVER_UNEXPECTED_ERROR; } - //construct result array + rc.Record("do search"); + + //step 5: construct result array for(uint64_t i = 0; i < record_count; i++) { auto& result = results[i]; const auto& record = record_array_[i]; thrift::TopKQueryResult thrift_topk_result; - for(auto id : result) { + for(auto& pair : result) { thrift::QueryResult thrift_result; - thrift_result.__set_id(id); - - //need get attributes? - if(record.selected_column_array.empty()) { - thrift_topk_result.query_result_arrays.emplace_back(thrift_result); - continue; - } - - std::string nid = std::to_string(id); - std::string attrib_str; - IVecIdMapper::GetInstance()->Get(nid, attrib_str, table_name_); - - AttribMap attrib_map; - AttributeSerializer::Decode(attrib_str, attrib_map); - - for(auto& attribute : record.selected_column_array) { - thrift_result.column_map[attribute] = attrib_map[attribute]; - } + thrift_result.__set_id(pair.first); + thrift_result.__set_score(pair.second); thrift_topk_result.query_result_arrays.emplace_back(thrift_result); } @@ -466,6 +422,32 @@ ServerError SearchVectorTask::OnExecute() { return SERVER_SUCCESS; } +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +GetTableRowCountTask::GetTableRowCountTask(const std::string& table_name, int64_t& row_count) +: BaseTask(DQL_TASK_GROUP), + table_name_(table_name), + row_count_(row_count) { + +} + +BaseTaskPtr GetTableRowCountTask::Create(const std::string& table_name, int64_t& row_count) { + return std::shared_ptr(new GetTableRowCountTask(table_name, row_count)); +} + +ServerError GetTableRowCountTask::OnExecute() { + if(table_name_.empty()) { + error_code_ = SERVER_UNEXPECTED_ERROR; + error_msg_ = "Table name cannot be empty"; + SERVER_LOG_ERROR << error_msg_; + return error_code_; + } + + error_code_ = SERVER_NOT_IMPLEMENT; + error_msg_ = "Not implemented"; + SERVER_LOG_ERROR << error_msg_; + return error_code_; +} + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// PingTask::PingTask(const std::string& cmd, std::string& result) : BaseTask(PING_TASK_GROUP), @@ -480,7 +462,7 @@ BaseTaskPtr PingTask::Create(const std::string& cmd, std::string& result) { ServerError PingTask::OnExecute() { if(cmd_ == "version") { - result_ = "v1.2.0";//currently hardcode + result_ = MEGASEARCH_VERSION; } return SERVER_SUCCESS; diff --git a/cpp/src/server/MegasearchTask.h b/cpp/src/server/MegasearchTask.h index af4178eb2154530460f1f771c231e2426baf72ad..63a08751d41d775c41a0606be11d3ece53b3faa8 100644 --- a/cpp/src/server/MegasearchTask.h +++ b/cpp/src/server/MegasearchTask.h @@ -65,36 +65,6 @@ private: std::string table_name_; }; -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class CreateTablePartitionTask : public BaseTask { -public: - static BaseTaskPtr Create(const thrift::CreateTablePartitionParam ¶m); - -protected: - CreateTablePartitionTask(const thrift::CreateTablePartitionParam ¶m); - - ServerError OnExecute() override; - - -private: - const thrift::CreateTablePartitionParam ¶m_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class DeleteTablePartitionTask : public BaseTask { -public: - static BaseTaskPtr Create(const thrift::DeleteTablePartitionParam ¶m); - -protected: - DeleteTablePartitionTask(const thrift::DeleteTablePartitionParam ¶m); - - ServerError OnExecute() override; - - -private: - const thrift::DeleteTablePartitionParam ¶m_; -}; - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class ShowTablesTask : public BaseTask { public: @@ -133,14 +103,16 @@ private: class SearchVectorTask : public BaseTask { public: static BaseTaskPtr Create(const std::string& table_name, - const std::vector& record_array, + const std::vector & query_record_array, + const std::vector & query_range_array, const int64_t top_k, std::vector& result_array); protected: SearchVectorTask(const std::string& table_name, + const std::vector & query_record_array, + const std::vector & query_range_array, const int64_t top_k, - const std::vector& record_array, std::vector& result_array); ServerError OnExecute() override; @@ -148,10 +120,26 @@ protected: private: std::string table_name_; int64_t top_k_; - const std::vector& record_array_; + const std::vector& record_array_; + const std::vector& range_array_; std::vector& result_array_; }; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +class GetTableRowCountTask : public BaseTask { +public: + static BaseTaskPtr Create(const std::string& table_name, int64_t& row_count); + +protected: + GetTableRowCountTask(const std::string& table_name, int64_t& row_count); + + ServerError OnExecute() override; + +private: + std::string table_name_; + int64_t& row_count_; +}; + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class PingTask : public BaseTask { public: diff --git a/cpp/src/server/RocksIdMapper.cpp b/cpp/src/server/RocksIdMapper.cpp deleted file mode 100644 index 3ac887a61087af1c16dd03d74b0d7dd2386eca43..0000000000000000000000000000000000000000 --- a/cpp/src/server/RocksIdMapper.cpp +++ /dev/null @@ -1,327 +0,0 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ - -#include "RocksIdMapper.h" -#include "ServerConfig.h" -#include "utils/Log.h" -#include "utils/CommonUtil.h" - -#include "rocksdb/db.h" -#include "rocksdb/slice.h" -#include "rocksdb/options.h" - -#include - -namespace zilliz { -namespace vecwise { -namespace server { - -static const std::string ROCKSDB_DEFAULT_GROUP = "default"; - -RocksIdMapper::RocksIdMapper() - : db_(nullptr) { - OpenDb(); -} - -RocksIdMapper::~RocksIdMapper() { - CloseDb(); -} - -void RocksIdMapper::OpenDb() { - std::lock_guard lck(db_mutex_); - - if(db_) { - return; - } - - ConfigNode& config = ServerConfig::GetInstance().GetConfig(CONFIG_DB); - std::string db_path = config.GetValue(CONFIG_DB_PATH); - db_path += "/id_mapping"; - CommonUtil::CreateDirectory(db_path); - - rocksdb::Options options; - // Optimize RocksDB. This is the easiest way to get RocksDB to perform well - options.IncreaseParallelism(); - options.OptimizeLevelStyleCompaction(); - // create the DB if it's not already present - options.create_if_missing = true; - options.max_open_files = config.GetInt32Value(CONFIG_DB_IDMAPPER_MAX_FILE, 512); - - //load column families - std::vector column_names; - rocksdb::Status s = rocksdb::DB::ListColumnFamilies(options, db_path, &column_names); - if (!s.ok()) { - SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString(); - } - - if(column_names.empty()) { - column_names.push_back("default"); - } - SERVER_LOG_INFO << "ID mapper has " << std::to_string(column_names.size()) << " groups"; - - std::vector column_families; - for(auto& column_name : column_names) { - rocksdb::ColumnFamilyDescriptor desc; - desc.name = column_name; - column_families.emplace_back(desc); - } - - // open DB - std::vector column_handles; - s = rocksdb::DB::Open(options, db_path, column_families, &column_handles, &db_); - if(!s.ok()) { - SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString(); - db_ = nullptr; - } - - column_handles_.clear(); - for(auto handler : column_handles) { - column_handles_.insert(std::make_pair(handler->GetName(), handler)); - } -} - -void RocksIdMapper::CloseDb() { - std::lock_guard lck(db_mutex_); - - for(auto& iter : column_handles_) { - delete iter.second; - } - column_handles_.clear(); - - if(db_) { - db_->Close(); - delete db_; - } -} - -ServerError RocksIdMapper::AddGroup(const std::string& group) { - std::lock_guard lck(db_mutex_); - - return AddGroupInternal(group); -} - -bool RocksIdMapper::IsGroupExist(const std::string& group) const { - std::lock_guard lck(db_mutex_); - - return IsGroupExistInternal(group); -} - -ServerError RocksIdMapper::AllGroups(std::vector& groups) const { - groups.clear(); - - std::lock_guard lck(db_mutex_); - for(auto& pair : column_handles_) { - if(pair.first == ROCKSDB_DEFAULT_GROUP) { - continue; - } - groups.push_back(pair.first); - } - - return SERVER_SUCCESS; -} - -ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) { - std::lock_guard lck(db_mutex_); - - return PutInternal(nid, sid, group); -} - -ServerError RocksIdMapper::Put(const std::vector& nid, const std::vector& sid, const std::string& group) { - if(nid.size() != sid.size()) { - return SERVER_INVALID_ARGUMENT; - } - - std::lock_guard lck(db_mutex_); - ServerError err = SERVER_SUCCESS; - for(size_t i = 0; i < nid.size(); i++) { - err = PutInternal(nid[i], sid[i], group); - if(err != SERVER_SUCCESS) { - return err; - } - } - - return err; -} - -ServerError RocksIdMapper::Get(const std::string& nid, std::string& sid, const std::string& group) const { - std::lock_guard lck(db_mutex_); - - return GetInternal(nid, sid, group); -} - -ServerError RocksIdMapper::Get(const std::vector& nid, std::vector& sid, const std::string& group) const { - sid.clear(); - - std::lock_guard lck(db_mutex_); - - ServerError err = SERVER_SUCCESS; - for(size_t i = 0; i < nid.size(); i++) { - std::string str_id; - ServerError temp_err = GetInternal(nid[i], str_id, group); - if(temp_err != SERVER_SUCCESS) { - sid.push_back(""); - SERVER_LOG_ERROR << "ID mapper failed to get id: " << nid[i]; - err = temp_err; - continue; - } - - sid.push_back(str_id); - } - - return err; -} - -ServerError RocksIdMapper::Delete(const std::string& nid, const std::string& group) { - std::lock_guard lck(db_mutex_); - - return DeleteInternal(nid, group); -} - -ServerError RocksIdMapper::DeleteGroup(const std::string& group) { - std::lock_guard lck(db_mutex_); - - return DeleteGroupInternal(group); -} - -//internal methods(whitout lock) -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -ServerError RocksIdMapper::AddGroupInternal(const std::string& group) { - if(!IsGroupExistInternal(group)) { - if(db_ == nullptr) { - return SERVER_NULL_POINTER; - } - - try {//add group - rocksdb::ColumnFamilyHandle *cfh = nullptr; - rocksdb::Status s = db_->CreateColumnFamily(rocksdb::ColumnFamilyOptions(), group, &cfh); - if (!s.ok()) { - SERVER_LOG_ERROR << "ID mapper failed to create group:" << s.ToString(); - return SERVER_UNEXPECTED_ERROR; - } else { - column_handles_.insert(std::make_pair(group, cfh)); - } - } catch(std::exception& ex) { - SERVER_LOG_ERROR << "ID mapper failed to create group: " << ex.what(); - return SERVER_UNEXPECTED_ERROR; - } - } - - return SERVER_SUCCESS; -} - -bool RocksIdMapper::IsGroupExistInternal(const std::string& group) const { - std::string group_name = group; - if(group_name.empty()){ - group_name = ROCKSDB_DEFAULT_GROUP; - } - return (column_handles_.count(group_name) > 0 && column_handles_[group_name] != nullptr); -} - -ServerError RocksIdMapper::PutInternal(const std::string& nid, const std::string& sid, const std::string& group) { - if(db_ == nullptr) { - return SERVER_NULL_POINTER; - } - - rocksdb::Slice key(nid); - rocksdb::Slice value(sid); - if(group.empty()) {//to default group - rocksdb::Status s = db_->Put(rocksdb::WriteOptions(), key, value); - if (!s.ok()) { - SERVER_LOG_ERROR << "ID mapper failed to put:" << s.ToString(); - return SERVER_UNEXPECTED_ERROR; - } - } else { - //try create group - if(AddGroupInternal(group) != SERVER_SUCCESS){ - return SERVER_UNEXPECTED_ERROR; - } - - rocksdb::ColumnFamilyHandle *cfh = column_handles_[group]; - rocksdb::Status s = db_->Put(rocksdb::WriteOptions(), cfh, key, value); - if (!s.ok()) { - SERVER_LOG_ERROR << "ID mapper failed to put:" << s.ToString(); - return SERVER_UNEXPECTED_ERROR; - } - } - - return SERVER_SUCCESS; -} - -ServerError RocksIdMapper::GetInternal(const std::string& nid, std::string& sid, const std::string& group) const { - sid = ""; - if(db_ == nullptr) { - return SERVER_NULL_POINTER; - } - - rocksdb::ColumnFamilyHandle *cfh = nullptr; - if(column_handles_.count(group) != 0) { - cfh = column_handles_.at(group); - } - - rocksdb::Slice key(nid); - rocksdb::Status s; - if(cfh){ - s = db_->Get(rocksdb::ReadOptions(), cfh, key, &sid); - } else { - s = db_->Get(rocksdb::ReadOptions(), key, &sid); - } - - if(!s.ok()) { - SERVER_LOG_ERROR << "ID mapper failed to get:" << s.ToString(); - return SERVER_UNEXPECTED_ERROR; - } - - return SERVER_SUCCESS; -} - -ServerError RocksIdMapper::DeleteInternal(const std::string& nid, const std::string& group) { - if(db_ == nullptr) { - return SERVER_NULL_POINTER; - } - - rocksdb::ColumnFamilyHandle *cfh = nullptr; - if(column_handles_.count(group) != 0) { - cfh = column_handles_.at(group); - } - - rocksdb::Slice key(nid); - rocksdb::Status s; - if(cfh){ - s = db_->Delete(rocksdb::WriteOptions(), cfh, key); - } else { - s = db_->Delete(rocksdb::WriteOptions(), key); - } - if(!s.ok()) { - SERVER_LOG_ERROR << "ID mapper failed to delete:" << s.ToString(); - return SERVER_UNEXPECTED_ERROR; - } - - return SERVER_SUCCESS; -} - -ServerError RocksIdMapper::DeleteGroupInternal(const std::string& group) { - if(db_ == nullptr) { - return SERVER_NULL_POINTER; - } - - rocksdb::ColumnFamilyHandle *cfh = nullptr; - if(column_handles_.count(group) != 0) { - cfh = column_handles_.at(group); - } - - if(cfh) { - db_->DropColumnFamily(cfh); - db_->DestroyColumnFamilyHandle(cfh); - column_handles_.erase(group); - } - - return SERVER_SUCCESS; -} - - -} -} -} \ No newline at end of file diff --git a/cpp/src/server/RocksIdMapper.h b/cpp/src/server/RocksIdMapper.h deleted file mode 100644 index 777fbf08a6645c646c60d90911ddb9c690cdbddc..0000000000000000000000000000000000000000 --- a/cpp/src/server/RocksIdMapper.h +++ /dev/null @@ -1,68 +0,0 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ -#pragma once - -#include "utils/Error.h" -#include "VecIdMapper.h" - -#include -#include -#include -#include - -namespace rocksdb { - class DB; - class ColumnFamilyHandle; -} - -namespace zilliz { -namespace vecwise { -namespace server { - -class RocksIdMapper : public IVecIdMapper{ - public: - RocksIdMapper(); - ~RocksIdMapper(); - - ServerError AddGroup(const std::string& group) override; - bool IsGroupExist(const std::string& group) const override; - ServerError AllGroups(std::vector& groups) const override; - - ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override; - ServerError Put(const std::vector& nid, const std::vector& sid, const std::string& group = "") override; - - ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const override; - ServerError Get(const std::vector& nid, std::vector& sid, const std::string& group = "") const override; - - ServerError Delete(const std::string& nid, const std::string& group = "") override; - ServerError DeleteGroup(const std::string& group) override; - - private: - void OpenDb(); - void CloseDb(); - - ServerError AddGroupInternal(const std::string& group); - - bool IsGroupExistInternal(const std::string& group) const; - - ServerError PutInternal(const std::string& nid, const std::string& sid, const std::string& group); - - ServerError GetInternal(const std::string& nid, std::string& sid, const std::string& group) const; - - ServerError DeleteInternal(const std::string& nid, const std::string& group); - - ServerError DeleteGroupInternal(const std::string& group); - - private: - rocksdb::DB* db_; - mutable std::unordered_map column_handles_; - mutable std::mutex db_mutex_; -}; - - -} -} -} diff --git a/cpp/src/server/VecIdMapper.cpp b/cpp/src/server/VecIdMapper.cpp deleted file mode 100644 index aece541d9105018c6ad558d94f99f0331b69f463..0000000000000000000000000000000000000000 --- a/cpp/src/server/VecIdMapper.cpp +++ /dev/null @@ -1,162 +0,0 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ - -#include "VecIdMapper.h" -#include "RocksIdMapper.h" -#include "ServerConfig.h" -#include "utils/Log.h" -#include "utils/CommonUtil.h" - -#include "rocksdb/db.h" -#include "rocksdb/slice.h" -#include "rocksdb/options.h" - -#include -#include - -namespace zilliz { -namespace vecwise { -namespace server { - -IVecIdMapper* IVecIdMapper::GetInstance() { -#if 0 - static SimpleIdMapper s_mapper; - return &s_mapper; -#else - static RocksIdMapper s_mapper; - return &s_mapper; -#endif -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class SimpleIdMapper : public IVecIdMapper{ -public: - SimpleIdMapper(); - ~SimpleIdMapper(); - - ServerError AddGroup(const std::string& group) override; - bool IsGroupExist(const std::string& group) const override; - ServerError AllGroups(std::vector& groups) const override; - - ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override; - ServerError Put(const std::vector& nid, const std::vector& sid, const std::string& group = "") override; - - ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const override; - ServerError Get(const std::vector& nid, std::vector& sid, const std::string& group = "") const override; - - ServerError Delete(const std::string& nid, const std::string& group = "") override; - ServerError DeleteGroup(const std::string& group) override; - -private: - using ID_MAPPING = std::unordered_map; - mutable std::unordered_map id_groups_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -SimpleIdMapper::SimpleIdMapper() { - -} - -SimpleIdMapper::~SimpleIdMapper() { - -} - -ServerError -SimpleIdMapper::AddGroup(const std::string& group) { - if(id_groups_.count(group) == 0) { - id_groups_.insert(std::make_pair(group, ID_MAPPING())); - } -} - -//not thread-safe -bool -SimpleIdMapper::IsGroupExist(const std::string& group) const { - return id_groups_.count(group) > 0; -} - -ServerError SimpleIdMapper::AllGroups(std::vector& groups) const { - groups.clear(); - - for(auto& pair : id_groups_) { - groups.push_back(pair.first); - } - - return SERVER_SUCCESS; -} - -//not thread-safe -ServerError SimpleIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) { - ID_MAPPING& mapping = id_groups_[group]; - mapping[nid] = sid; - return SERVER_SUCCESS; -} - -//not thread-safe -ServerError SimpleIdMapper::Put(const std::vector& nid, const std::vector& sid, const std::string& group) { - if(nid.size() != sid.size()) { - return SERVER_INVALID_ARGUMENT; - } - - ID_MAPPING& mapping = id_groups_[group]; - for(size_t i = 0; i < nid.size(); i++) { - mapping[nid[i]] = sid[i]; - } - - return SERVER_SUCCESS; -} - -//not thread-safe -ServerError SimpleIdMapper::Get(const std::string& nid, std::string& sid, const std::string& group) const { - ID_MAPPING& mapping = id_groups_[group]; - - auto iter = mapping.find(nid); - if(iter == mapping.end()) { - return SERVER_INVALID_ARGUMENT; - } - - sid = iter->second; - - return SERVER_SUCCESS; -} - -//not thread-safe -ServerError SimpleIdMapper::Get(const std::vector& nid, std::vector& sid, const std::string& group) const { - sid.clear(); - - ID_MAPPING& mapping = id_groups_[group]; - - ServerError err = SERVER_SUCCESS; - for(size_t i = 0; i < nid.size(); i++) { - auto iter = mapping.find(nid[i]); - if(iter == mapping.end()) { - sid.push_back(""); - SERVER_LOG_ERROR << "ID mapper failed to find id: " << nid[i]; - err = SERVER_INVALID_ARGUMENT; - continue; - } - - sid.push_back(iter->second); - } - - return err; -} - -//not thread-safe -ServerError SimpleIdMapper::Delete(const std::string& nid, const std::string& group) { - ID_MAPPING& mapping = id_groups_[group]; - mapping.erase(nid); - return SERVER_SUCCESS; -} - -//not thread-safe -ServerError SimpleIdMapper::DeleteGroup(const std::string& group) { - id_groups_.erase(group); - return SERVER_SUCCESS; -} - -} -} -} \ No newline at end of file diff --git a/cpp/src/server/VecIdMapper.h b/cpp/src/server/VecIdMapper.h deleted file mode 100644 index 658eb7f3fb43953b0da4b6b466eb0b057afc4ed9..0000000000000000000000000000000000000000 --- a/cpp/src/server/VecIdMapper.h +++ /dev/null @@ -1,40 +0,0 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ -#pragma once - -#include "utils/Error.h" - -#include -#include - -namespace zilliz { -namespace vecwise { -namespace server { - -class IVecIdMapper { -public: - static IVecIdMapper* GetInstance(); - - virtual ~IVecIdMapper(){} - - virtual ServerError AddGroup(const std::string& group) = 0; - virtual bool IsGroupExist(const std::string& group) const = 0; - virtual ServerError AllGroups(std::vector& groups) const = 0; - - virtual ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") = 0; - virtual ServerError Put(const std::vector& nid, const std::vector& sid, const std::string& group = "") = 0; - - virtual ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const = 0; - //NOTE: the 'sid' will be cleared at begin of the function - virtual ServerError Get(const std::vector& nid, std::vector& sid, const std::string& group = "") const = 0; - - virtual ServerError Delete(const std::string& nid, const std::string& group = "") = 0; - virtual ServerError DeleteGroup(const std::string& group) = 0; -}; - -} -} -} diff --git a/cpp/src/thrift/gen-cpp/MegasearchService.cpp b/cpp/src/thrift/gen-cpp/MegasearchService.cpp index 6947c5be8f96f6f5c67c54cb660de4131d172e5d..a0d8a218100d1c2a32289d2f6945cd76821b62c8 100644 --- a/cpp/src/thrift/gen-cpp/MegasearchService.cpp +++ b/cpp/src/thrift/gen-cpp/MegasearchService.cpp @@ -383,11 +383,11 @@ uint32_t MegasearchService_DeleteTable_presult::read(::apache::thrift::protocol: } -MegasearchService_CreateTablePartition_args::~MegasearchService_CreateTablePartition_args() throw() { +MegasearchService_AddVector_args::~MegasearchService_AddVector_args() throw() { } -uint32_t MegasearchService_CreateTablePartition_args::read(::apache::thrift::protocol::TProtocol* iprot) { +uint32_t MegasearchService_AddVector_args::read(::apache::thrift::protocol::TProtocol* iprot) { ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); uint32_t xfer = 0; @@ -409,9 +409,29 @@ uint32_t MegasearchService_CreateTablePartition_args::read(::apache::thrift::pro switch (fid) { case 2: - if (ftype == ::apache::thrift::protocol::T_STRUCT) { - xfer += this->param.read(iprot); - this->__isset.param = true; + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(this->table_name); + this->__isset.table_name = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 3: + if (ftype == ::apache::thrift::protocol::T_LIST) { + { + this->record_array.clear(); + uint32_t _size19; + ::apache::thrift::protocol::TType _etype22; + xfer += iprot->readListBegin(_etype22, _size19); + this->record_array.resize(_size19); + uint32_t _i23; + for (_i23 = 0; _i23 < _size19; ++_i23) + { + xfer += this->record_array[_i23].read(iprot); + } + xfer += iprot->readListEnd(); + } + this->__isset.record_array = true; } else { xfer += iprot->skip(ftype); } @@ -428,153 +448,69 @@ uint32_t MegasearchService_CreateTablePartition_args::read(::apache::thrift::pro return xfer; } -uint32_t MegasearchService_CreateTablePartition_args::write(::apache::thrift::protocol::TProtocol* oprot) const { - uint32_t xfer = 0; - ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); - xfer += oprot->writeStructBegin("MegasearchService_CreateTablePartition_args"); - - xfer += oprot->writeFieldBegin("param", ::apache::thrift::protocol::T_STRUCT, 2); - xfer += this->param.write(oprot); - xfer += oprot->writeFieldEnd(); - - xfer += oprot->writeFieldStop(); - xfer += oprot->writeStructEnd(); - return xfer; -} - - -MegasearchService_CreateTablePartition_pargs::~MegasearchService_CreateTablePartition_pargs() throw() { -} - - -uint32_t MegasearchService_CreateTablePartition_pargs::write(::apache::thrift::protocol::TProtocol* oprot) const { +uint32_t MegasearchService_AddVector_args::write(::apache::thrift::protocol::TProtocol* oprot) const { uint32_t xfer = 0; ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); - xfer += oprot->writeStructBegin("MegasearchService_CreateTablePartition_pargs"); + xfer += oprot->writeStructBegin("MegasearchService_AddVector_args"); - xfer += oprot->writeFieldBegin("param", ::apache::thrift::protocol::T_STRUCT, 2); - xfer += (*(this->param)).write(oprot); + xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeString(this->table_name); xfer += oprot->writeFieldEnd(); - xfer += oprot->writeFieldStop(); - xfer += oprot->writeStructEnd(); - return xfer; -} - - -MegasearchService_CreateTablePartition_result::~MegasearchService_CreateTablePartition_result() throw() { -} - - -uint32_t MegasearchService_CreateTablePartition_result::read(::apache::thrift::protocol::TProtocol* iprot) { - - ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); - uint32_t xfer = 0; - std::string fname; - ::apache::thrift::protocol::TType ftype; - int16_t fid; - - xfer += iprot->readStructBegin(fname); - - using ::apache::thrift::protocol::TProtocolException; - - - while (true) + xfer += oprot->writeFieldBegin("record_array", ::apache::thrift::protocol::T_LIST, 3); { - xfer += iprot->readFieldBegin(fname, ftype, fid); - if (ftype == ::apache::thrift::protocol::T_STOP) { - break; - } - switch (fid) + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->record_array.size())); + std::vector ::const_iterator _iter24; + for (_iter24 = this->record_array.begin(); _iter24 != this->record_array.end(); ++_iter24) { - case 1: - if (ftype == ::apache::thrift::protocol::T_STRUCT) { - xfer += this->e.read(iprot); - this->__isset.e = true; - } else { - xfer += iprot->skip(ftype); - } - break; - default: - xfer += iprot->skip(ftype); - break; + xfer += (*_iter24).write(oprot); } - xfer += iprot->readFieldEnd(); + xfer += oprot->writeListEnd(); } + xfer += oprot->writeFieldEnd(); - xfer += iprot->readStructEnd(); - - return xfer; -} - -uint32_t MegasearchService_CreateTablePartition_result::write(::apache::thrift::protocol::TProtocol* oprot) const { - - uint32_t xfer = 0; - - xfer += oprot->writeStructBegin("MegasearchService_CreateTablePartition_result"); - - if (this->__isset.e) { - xfer += oprot->writeFieldBegin("e", ::apache::thrift::protocol::T_STRUCT, 1); - xfer += this->e.write(oprot); - xfer += oprot->writeFieldEnd(); - } xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; } -MegasearchService_CreateTablePartition_presult::~MegasearchService_CreateTablePartition_presult() throw() { +MegasearchService_AddVector_pargs::~MegasearchService_AddVector_pargs() throw() { } -uint32_t MegasearchService_CreateTablePartition_presult::read(::apache::thrift::protocol::TProtocol* iprot) { - - ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); +uint32_t MegasearchService_AddVector_pargs::write(::apache::thrift::protocol::TProtocol* oprot) const { uint32_t xfer = 0; - std::string fname; - ::apache::thrift::protocol::TType ftype; - int16_t fid; - - xfer += iprot->readStructBegin(fname); - - using ::apache::thrift::protocol::TProtocolException; + ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); + xfer += oprot->writeStructBegin("MegasearchService_AddVector_pargs"); + xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeString((*(this->table_name))); + xfer += oprot->writeFieldEnd(); - while (true) + xfer += oprot->writeFieldBegin("record_array", ::apache::thrift::protocol::T_LIST, 3); { - xfer += iprot->readFieldBegin(fname, ftype, fid); - if (ftype == ::apache::thrift::protocol::T_STOP) { - break; - } - switch (fid) + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast((*(this->record_array)).size())); + std::vector ::const_iterator _iter25; + for (_iter25 = (*(this->record_array)).begin(); _iter25 != (*(this->record_array)).end(); ++_iter25) { - case 1: - if (ftype == ::apache::thrift::protocol::T_STRUCT) { - xfer += this->e.read(iprot); - this->__isset.e = true; - } else { - xfer += iprot->skip(ftype); - } - break; - default: - xfer += iprot->skip(ftype); - break; + xfer += (*_iter25).write(oprot); } - xfer += iprot->readFieldEnd(); + xfer += oprot->writeListEnd(); } + xfer += oprot->writeFieldEnd(); - xfer += iprot->readStructEnd(); - + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); return xfer; } -MegasearchService_DeleteTablePartition_args::~MegasearchService_DeleteTablePartition_args() throw() { +MegasearchService_AddVector_result::~MegasearchService_AddVector_result() throw() { } -uint32_t MegasearchService_DeleteTablePartition_args::read(::apache::thrift::protocol::TProtocol* iprot) { +uint32_t MegasearchService_AddVector_result::read(::apache::thrift::protocol::TProtocol* iprot) { ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); uint32_t xfer = 0; @@ -595,85 +531,26 @@ uint32_t MegasearchService_DeleteTablePartition_args::read(::apache::thrift::pro } switch (fid) { - case 2: - if (ftype == ::apache::thrift::protocol::T_STRUCT) { - xfer += this->param.read(iprot); - this->__isset.param = true; + case 0: + if (ftype == ::apache::thrift::protocol::T_LIST) { + { + this->success.clear(); + uint32_t _size26; + ::apache::thrift::protocol::TType _etype29; + xfer += iprot->readListBegin(_etype29, _size26); + this->success.resize(_size26); + uint32_t _i30; + for (_i30 = 0; _i30 < _size26; ++_i30) + { + xfer += iprot->readI64(this->success[_i30]); + } + xfer += iprot->readListEnd(); + } + this->__isset.success = true; } else { xfer += iprot->skip(ftype); } break; - default: - xfer += iprot->skip(ftype); - break; - } - xfer += iprot->readFieldEnd(); - } - - xfer += iprot->readStructEnd(); - - return xfer; -} - -uint32_t MegasearchService_DeleteTablePartition_args::write(::apache::thrift::protocol::TProtocol* oprot) const { - uint32_t xfer = 0; - ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); - xfer += oprot->writeStructBegin("MegasearchService_DeleteTablePartition_args"); - - xfer += oprot->writeFieldBegin("param", ::apache::thrift::protocol::T_STRUCT, 2); - xfer += this->param.write(oprot); - xfer += oprot->writeFieldEnd(); - - xfer += oprot->writeFieldStop(); - xfer += oprot->writeStructEnd(); - return xfer; -} - - -MegasearchService_DeleteTablePartition_pargs::~MegasearchService_DeleteTablePartition_pargs() throw() { -} - - -uint32_t MegasearchService_DeleteTablePartition_pargs::write(::apache::thrift::protocol::TProtocol* oprot) const { - uint32_t xfer = 0; - ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); - xfer += oprot->writeStructBegin("MegasearchService_DeleteTablePartition_pargs"); - - xfer += oprot->writeFieldBegin("param", ::apache::thrift::protocol::T_STRUCT, 2); - xfer += (*(this->param)).write(oprot); - xfer += oprot->writeFieldEnd(); - - xfer += oprot->writeFieldStop(); - xfer += oprot->writeStructEnd(); - return xfer; -} - - -MegasearchService_DeleteTablePartition_result::~MegasearchService_DeleteTablePartition_result() throw() { -} - - -uint32_t MegasearchService_DeleteTablePartition_result::read(::apache::thrift::protocol::TProtocol* iprot) { - - ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); - uint32_t xfer = 0; - std::string fname; - ::apache::thrift::protocol::TType ftype; - int16_t fid; - - xfer += iprot->readStructBegin(fname); - - using ::apache::thrift::protocol::TProtocolException; - - - while (true) - { - xfer += iprot->readFieldBegin(fname, ftype, fid); - if (ftype == ::apache::thrift::protocol::T_STOP) { - break; - } - switch (fid) - { case 1: if (ftype == ::apache::thrift::protocol::T_STRUCT) { xfer += this->e.read(iprot); @@ -694,13 +571,25 @@ uint32_t MegasearchService_DeleteTablePartition_result::read(::apache::thrift::p return xfer; } -uint32_t MegasearchService_DeleteTablePartition_result::write(::apache::thrift::protocol::TProtocol* oprot) const { +uint32_t MegasearchService_AddVector_result::write(::apache::thrift::protocol::TProtocol* oprot) const { uint32_t xfer = 0; - xfer += oprot->writeStructBegin("MegasearchService_DeleteTablePartition_result"); + xfer += oprot->writeStructBegin("MegasearchService_AddVector_result"); - if (this->__isset.e) { + if (this->__isset.success) { + xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_LIST, 0); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I64, static_cast(this->success.size())); + std::vector ::const_iterator _iter31; + for (_iter31 = this->success.begin(); _iter31 != this->success.end(); ++_iter31) + { + xfer += oprot->writeI64((*_iter31)); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + } else if (this->__isset.e) { xfer += oprot->writeFieldBegin("e", ::apache::thrift::protocol::T_STRUCT, 1); xfer += this->e.write(oprot); xfer += oprot->writeFieldEnd(); @@ -711,11 +600,11 @@ uint32_t MegasearchService_DeleteTablePartition_result::write(::apache::thrift:: } -MegasearchService_DeleteTablePartition_presult::~MegasearchService_DeleteTablePartition_presult() throw() { +MegasearchService_AddVector_presult::~MegasearchService_AddVector_presult() throw() { } -uint32_t MegasearchService_DeleteTablePartition_presult::read(::apache::thrift::protocol::TProtocol* iprot) { +uint32_t MegasearchService_AddVector_presult::read(::apache::thrift::protocol::TProtocol* iprot) { ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); uint32_t xfer = 0; @@ -736,6 +625,26 @@ uint32_t MegasearchService_DeleteTablePartition_presult::read(::apache::thrift:: } switch (fid) { + case 0: + if (ftype == ::apache::thrift::protocol::T_LIST) { + { + (*(this->success)).clear(); + uint32_t _size32; + ::apache::thrift::protocol::TType _etype35; + xfer += iprot->readListBegin(_etype35, _size32); + (*(this->success)).resize(_size32); + uint32_t _i36; + for (_i36 = 0; _i36 < _size32; ++_i36) + { + xfer += iprot->readI64((*(this->success))[_i36]); + } + xfer += iprot->readListEnd(); + } + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; case 1: if (ftype == ::apache::thrift::protocol::T_STRUCT) { xfer += this->e.read(iprot); @@ -757,11 +666,11 @@ uint32_t MegasearchService_DeleteTablePartition_presult::read(::apache::thrift:: } -MegasearchService_AddVector_args::~MegasearchService_AddVector_args() throw() { +MegasearchService_SearchVector_args::~MegasearchService_SearchVector_args() throw() { } -uint32_t MegasearchService_AddVector_args::read(::apache::thrift::protocol::TProtocol* iprot) { +uint32_t MegasearchService_SearchVector_args::read(::apache::thrift::protocol::TProtocol* iprot) { ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); uint32_t xfer = 0; @@ -793,19 +702,47 @@ uint32_t MegasearchService_AddVector_args::read(::apache::thrift::protocol::TPro case 3: if (ftype == ::apache::thrift::protocol::T_LIST) { { - this->record_array.clear(); - uint32_t _size113; - ::apache::thrift::protocol::TType _etype116; - xfer += iprot->readListBegin(_etype116, _size113); - this->record_array.resize(_size113); - uint32_t _i117; - for (_i117 = 0; _i117 < _size113; ++_i117) + this->query_record_array.clear(); + uint32_t _size37; + ::apache::thrift::protocol::TType _etype40; + xfer += iprot->readListBegin(_etype40, _size37); + this->query_record_array.resize(_size37); + uint32_t _i41; + for (_i41 = 0; _i41 < _size37; ++_i41) { - xfer += this->record_array[_i117].read(iprot); + xfer += this->query_record_array[_i41].read(iprot); } xfer += iprot->readListEnd(); } - this->__isset.record_array = true; + this->__isset.query_record_array = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 4: + if (ftype == ::apache::thrift::protocol::T_LIST) { + { + this->query_range_array.clear(); + uint32_t _size42; + ::apache::thrift::protocol::TType _etype45; + xfer += iprot->readListBegin(_etype45, _size42); + this->query_range_array.resize(_size42); + uint32_t _i46; + for (_i46 = 0; _i46 < _size42; ++_i46) + { + xfer += this->query_range_array[_i46].read(iprot); + } + xfer += iprot->readListEnd(); + } + this->__isset.query_range_array = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 5: + if (ftype == ::apache::thrift::protocol::T_I64) { + xfer += iprot->readI64(this->topk); + this->__isset.topk = true; } else { xfer += iprot->skip(ftype); } @@ -822,69 +759,101 @@ uint32_t MegasearchService_AddVector_args::read(::apache::thrift::protocol::TPro return xfer; } -uint32_t MegasearchService_AddVector_args::write(::apache::thrift::protocol::TProtocol* oprot) const { +uint32_t MegasearchService_SearchVector_args::write(::apache::thrift::protocol::TProtocol* oprot) const { uint32_t xfer = 0; ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); - xfer += oprot->writeStructBegin("MegasearchService_AddVector_args"); + xfer += oprot->writeStructBegin("MegasearchService_SearchVector_args"); xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 2); xfer += oprot->writeString(this->table_name); xfer += oprot->writeFieldEnd(); - xfer += oprot->writeFieldBegin("record_array", ::apache::thrift::protocol::T_LIST, 3); + xfer += oprot->writeFieldBegin("query_record_array", ::apache::thrift::protocol::T_LIST, 3); { - xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->record_array.size())); - std::vector ::const_iterator _iter118; - for (_iter118 = this->record_array.begin(); _iter118 != this->record_array.end(); ++_iter118) + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->query_record_array.size())); + std::vector ::const_iterator _iter47; + for (_iter47 = this->query_record_array.begin(); _iter47 != this->query_record_array.end(); ++_iter47) { - xfer += (*_iter118).write(oprot); + xfer += (*_iter47).write(oprot); } xfer += oprot->writeListEnd(); } xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("query_range_array", ::apache::thrift::protocol::T_LIST, 4); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->query_range_array.size())); + std::vector ::const_iterator _iter48; + for (_iter48 = this->query_range_array.begin(); _iter48 != this->query_range_array.end(); ++_iter48) + { + xfer += (*_iter48).write(oprot); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("topk", ::apache::thrift::protocol::T_I64, 5); + xfer += oprot->writeI64(this->topk); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; } -MegasearchService_AddVector_pargs::~MegasearchService_AddVector_pargs() throw() { +MegasearchService_SearchVector_pargs::~MegasearchService_SearchVector_pargs() throw() { } -uint32_t MegasearchService_AddVector_pargs::write(::apache::thrift::protocol::TProtocol* oprot) const { +uint32_t MegasearchService_SearchVector_pargs::write(::apache::thrift::protocol::TProtocol* oprot) const { uint32_t xfer = 0; ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); - xfer += oprot->writeStructBegin("MegasearchService_AddVector_pargs"); + xfer += oprot->writeStructBegin("MegasearchService_SearchVector_pargs"); xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 2); xfer += oprot->writeString((*(this->table_name))); xfer += oprot->writeFieldEnd(); - xfer += oprot->writeFieldBegin("record_array", ::apache::thrift::protocol::T_LIST, 3); + xfer += oprot->writeFieldBegin("query_record_array", ::apache::thrift::protocol::T_LIST, 3); { - xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast((*(this->record_array)).size())); - std::vector ::const_iterator _iter119; - for (_iter119 = (*(this->record_array)).begin(); _iter119 != (*(this->record_array)).end(); ++_iter119) + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast((*(this->query_record_array)).size())); + std::vector ::const_iterator _iter49; + for (_iter49 = (*(this->query_record_array)).begin(); _iter49 != (*(this->query_record_array)).end(); ++_iter49) + { + xfer += (*_iter49).write(oprot); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("query_range_array", ::apache::thrift::protocol::T_LIST, 4); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast((*(this->query_range_array)).size())); + std::vector ::const_iterator _iter50; + for (_iter50 = (*(this->query_range_array)).begin(); _iter50 != (*(this->query_range_array)).end(); ++_iter50) { - xfer += (*_iter119).write(oprot); + xfer += (*_iter50).write(oprot); } xfer += oprot->writeListEnd(); } xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("topk", ::apache::thrift::protocol::T_I64, 5); + xfer += oprot->writeI64((*(this->topk))); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; } -MegasearchService_AddVector_result::~MegasearchService_AddVector_result() throw() { +MegasearchService_SearchVector_result::~MegasearchService_SearchVector_result() throw() { } -uint32_t MegasearchService_AddVector_result::read(::apache::thrift::protocol::TProtocol* iprot) { +uint32_t MegasearchService_SearchVector_result::read(::apache::thrift::protocol::TProtocol* iprot) { ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); uint32_t xfer = 0; @@ -909,14 +878,14 @@ uint32_t MegasearchService_AddVector_result::read(::apache::thrift::protocol::TP if (ftype == ::apache::thrift::protocol::T_LIST) { { this->success.clear(); - uint32_t _size120; - ::apache::thrift::protocol::TType _etype123; - xfer += iprot->readListBegin(_etype123, _size120); - this->success.resize(_size120); - uint32_t _i124; - for (_i124 = 0; _i124 < _size120; ++_i124) + uint32_t _size51; + ::apache::thrift::protocol::TType _etype54; + xfer += iprot->readListBegin(_etype54, _size51); + this->success.resize(_size51); + uint32_t _i55; + for (_i55 = 0; _i55 < _size51; ++_i55) { - xfer += iprot->readI64(this->success[_i124]); + xfer += this->success[_i55].read(iprot); } xfer += iprot->readListEnd(); } @@ -945,20 +914,20 @@ uint32_t MegasearchService_AddVector_result::read(::apache::thrift::protocol::TP return xfer; } -uint32_t MegasearchService_AddVector_result::write(::apache::thrift::protocol::TProtocol* oprot) const { +uint32_t MegasearchService_SearchVector_result::write(::apache::thrift::protocol::TProtocol* oprot) const { uint32_t xfer = 0; - xfer += oprot->writeStructBegin("MegasearchService_AddVector_result"); + xfer += oprot->writeStructBegin("MegasearchService_SearchVector_result"); if (this->__isset.success) { xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_LIST, 0); { - xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I64, static_cast(this->success.size())); - std::vector ::const_iterator _iter125; - for (_iter125 = this->success.begin(); _iter125 != this->success.end(); ++_iter125) + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->success.size())); + std::vector ::const_iterator _iter56; + for (_iter56 = this->success.begin(); _iter56 != this->success.end(); ++_iter56) { - xfer += oprot->writeI64((*_iter125)); + xfer += (*_iter56).write(oprot); } xfer += oprot->writeListEnd(); } @@ -974,11 +943,11 @@ uint32_t MegasearchService_AddVector_result::write(::apache::thrift::protocol::T } -MegasearchService_AddVector_presult::~MegasearchService_AddVector_presult() throw() { +MegasearchService_SearchVector_presult::~MegasearchService_SearchVector_presult() throw() { } -uint32_t MegasearchService_AddVector_presult::read(::apache::thrift::protocol::TProtocol* iprot) { +uint32_t MegasearchService_SearchVector_presult::read(::apache::thrift::protocol::TProtocol* iprot) { ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); uint32_t xfer = 0; @@ -1003,14 +972,14 @@ uint32_t MegasearchService_AddVector_presult::read(::apache::thrift::protocol::T if (ftype == ::apache::thrift::protocol::T_LIST) { { (*(this->success)).clear(); - uint32_t _size126; - ::apache::thrift::protocol::TType _etype129; - xfer += iprot->readListBegin(_etype129, _size126); - (*(this->success)).resize(_size126); - uint32_t _i130; - for (_i130 = 0; _i130 < _size126; ++_i130) + uint32_t _size57; + ::apache::thrift::protocol::TType _etype60; + xfer += iprot->readListBegin(_etype60, _size57); + (*(this->success)).resize(_size57); + uint32_t _i61; + for (_i61 = 0; _i61 < _size57; ++_i61) { - xfer += iprot->readI64((*(this->success))[_i130]); + xfer += (*(this->success))[_i61].read(iprot); } xfer += iprot->readListEnd(); } @@ -1040,11 +1009,11 @@ uint32_t MegasearchService_AddVector_presult::read(::apache::thrift::protocol::T } -MegasearchService_SearchVector_args::~MegasearchService_SearchVector_args() throw() { +MegasearchService_DescribeTable_args::~MegasearchService_DescribeTable_args() throw() { } -uint32_t MegasearchService_SearchVector_args::read(::apache::thrift::protocol::TProtocol* iprot) { +uint32_t MegasearchService_DescribeTable_args::read(::apache::thrift::protocol::TProtocol* iprot) { ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); uint32_t xfer = 0; @@ -1073,34 +1042,6 @@ uint32_t MegasearchService_SearchVector_args::read(::apache::thrift::protocol::T xfer += iprot->skip(ftype); } break; - case 3: - if (ftype == ::apache::thrift::protocol::T_LIST) { - { - this->query_record_array.clear(); - uint32_t _size131; - ::apache::thrift::protocol::TType _etype134; - xfer += iprot->readListBegin(_etype134, _size131); - this->query_record_array.resize(_size131); - uint32_t _i135; - for (_i135 = 0; _i135 < _size131; ++_i135) - { - xfer += this->query_record_array[_i135].read(iprot); - } - xfer += iprot->readListEnd(); - } - this->__isset.query_record_array = true; - } else { - xfer += iprot->skip(ftype); - } - break; - case 4: - if (ftype == ::apache::thrift::protocol::T_I64) { - xfer += iprot->readI64(this->topk); - this->__isset.topk = true; - } else { - xfer += iprot->skip(ftype); - } - break; default: xfer += iprot->skip(ftype); break; @@ -1113,77 +1054,45 @@ uint32_t MegasearchService_SearchVector_args::read(::apache::thrift::protocol::T return xfer; } -uint32_t MegasearchService_SearchVector_args::write(::apache::thrift::protocol::TProtocol* oprot) const { +uint32_t MegasearchService_DescribeTable_args::write(::apache::thrift::protocol::TProtocol* oprot) const { uint32_t xfer = 0; ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); - xfer += oprot->writeStructBegin("MegasearchService_SearchVector_args"); + xfer += oprot->writeStructBegin("MegasearchService_DescribeTable_args"); xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 2); xfer += oprot->writeString(this->table_name); xfer += oprot->writeFieldEnd(); - xfer += oprot->writeFieldBegin("query_record_array", ::apache::thrift::protocol::T_LIST, 3); - { - xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->query_record_array.size())); - std::vector ::const_iterator _iter136; - for (_iter136 = this->query_record_array.begin(); _iter136 != this->query_record_array.end(); ++_iter136) - { - xfer += (*_iter136).write(oprot); - } - xfer += oprot->writeListEnd(); - } - xfer += oprot->writeFieldEnd(); - - xfer += oprot->writeFieldBegin("topk", ::apache::thrift::protocol::T_I64, 4); - xfer += oprot->writeI64(this->topk); - xfer += oprot->writeFieldEnd(); - xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; } -MegasearchService_SearchVector_pargs::~MegasearchService_SearchVector_pargs() throw() { +MegasearchService_DescribeTable_pargs::~MegasearchService_DescribeTable_pargs() throw() { } -uint32_t MegasearchService_SearchVector_pargs::write(::apache::thrift::protocol::TProtocol* oprot) const { +uint32_t MegasearchService_DescribeTable_pargs::write(::apache::thrift::protocol::TProtocol* oprot) const { uint32_t xfer = 0; ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); - xfer += oprot->writeStructBegin("MegasearchService_SearchVector_pargs"); + xfer += oprot->writeStructBegin("MegasearchService_DescribeTable_pargs"); xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 2); xfer += oprot->writeString((*(this->table_name))); xfer += oprot->writeFieldEnd(); - xfer += oprot->writeFieldBegin("query_record_array", ::apache::thrift::protocol::T_LIST, 3); - { - xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast((*(this->query_record_array)).size())); - std::vector ::const_iterator _iter137; - for (_iter137 = (*(this->query_record_array)).begin(); _iter137 != (*(this->query_record_array)).end(); ++_iter137) - { - xfer += (*_iter137).write(oprot); - } - xfer += oprot->writeListEnd(); - } - xfer += oprot->writeFieldEnd(); - - xfer += oprot->writeFieldBegin("topk", ::apache::thrift::protocol::T_I64, 4); - xfer += oprot->writeI64((*(this->topk))); - xfer += oprot->writeFieldEnd(); - xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; } -MegasearchService_SearchVector_result::~MegasearchService_SearchVector_result() throw() { +MegasearchService_DescribeTable_result::~MegasearchService_DescribeTable_result() throw() { } -uint32_t MegasearchService_SearchVector_result::read(::apache::thrift::protocol::TProtocol* iprot) { +uint32_t MegasearchService_DescribeTable_result::read(::apache::thrift::protocol::TProtocol* iprot) { ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); uint32_t xfer = 0; @@ -1205,20 +1114,8 @@ uint32_t MegasearchService_SearchVector_result::read(::apache::thrift::protocol: switch (fid) { case 0: - if (ftype == ::apache::thrift::protocol::T_LIST) { - { - this->success.clear(); - uint32_t _size138; - ::apache::thrift::protocol::TType _etype141; - xfer += iprot->readListBegin(_etype141, _size138); - this->success.resize(_size138); - uint32_t _i142; - for (_i142 = 0; _i142 < _size138; ++_i142) - { - xfer += this->success[_i142].read(iprot); - } - xfer += iprot->readListEnd(); - } + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->success.read(iprot); this->__isset.success = true; } else { xfer += iprot->skip(ftype); @@ -1244,23 +1141,15 @@ uint32_t MegasearchService_SearchVector_result::read(::apache::thrift::protocol: return xfer; } -uint32_t MegasearchService_SearchVector_result::write(::apache::thrift::protocol::TProtocol* oprot) const { - - uint32_t xfer = 0; - - xfer += oprot->writeStructBegin("MegasearchService_SearchVector_result"); +uint32_t MegasearchService_DescribeTable_result::write(::apache::thrift::protocol::TProtocol* oprot) const { - if (this->__isset.success) { - xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_LIST, 0); - { - xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->success.size())); - std::vector ::const_iterator _iter143; - for (_iter143 = this->success.begin(); _iter143 != this->success.end(); ++_iter143) - { - xfer += (*_iter143).write(oprot); - } - xfer += oprot->writeListEnd(); - } + uint32_t xfer = 0; + + xfer += oprot->writeStructBegin("MegasearchService_DescribeTable_result"); + + if (this->__isset.success) { + xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_STRUCT, 0); + xfer += this->success.write(oprot); xfer += oprot->writeFieldEnd(); } else if (this->__isset.e) { xfer += oprot->writeFieldBegin("e", ::apache::thrift::protocol::T_STRUCT, 1); @@ -1273,11 +1162,11 @@ uint32_t MegasearchService_SearchVector_result::write(::apache::thrift::protocol } -MegasearchService_SearchVector_presult::~MegasearchService_SearchVector_presult() throw() { +MegasearchService_DescribeTable_presult::~MegasearchService_DescribeTable_presult() throw() { } -uint32_t MegasearchService_SearchVector_presult::read(::apache::thrift::protocol::TProtocol* iprot) { +uint32_t MegasearchService_DescribeTable_presult::read(::apache::thrift::protocol::TProtocol* iprot) { ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); uint32_t xfer = 0; @@ -1299,20 +1188,8 @@ uint32_t MegasearchService_SearchVector_presult::read(::apache::thrift::protocol switch (fid) { case 0: - if (ftype == ::apache::thrift::protocol::T_LIST) { - { - (*(this->success)).clear(); - uint32_t _size144; - ::apache::thrift::protocol::TType _etype147; - xfer += iprot->readListBegin(_etype147, _size144); - (*(this->success)).resize(_size144); - uint32_t _i148; - for (_i148 = 0; _i148 < _size144; ++_i148) - { - xfer += (*(this->success))[_i148].read(iprot); - } - xfer += iprot->readListEnd(); - } + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += (*(this->success)).read(iprot); this->__isset.success = true; } else { xfer += iprot->skip(ftype); @@ -1339,11 +1216,11 @@ uint32_t MegasearchService_SearchVector_presult::read(::apache::thrift::protocol } -MegasearchService_DescribeTable_args::~MegasearchService_DescribeTable_args() throw() { +MegasearchService_GetTableRowCount_args::~MegasearchService_GetTableRowCount_args() throw() { } -uint32_t MegasearchService_DescribeTable_args::read(::apache::thrift::protocol::TProtocol* iprot) { +uint32_t MegasearchService_GetTableRowCount_args::read(::apache::thrift::protocol::TProtocol* iprot) { ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); uint32_t xfer = 0; @@ -1384,10 +1261,10 @@ uint32_t MegasearchService_DescribeTable_args::read(::apache::thrift::protocol:: return xfer; } -uint32_t MegasearchService_DescribeTable_args::write(::apache::thrift::protocol::TProtocol* oprot) const { +uint32_t MegasearchService_GetTableRowCount_args::write(::apache::thrift::protocol::TProtocol* oprot) const { uint32_t xfer = 0; ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); - xfer += oprot->writeStructBegin("MegasearchService_DescribeTable_args"); + xfer += oprot->writeStructBegin("MegasearchService_GetTableRowCount_args"); xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 2); xfer += oprot->writeString(this->table_name); @@ -1399,14 +1276,14 @@ uint32_t MegasearchService_DescribeTable_args::write(::apache::thrift::protocol: } -MegasearchService_DescribeTable_pargs::~MegasearchService_DescribeTable_pargs() throw() { +MegasearchService_GetTableRowCount_pargs::~MegasearchService_GetTableRowCount_pargs() throw() { } -uint32_t MegasearchService_DescribeTable_pargs::write(::apache::thrift::protocol::TProtocol* oprot) const { +uint32_t MegasearchService_GetTableRowCount_pargs::write(::apache::thrift::protocol::TProtocol* oprot) const { uint32_t xfer = 0; ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); - xfer += oprot->writeStructBegin("MegasearchService_DescribeTable_pargs"); + xfer += oprot->writeStructBegin("MegasearchService_GetTableRowCount_pargs"); xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 2); xfer += oprot->writeString((*(this->table_name))); @@ -1418,11 +1295,11 @@ uint32_t MegasearchService_DescribeTable_pargs::write(::apache::thrift::protocol } -MegasearchService_DescribeTable_result::~MegasearchService_DescribeTable_result() throw() { +MegasearchService_GetTableRowCount_result::~MegasearchService_GetTableRowCount_result() throw() { } -uint32_t MegasearchService_DescribeTable_result::read(::apache::thrift::protocol::TProtocol* iprot) { +uint32_t MegasearchService_GetTableRowCount_result::read(::apache::thrift::protocol::TProtocol* iprot) { ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); uint32_t xfer = 0; @@ -1444,8 +1321,8 @@ uint32_t MegasearchService_DescribeTable_result::read(::apache::thrift::protocol switch (fid) { case 0: - if (ftype == ::apache::thrift::protocol::T_STRUCT) { - xfer += this->success.read(iprot); + if (ftype == ::apache::thrift::protocol::T_I64) { + xfer += iprot->readI64(this->success); this->__isset.success = true; } else { xfer += iprot->skip(ftype); @@ -1471,15 +1348,15 @@ uint32_t MegasearchService_DescribeTable_result::read(::apache::thrift::protocol return xfer; } -uint32_t MegasearchService_DescribeTable_result::write(::apache::thrift::protocol::TProtocol* oprot) const { +uint32_t MegasearchService_GetTableRowCount_result::write(::apache::thrift::protocol::TProtocol* oprot) const { uint32_t xfer = 0; - xfer += oprot->writeStructBegin("MegasearchService_DescribeTable_result"); + xfer += oprot->writeStructBegin("MegasearchService_GetTableRowCount_result"); if (this->__isset.success) { - xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_STRUCT, 0); - xfer += this->success.write(oprot); + xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_I64, 0); + xfer += oprot->writeI64(this->success); xfer += oprot->writeFieldEnd(); } else if (this->__isset.e) { xfer += oprot->writeFieldBegin("e", ::apache::thrift::protocol::T_STRUCT, 1); @@ -1492,11 +1369,11 @@ uint32_t MegasearchService_DescribeTable_result::write(::apache::thrift::protoco } -MegasearchService_DescribeTable_presult::~MegasearchService_DescribeTable_presult() throw() { +MegasearchService_GetTableRowCount_presult::~MegasearchService_GetTableRowCount_presult() throw() { } -uint32_t MegasearchService_DescribeTable_presult::read(::apache::thrift::protocol::TProtocol* iprot) { +uint32_t MegasearchService_GetTableRowCount_presult::read(::apache::thrift::protocol::TProtocol* iprot) { ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); uint32_t xfer = 0; @@ -1518,8 +1395,8 @@ uint32_t MegasearchService_DescribeTable_presult::read(::apache::thrift::protoco switch (fid) { case 0: - if (ftype == ::apache::thrift::protocol::T_STRUCT) { - xfer += (*(this->success)).read(iprot); + if (ftype == ::apache::thrift::protocol::T_I64) { + xfer += iprot->readI64((*(this->success))); this->__isset.success = true; } else { xfer += iprot->skip(ftype); @@ -1633,14 +1510,14 @@ uint32_t MegasearchService_ShowTables_result::read(::apache::thrift::protocol::T if (ftype == ::apache::thrift::protocol::T_LIST) { { this->success.clear(); - uint32_t _size149; - ::apache::thrift::protocol::TType _etype152; - xfer += iprot->readListBegin(_etype152, _size149); - this->success.resize(_size149); - uint32_t _i153; - for (_i153 = 0; _i153 < _size149; ++_i153) + uint32_t _size62; + ::apache::thrift::protocol::TType _etype65; + xfer += iprot->readListBegin(_etype65, _size62); + this->success.resize(_size62); + uint32_t _i66; + for (_i66 = 0; _i66 < _size62; ++_i66) { - xfer += iprot->readString(this->success[_i153]); + xfer += iprot->readString(this->success[_i66]); } xfer += iprot->readListEnd(); } @@ -1679,10 +1556,10 @@ uint32_t MegasearchService_ShowTables_result::write(::apache::thrift::protocol:: xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_LIST, 0); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->success.size())); - std::vector ::const_iterator _iter154; - for (_iter154 = this->success.begin(); _iter154 != this->success.end(); ++_iter154) + std::vector ::const_iterator _iter67; + for (_iter67 = this->success.begin(); _iter67 != this->success.end(); ++_iter67) { - xfer += oprot->writeString((*_iter154)); + xfer += oprot->writeString((*_iter67)); } xfer += oprot->writeListEnd(); } @@ -1727,14 +1604,14 @@ uint32_t MegasearchService_ShowTables_presult::read(::apache::thrift::protocol:: if (ftype == ::apache::thrift::protocol::T_LIST) { { (*(this->success)).clear(); - uint32_t _size155; - ::apache::thrift::protocol::TType _etype158; - xfer += iprot->readListBegin(_etype158, _size155); - (*(this->success)).resize(_size155); - uint32_t _i159; - for (_i159 = 0; _i159 < _size155; ++_i159) + uint32_t _size68; + ::apache::thrift::protocol::TType _etype71; + xfer += iprot->readListBegin(_etype71, _size68); + (*(this->success)).resize(_size68); + uint32_t _i72; + for (_i72 = 0; _i72 < _size68; ++_i72) { - xfer += iprot->readString((*(this->success))[_i159]); + xfer += iprot->readString((*(this->success))[_i72]); } xfer += iprot->readListEnd(); } @@ -2082,19 +1959,20 @@ void MegasearchServiceClient::recv_DeleteTable() return; } -void MegasearchServiceClient::CreateTablePartition(const CreateTablePartitionParam& param) +void MegasearchServiceClient::AddVector(std::vector & _return, const std::string& table_name, const std::vector & record_array) { - send_CreateTablePartition(param); - recv_CreateTablePartition(); + send_AddVector(table_name, record_array); + recv_AddVector(_return); } -void MegasearchServiceClient::send_CreateTablePartition(const CreateTablePartitionParam& param) +void MegasearchServiceClient::send_AddVector(const std::string& table_name, const std::vector & record_array) { int32_t cseqid = 0; - oprot_->writeMessageBegin("CreateTablePartition", ::apache::thrift::protocol::T_CALL, cseqid); + oprot_->writeMessageBegin("AddVector", ::apache::thrift::protocol::T_CALL, cseqid); - MegasearchService_CreateTablePartition_pargs args; - args.param = ¶m; + MegasearchService_AddVector_pargs args; + args.table_name = &table_name; + args.record_array = &record_array; args.write(oprot_); oprot_->writeMessageEnd(); @@ -2102,7 +1980,7 @@ void MegasearchServiceClient::send_CreateTablePartition(const CreateTablePartiti oprot_->getTransport()->flush(); } -void MegasearchServiceClient::recv_CreateTablePartition() +void MegasearchServiceClient::recv_AddVector(std::vector & _return) { int32_t rseqid = 0; @@ -2122,92 +2000,43 @@ void MegasearchServiceClient::recv_CreateTablePartition() iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); } - if (fname.compare("CreateTablePartition") != 0) { + if (fname.compare("AddVector") != 0) { iprot_->skip(::apache::thrift::protocol::T_STRUCT); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); } - MegasearchService_CreateTablePartition_presult result; + MegasearchService_AddVector_presult result; + result.success = &_return; result.read(iprot_); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); - if (result.__isset.e) { - throw result.e; - } - return; -} - -void MegasearchServiceClient::DeleteTablePartition(const DeleteTablePartitionParam& param) -{ - send_DeleteTablePartition(param); - recv_DeleteTablePartition(); -} - -void MegasearchServiceClient::send_DeleteTablePartition(const DeleteTablePartitionParam& param) -{ - int32_t cseqid = 0; - oprot_->writeMessageBegin("DeleteTablePartition", ::apache::thrift::protocol::T_CALL, cseqid); - - MegasearchService_DeleteTablePartition_pargs args; - args.param = ¶m; - args.write(oprot_); - - oprot_->writeMessageEnd(); - oprot_->getTransport()->writeEnd(); - oprot_->getTransport()->flush(); -} - -void MegasearchServiceClient::recv_DeleteTablePartition() -{ - - int32_t rseqid = 0; - std::string fname; - ::apache::thrift::protocol::TMessageType mtype; - - iprot_->readMessageBegin(fname, mtype, rseqid); - if (mtype == ::apache::thrift::protocol::T_EXCEPTION) { - ::apache::thrift::TApplicationException x; - x.read(iprot_); - iprot_->readMessageEnd(); - iprot_->getTransport()->readEnd(); - throw x; - } - if (mtype != ::apache::thrift::protocol::T_REPLY) { - iprot_->skip(::apache::thrift::protocol::T_STRUCT); - iprot_->readMessageEnd(); - iprot_->getTransport()->readEnd(); - } - if (fname.compare("DeleteTablePartition") != 0) { - iprot_->skip(::apache::thrift::protocol::T_STRUCT); - iprot_->readMessageEnd(); - iprot_->getTransport()->readEnd(); + if (result.__isset.success) { + // _return pointer has now been filled + return; } - MegasearchService_DeleteTablePartition_presult result; - result.read(iprot_); - iprot_->readMessageEnd(); - iprot_->getTransport()->readEnd(); - if (result.__isset.e) { throw result.e; } - return; + throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "AddVector failed: unknown result"); } -void MegasearchServiceClient::AddVector(std::vector & _return, const std::string& table_name, const std::vector & record_array) +void MegasearchServiceClient::SearchVector(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) { - send_AddVector(table_name, record_array); - recv_AddVector(_return); + send_SearchVector(table_name, query_record_array, query_range_array, topk); + recv_SearchVector(_return); } -void MegasearchServiceClient::send_AddVector(const std::string& table_name, const std::vector & record_array) +void MegasearchServiceClient::send_SearchVector(const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) { int32_t cseqid = 0; - oprot_->writeMessageBegin("AddVector", ::apache::thrift::protocol::T_CALL, cseqid); + oprot_->writeMessageBegin("SearchVector", ::apache::thrift::protocol::T_CALL, cseqid); - MegasearchService_AddVector_pargs args; + MegasearchService_SearchVector_pargs args; args.table_name = &table_name; - args.record_array = &record_array; + args.query_record_array = &query_record_array; + args.query_range_array = &query_range_array; + args.topk = &topk; args.write(oprot_); oprot_->writeMessageEnd(); @@ -2215,7 +2044,7 @@ void MegasearchServiceClient::send_AddVector(const std::string& table_name, cons oprot_->getTransport()->flush(); } -void MegasearchServiceClient::recv_AddVector(std::vector & _return) +void MegasearchServiceClient::recv_SearchVector(std::vector & _return) { int32_t rseqid = 0; @@ -2235,12 +2064,12 @@ void MegasearchServiceClient::recv_AddVector(std::vector & _return) iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); } - if (fname.compare("AddVector") != 0) { + if (fname.compare("SearchVector") != 0) { iprot_->skip(::apache::thrift::protocol::T_STRUCT); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); } - MegasearchService_AddVector_presult result; + MegasearchService_SearchVector_presult result; result.success = &_return; result.read(iprot_); iprot_->readMessageEnd(); @@ -2253,24 +2082,22 @@ void MegasearchServiceClient::recv_AddVector(std::vector & _return) if (result.__isset.e) { throw result.e; } - throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "AddVector failed: unknown result"); + throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "SearchVector failed: unknown result"); } -void MegasearchServiceClient::SearchVector(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const int64_t topk) +void MegasearchServiceClient::DescribeTable(TableSchema& _return, const std::string& table_name) { - send_SearchVector(table_name, query_record_array, topk); - recv_SearchVector(_return); + send_DescribeTable(table_name); + recv_DescribeTable(_return); } -void MegasearchServiceClient::send_SearchVector(const std::string& table_name, const std::vector & query_record_array, const int64_t topk) +void MegasearchServiceClient::send_DescribeTable(const std::string& table_name) { int32_t cseqid = 0; - oprot_->writeMessageBegin("SearchVector", ::apache::thrift::protocol::T_CALL, cseqid); + oprot_->writeMessageBegin("DescribeTable", ::apache::thrift::protocol::T_CALL, cseqid); - MegasearchService_SearchVector_pargs args; + MegasearchService_DescribeTable_pargs args; args.table_name = &table_name; - args.query_record_array = &query_record_array; - args.topk = &topk; args.write(oprot_); oprot_->writeMessageEnd(); @@ -2278,7 +2105,7 @@ void MegasearchServiceClient::send_SearchVector(const std::string& table_name, c oprot_->getTransport()->flush(); } -void MegasearchServiceClient::recv_SearchVector(std::vector & _return) +void MegasearchServiceClient::recv_DescribeTable(TableSchema& _return) { int32_t rseqid = 0; @@ -2298,12 +2125,12 @@ void MegasearchServiceClient::recv_SearchVector(std::vector & _ iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); } - if (fname.compare("SearchVector") != 0) { + if (fname.compare("DescribeTable") != 0) { iprot_->skip(::apache::thrift::protocol::T_STRUCT); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); } - MegasearchService_SearchVector_presult result; + MegasearchService_DescribeTable_presult result; result.success = &_return; result.read(iprot_); iprot_->readMessageEnd(); @@ -2316,21 +2143,21 @@ void MegasearchServiceClient::recv_SearchVector(std::vector & _ if (result.__isset.e) { throw result.e; } - throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "SearchVector failed: unknown result"); + throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "DescribeTable failed: unknown result"); } -void MegasearchServiceClient::DescribeTable(TableSchema& _return, const std::string& table_name) +int64_t MegasearchServiceClient::GetTableRowCount(const std::string& table_name) { - send_DescribeTable(table_name); - recv_DescribeTable(_return); + send_GetTableRowCount(table_name); + return recv_GetTableRowCount(); } -void MegasearchServiceClient::send_DescribeTable(const std::string& table_name) +void MegasearchServiceClient::send_GetTableRowCount(const std::string& table_name) { int32_t cseqid = 0; - oprot_->writeMessageBegin("DescribeTable", ::apache::thrift::protocol::T_CALL, cseqid); + oprot_->writeMessageBegin("GetTableRowCount", ::apache::thrift::protocol::T_CALL, cseqid); - MegasearchService_DescribeTable_pargs args; + MegasearchService_GetTableRowCount_pargs args; args.table_name = &table_name; args.write(oprot_); @@ -2339,7 +2166,7 @@ void MegasearchServiceClient::send_DescribeTable(const std::string& table_name) oprot_->getTransport()->flush(); } -void MegasearchServiceClient::recv_DescribeTable(TableSchema& _return) +int64_t MegasearchServiceClient::recv_GetTableRowCount() { int32_t rseqid = 0; @@ -2359,25 +2186,25 @@ void MegasearchServiceClient::recv_DescribeTable(TableSchema& _return) iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); } - if (fname.compare("DescribeTable") != 0) { + if (fname.compare("GetTableRowCount") != 0) { iprot_->skip(::apache::thrift::protocol::T_STRUCT); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); } - MegasearchService_DescribeTable_presult result; + int64_t _return; + MegasearchService_GetTableRowCount_presult result; result.success = &_return; result.read(iprot_); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); if (result.__isset.success) { - // _return pointer has now been filled - return; + return _return; } if (result.__isset.e) { throw result.e; } - throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "DescribeTable failed: unknown result"); + throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "GetTableRowCount failed: unknown result"); } void MegasearchServiceClient::ShowTables(std::vector & _return) @@ -2495,177 +2322,65 @@ void MegasearchServiceClient::recv_Ping(std::string& _return) // _return pointer has now been filled return; } - if (result.__isset.e) { - throw result.e; - } - throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "Ping failed: unknown result"); -} - -bool MegasearchServiceProcessor::dispatchCall(::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, const std::string& fname, int32_t seqid, void* callContext) { - ProcessMap::iterator pfn; - pfn = processMap_.find(fname); - if (pfn == processMap_.end()) { - iprot->skip(::apache::thrift::protocol::T_STRUCT); - iprot->readMessageEnd(); - iprot->getTransport()->readEnd(); - ::apache::thrift::TApplicationException x(::apache::thrift::TApplicationException::UNKNOWN_METHOD, "Invalid method name: '"+fname+"'"); - oprot->writeMessageBegin(fname, ::apache::thrift::protocol::T_EXCEPTION, seqid); - x.write(oprot); - oprot->writeMessageEnd(); - oprot->getTransport()->writeEnd(); - oprot->getTransport()->flush(); - return true; - } - (this->*(pfn->second))(seqid, iprot, oprot, callContext); - return true; -} - -void MegasearchServiceProcessor::process_CreateTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext) -{ - void* ctx = NULL; - if (this->eventHandler_.get() != NULL) { - ctx = this->eventHandler_->getContext("MegasearchService.CreateTable", callContext); - } - ::apache::thrift::TProcessorContextFreer freer(this->eventHandler_.get(), ctx, "MegasearchService.CreateTable"); - - if (this->eventHandler_.get() != NULL) { - this->eventHandler_->preRead(ctx, "MegasearchService.CreateTable"); - } - - MegasearchService_CreateTable_args args; - args.read(iprot); - iprot->readMessageEnd(); - uint32_t bytes = iprot->getTransport()->readEnd(); - - if (this->eventHandler_.get() != NULL) { - this->eventHandler_->postRead(ctx, "MegasearchService.CreateTable", bytes); - } - - MegasearchService_CreateTable_result result; - try { - iface_->CreateTable(args.param); - } catch (Exception &e) { - result.e = e; - result.__isset.e = true; - } catch (const std::exception& e) { - if (this->eventHandler_.get() != NULL) { - this->eventHandler_->handlerError(ctx, "MegasearchService.CreateTable"); - } - - ::apache::thrift::TApplicationException x(e.what()); - oprot->writeMessageBegin("CreateTable", ::apache::thrift::protocol::T_EXCEPTION, seqid); - x.write(oprot); - oprot->writeMessageEnd(); - oprot->getTransport()->writeEnd(); - oprot->getTransport()->flush(); - return; - } - - if (this->eventHandler_.get() != NULL) { - this->eventHandler_->preWrite(ctx, "MegasearchService.CreateTable"); - } - - oprot->writeMessageBegin("CreateTable", ::apache::thrift::protocol::T_REPLY, seqid); - result.write(oprot); - oprot->writeMessageEnd(); - bytes = oprot->getTransport()->writeEnd(); - oprot->getTransport()->flush(); - - if (this->eventHandler_.get() != NULL) { - this->eventHandler_->postWrite(ctx, "MegasearchService.CreateTable", bytes); - } -} - -void MegasearchServiceProcessor::process_DeleteTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext) -{ - void* ctx = NULL; - if (this->eventHandler_.get() != NULL) { - ctx = this->eventHandler_->getContext("MegasearchService.DeleteTable", callContext); - } - ::apache::thrift::TProcessorContextFreer freer(this->eventHandler_.get(), ctx, "MegasearchService.DeleteTable"); - - if (this->eventHandler_.get() != NULL) { - this->eventHandler_->preRead(ctx, "MegasearchService.DeleteTable"); - } - - MegasearchService_DeleteTable_args args; - args.read(iprot); - iprot->readMessageEnd(); - uint32_t bytes = iprot->getTransport()->readEnd(); - - if (this->eventHandler_.get() != NULL) { - this->eventHandler_->postRead(ctx, "MegasearchService.DeleteTable", bytes); + if (result.__isset.e) { + throw result.e; } + throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "Ping failed: unknown result"); +} - MegasearchService_DeleteTable_result result; - try { - iface_->DeleteTable(args.table_name); - } catch (Exception &e) { - result.e = e; - result.__isset.e = true; - } catch (const std::exception& e) { - if (this->eventHandler_.get() != NULL) { - this->eventHandler_->handlerError(ctx, "MegasearchService.DeleteTable"); - } - - ::apache::thrift::TApplicationException x(e.what()); - oprot->writeMessageBegin("DeleteTable", ::apache::thrift::protocol::T_EXCEPTION, seqid); +bool MegasearchServiceProcessor::dispatchCall(::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, const std::string& fname, int32_t seqid, void* callContext) { + ProcessMap::iterator pfn; + pfn = processMap_.find(fname); + if (pfn == processMap_.end()) { + iprot->skip(::apache::thrift::protocol::T_STRUCT); + iprot->readMessageEnd(); + iprot->getTransport()->readEnd(); + ::apache::thrift::TApplicationException x(::apache::thrift::TApplicationException::UNKNOWN_METHOD, "Invalid method name: '"+fname+"'"); + oprot->writeMessageBegin(fname, ::apache::thrift::protocol::T_EXCEPTION, seqid); x.write(oprot); oprot->writeMessageEnd(); oprot->getTransport()->writeEnd(); oprot->getTransport()->flush(); - return; - } - - if (this->eventHandler_.get() != NULL) { - this->eventHandler_->preWrite(ctx, "MegasearchService.DeleteTable"); - } - - oprot->writeMessageBegin("DeleteTable", ::apache::thrift::protocol::T_REPLY, seqid); - result.write(oprot); - oprot->writeMessageEnd(); - bytes = oprot->getTransport()->writeEnd(); - oprot->getTransport()->flush(); - - if (this->eventHandler_.get() != NULL) { - this->eventHandler_->postWrite(ctx, "MegasearchService.DeleteTable", bytes); + return true; } + (this->*(pfn->second))(seqid, iprot, oprot, callContext); + return true; } -void MegasearchServiceProcessor::process_CreateTablePartition(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext) +void MegasearchServiceProcessor::process_CreateTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext) { void* ctx = NULL; if (this->eventHandler_.get() != NULL) { - ctx = this->eventHandler_->getContext("MegasearchService.CreateTablePartition", callContext); + ctx = this->eventHandler_->getContext("MegasearchService.CreateTable", callContext); } - ::apache::thrift::TProcessorContextFreer freer(this->eventHandler_.get(), ctx, "MegasearchService.CreateTablePartition"); + ::apache::thrift::TProcessorContextFreer freer(this->eventHandler_.get(), ctx, "MegasearchService.CreateTable"); if (this->eventHandler_.get() != NULL) { - this->eventHandler_->preRead(ctx, "MegasearchService.CreateTablePartition"); + this->eventHandler_->preRead(ctx, "MegasearchService.CreateTable"); } - MegasearchService_CreateTablePartition_args args; + MegasearchService_CreateTable_args args; args.read(iprot); iprot->readMessageEnd(); uint32_t bytes = iprot->getTransport()->readEnd(); if (this->eventHandler_.get() != NULL) { - this->eventHandler_->postRead(ctx, "MegasearchService.CreateTablePartition", bytes); + this->eventHandler_->postRead(ctx, "MegasearchService.CreateTable", bytes); } - MegasearchService_CreateTablePartition_result result; + MegasearchService_CreateTable_result result; try { - iface_->CreateTablePartition(args.param); + iface_->CreateTable(args.param); } catch (Exception &e) { result.e = e; result.__isset.e = true; } catch (const std::exception& e) { if (this->eventHandler_.get() != NULL) { - this->eventHandler_->handlerError(ctx, "MegasearchService.CreateTablePartition"); + this->eventHandler_->handlerError(ctx, "MegasearchService.CreateTable"); } ::apache::thrift::TApplicationException x(e.what()); - oprot->writeMessageBegin("CreateTablePartition", ::apache::thrift::protocol::T_EXCEPTION, seqid); + oprot->writeMessageBegin("CreateTable", ::apache::thrift::protocol::T_EXCEPTION, seqid); x.write(oprot); oprot->writeMessageEnd(); oprot->getTransport()->writeEnd(); @@ -2674,54 +2389,54 @@ void MegasearchServiceProcessor::process_CreateTablePartition(int32_t seqid, ::a } if (this->eventHandler_.get() != NULL) { - this->eventHandler_->preWrite(ctx, "MegasearchService.CreateTablePartition"); + this->eventHandler_->preWrite(ctx, "MegasearchService.CreateTable"); } - oprot->writeMessageBegin("CreateTablePartition", ::apache::thrift::protocol::T_REPLY, seqid); + oprot->writeMessageBegin("CreateTable", ::apache::thrift::protocol::T_REPLY, seqid); result.write(oprot); oprot->writeMessageEnd(); bytes = oprot->getTransport()->writeEnd(); oprot->getTransport()->flush(); if (this->eventHandler_.get() != NULL) { - this->eventHandler_->postWrite(ctx, "MegasearchService.CreateTablePartition", bytes); + this->eventHandler_->postWrite(ctx, "MegasearchService.CreateTable", bytes); } } -void MegasearchServiceProcessor::process_DeleteTablePartition(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext) +void MegasearchServiceProcessor::process_DeleteTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext) { void* ctx = NULL; if (this->eventHandler_.get() != NULL) { - ctx = this->eventHandler_->getContext("MegasearchService.DeleteTablePartition", callContext); + ctx = this->eventHandler_->getContext("MegasearchService.DeleteTable", callContext); } - ::apache::thrift::TProcessorContextFreer freer(this->eventHandler_.get(), ctx, "MegasearchService.DeleteTablePartition"); + ::apache::thrift::TProcessorContextFreer freer(this->eventHandler_.get(), ctx, "MegasearchService.DeleteTable"); if (this->eventHandler_.get() != NULL) { - this->eventHandler_->preRead(ctx, "MegasearchService.DeleteTablePartition"); + this->eventHandler_->preRead(ctx, "MegasearchService.DeleteTable"); } - MegasearchService_DeleteTablePartition_args args; + MegasearchService_DeleteTable_args args; args.read(iprot); iprot->readMessageEnd(); uint32_t bytes = iprot->getTransport()->readEnd(); if (this->eventHandler_.get() != NULL) { - this->eventHandler_->postRead(ctx, "MegasearchService.DeleteTablePartition", bytes); + this->eventHandler_->postRead(ctx, "MegasearchService.DeleteTable", bytes); } - MegasearchService_DeleteTablePartition_result result; + MegasearchService_DeleteTable_result result; try { - iface_->DeleteTablePartition(args.param); + iface_->DeleteTable(args.table_name); } catch (Exception &e) { result.e = e; result.__isset.e = true; } catch (const std::exception& e) { if (this->eventHandler_.get() != NULL) { - this->eventHandler_->handlerError(ctx, "MegasearchService.DeleteTablePartition"); + this->eventHandler_->handlerError(ctx, "MegasearchService.DeleteTable"); } ::apache::thrift::TApplicationException x(e.what()); - oprot->writeMessageBegin("DeleteTablePartition", ::apache::thrift::protocol::T_EXCEPTION, seqid); + oprot->writeMessageBegin("DeleteTable", ::apache::thrift::protocol::T_EXCEPTION, seqid); x.write(oprot); oprot->writeMessageEnd(); oprot->getTransport()->writeEnd(); @@ -2730,17 +2445,17 @@ void MegasearchServiceProcessor::process_DeleteTablePartition(int32_t seqid, ::a } if (this->eventHandler_.get() != NULL) { - this->eventHandler_->preWrite(ctx, "MegasearchService.DeleteTablePartition"); + this->eventHandler_->preWrite(ctx, "MegasearchService.DeleteTable"); } - oprot->writeMessageBegin("DeleteTablePartition", ::apache::thrift::protocol::T_REPLY, seqid); + oprot->writeMessageBegin("DeleteTable", ::apache::thrift::protocol::T_REPLY, seqid); result.write(oprot); oprot->writeMessageEnd(); bytes = oprot->getTransport()->writeEnd(); oprot->getTransport()->flush(); if (this->eventHandler_.get() != NULL) { - this->eventHandler_->postWrite(ctx, "MegasearchService.DeleteTablePartition", bytes); + this->eventHandler_->postWrite(ctx, "MegasearchService.DeleteTable", bytes); } } @@ -2824,7 +2539,7 @@ void MegasearchServiceProcessor::process_SearchVector(int32_t seqid, ::apache::t MegasearchService_SearchVector_result result; try { - iface_->SearchVector(result.success, args.table_name, args.query_record_array, args.topk); + iface_->SearchVector(result.success, args.table_name, args.query_record_array, args.query_range_array, args.topk); result.__isset.success = true; } catch (Exception &e) { result.e = e; @@ -2915,6 +2630,63 @@ void MegasearchServiceProcessor::process_DescribeTable(int32_t seqid, ::apache:: } } +void MegasearchServiceProcessor::process_GetTableRowCount(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext) +{ + void* ctx = NULL; + if (this->eventHandler_.get() != NULL) { + ctx = this->eventHandler_->getContext("MegasearchService.GetTableRowCount", callContext); + } + ::apache::thrift::TProcessorContextFreer freer(this->eventHandler_.get(), ctx, "MegasearchService.GetTableRowCount"); + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->preRead(ctx, "MegasearchService.GetTableRowCount"); + } + + MegasearchService_GetTableRowCount_args args; + args.read(iprot); + iprot->readMessageEnd(); + uint32_t bytes = iprot->getTransport()->readEnd(); + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->postRead(ctx, "MegasearchService.GetTableRowCount", bytes); + } + + MegasearchService_GetTableRowCount_result result; + try { + result.success = iface_->GetTableRowCount(args.table_name); + result.__isset.success = true; + } catch (Exception &e) { + result.e = e; + result.__isset.e = true; + } catch (const std::exception& e) { + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->handlerError(ctx, "MegasearchService.GetTableRowCount"); + } + + ::apache::thrift::TApplicationException x(e.what()); + oprot->writeMessageBegin("GetTableRowCount", ::apache::thrift::protocol::T_EXCEPTION, seqid); + x.write(oprot); + oprot->writeMessageEnd(); + oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + return; + } + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->preWrite(ctx, "MegasearchService.GetTableRowCount"); + } + + oprot->writeMessageBegin("GetTableRowCount", ::apache::thrift::protocol::T_REPLY, seqid); + result.write(oprot); + oprot->writeMessageEnd(); + bytes = oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->postWrite(ctx, "MegasearchService.GetTableRowCount", bytes); + } +} + void MegasearchServiceProcessor::process_ShowTables(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext) { void* ctx = NULL; @@ -3200,20 +2972,21 @@ void MegasearchServiceConcurrentClient::recv_DeleteTable(const int32_t seqid) } // end while(true) } -void MegasearchServiceConcurrentClient::CreateTablePartition(const CreateTablePartitionParam& param) +void MegasearchServiceConcurrentClient::AddVector(std::vector & _return, const std::string& table_name, const std::vector & record_array) { - int32_t seqid = send_CreateTablePartition(param); - recv_CreateTablePartition(seqid); + int32_t seqid = send_AddVector(table_name, record_array); + recv_AddVector(_return, seqid); } -int32_t MegasearchServiceConcurrentClient::send_CreateTablePartition(const CreateTablePartitionParam& param) +int32_t MegasearchServiceConcurrentClient::send_AddVector(const std::string& table_name, const std::vector & record_array) { int32_t cseqid = this->sync_.generateSeqId(); ::apache::thrift::async::TConcurrentSendSentry sentry(&this->sync_); - oprot_->writeMessageBegin("CreateTablePartition", ::apache::thrift::protocol::T_CALL, cseqid); + oprot_->writeMessageBegin("AddVector", ::apache::thrift::protocol::T_CALL, cseqid); - MegasearchService_CreateTablePartition_pargs args; - args.param = ¶m; + MegasearchService_AddVector_pargs args; + args.table_name = &table_name; + args.record_array = &record_array; args.write(oprot_); oprot_->writeMessageEnd(); @@ -3224,7 +2997,7 @@ int32_t MegasearchServiceConcurrentClient::send_CreateTablePartition(const Creat return cseqid; } -void MegasearchServiceConcurrentClient::recv_CreateTablePartition(const int32_t seqid) +void MegasearchServiceConcurrentClient::recv_AddVector(std::vector & _return, const int32_t seqid) { int32_t rseqid = 0; @@ -3253,7 +3026,7 @@ void MegasearchServiceConcurrentClient::recv_CreateTablePartition(const int32_t iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); } - if (fname.compare("CreateTablePartition") != 0) { + if (fname.compare("AddVector") != 0) { iprot_->skip(::apache::thrift::protocol::T_STRUCT); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); @@ -3262,99 +3035,23 @@ void MegasearchServiceConcurrentClient::recv_CreateTablePartition(const int32_t using ::apache::thrift::protocol::TProtocolException; throw TProtocolException(TProtocolException::INVALID_DATA); } - MegasearchService_CreateTablePartition_presult result; + MegasearchService_AddVector_presult result; + result.success = &_return; result.read(iprot_); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); - if (result.__isset.e) { - sentry.commit(); - throw result.e; - } - sentry.commit(); - return; - } - // seqid != rseqid - this->sync_.updatePending(fname, mtype, rseqid); - - // this will temporarily unlock the readMutex, and let other clients get work done - this->sync_.waitForWork(seqid); - } // end while(true) -} - -void MegasearchServiceConcurrentClient::DeleteTablePartition(const DeleteTablePartitionParam& param) -{ - int32_t seqid = send_DeleteTablePartition(param); - recv_DeleteTablePartition(seqid); -} - -int32_t MegasearchServiceConcurrentClient::send_DeleteTablePartition(const DeleteTablePartitionParam& param) -{ - int32_t cseqid = this->sync_.generateSeqId(); - ::apache::thrift::async::TConcurrentSendSentry sentry(&this->sync_); - oprot_->writeMessageBegin("DeleteTablePartition", ::apache::thrift::protocol::T_CALL, cseqid); - - MegasearchService_DeleteTablePartition_pargs args; - args.param = ¶m; - args.write(oprot_); - - oprot_->writeMessageEnd(); - oprot_->getTransport()->writeEnd(); - oprot_->getTransport()->flush(); - - sentry.commit(); - return cseqid; -} - -void MegasearchServiceConcurrentClient::recv_DeleteTablePartition(const int32_t seqid) -{ - - int32_t rseqid = 0; - std::string fname; - ::apache::thrift::protocol::TMessageType mtype; - - // the read mutex gets dropped and reacquired as part of waitForWork() - // The destructor of this sentry wakes up other clients - ::apache::thrift::async::TConcurrentRecvSentry sentry(&this->sync_, seqid); - - while(true) { - if(!this->sync_.getPending(fname, mtype, rseqid)) { - iprot_->readMessageBegin(fname, mtype, rseqid); - } - if(seqid == rseqid) { - if (mtype == ::apache::thrift::protocol::T_EXCEPTION) { - ::apache::thrift::TApplicationException x; - x.read(iprot_); - iprot_->readMessageEnd(); - iprot_->getTransport()->readEnd(); + if (result.__isset.success) { + // _return pointer has now been filled sentry.commit(); - throw x; - } - if (mtype != ::apache::thrift::protocol::T_REPLY) { - iprot_->skip(::apache::thrift::protocol::T_STRUCT); - iprot_->readMessageEnd(); - iprot_->getTransport()->readEnd(); - } - if (fname.compare("DeleteTablePartition") != 0) { - iprot_->skip(::apache::thrift::protocol::T_STRUCT); - iprot_->readMessageEnd(); - iprot_->getTransport()->readEnd(); - - // in a bad state, don't commit - using ::apache::thrift::protocol::TProtocolException; - throw TProtocolException(TProtocolException::INVALID_DATA); + return; } - MegasearchService_DeleteTablePartition_presult result; - result.read(iprot_); - iprot_->readMessageEnd(); - iprot_->getTransport()->readEnd(); - if (result.__isset.e) { sentry.commit(); throw result.e; } - sentry.commit(); - return; + // in a bad state, don't commit + throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "AddVector failed: unknown result"); } // seqid != rseqid this->sync_.updatePending(fname, mtype, rseqid); @@ -3364,21 +3061,23 @@ void MegasearchServiceConcurrentClient::recv_DeleteTablePartition(const int32_t } // end while(true) } -void MegasearchServiceConcurrentClient::AddVector(std::vector & _return, const std::string& table_name, const std::vector & record_array) +void MegasearchServiceConcurrentClient::SearchVector(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) { - int32_t seqid = send_AddVector(table_name, record_array); - recv_AddVector(_return, seqid); + int32_t seqid = send_SearchVector(table_name, query_record_array, query_range_array, topk); + recv_SearchVector(_return, seqid); } -int32_t MegasearchServiceConcurrentClient::send_AddVector(const std::string& table_name, const std::vector & record_array) +int32_t MegasearchServiceConcurrentClient::send_SearchVector(const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) { int32_t cseqid = this->sync_.generateSeqId(); ::apache::thrift::async::TConcurrentSendSentry sentry(&this->sync_); - oprot_->writeMessageBegin("AddVector", ::apache::thrift::protocol::T_CALL, cseqid); + oprot_->writeMessageBegin("SearchVector", ::apache::thrift::protocol::T_CALL, cseqid); - MegasearchService_AddVector_pargs args; + MegasearchService_SearchVector_pargs args; args.table_name = &table_name; - args.record_array = &record_array; + args.query_record_array = &query_record_array; + args.query_range_array = &query_range_array; + args.topk = &topk; args.write(oprot_); oprot_->writeMessageEnd(); @@ -3389,7 +3088,7 @@ int32_t MegasearchServiceConcurrentClient::send_AddVector(const std::string& tab return cseqid; } -void MegasearchServiceConcurrentClient::recv_AddVector(std::vector & _return, const int32_t seqid) +void MegasearchServiceConcurrentClient::recv_SearchVector(std::vector & _return, const int32_t seqid) { int32_t rseqid = 0; @@ -3418,7 +3117,7 @@ void MegasearchServiceConcurrentClient::recv_AddVector(std::vector & _r iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); } - if (fname.compare("AddVector") != 0) { + if (fname.compare("SearchVector") != 0) { iprot_->skip(::apache::thrift::protocol::T_STRUCT); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); @@ -3427,7 +3126,7 @@ void MegasearchServiceConcurrentClient::recv_AddVector(std::vector & _r using ::apache::thrift::protocol::TProtocolException; throw TProtocolException(TProtocolException::INVALID_DATA); } - MegasearchService_AddVector_presult result; + MegasearchService_SearchVector_presult result; result.success = &_return; result.read(iprot_); iprot_->readMessageEnd(); @@ -3443,7 +3142,7 @@ void MegasearchServiceConcurrentClient::recv_AddVector(std::vector & _r throw result.e; } // in a bad state, don't commit - throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "AddVector failed: unknown result"); + throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "SearchVector failed: unknown result"); } // seqid != rseqid this->sync_.updatePending(fname, mtype, rseqid); @@ -3453,22 +3152,20 @@ void MegasearchServiceConcurrentClient::recv_AddVector(std::vector & _r } // end while(true) } -void MegasearchServiceConcurrentClient::SearchVector(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const int64_t topk) +void MegasearchServiceConcurrentClient::DescribeTable(TableSchema& _return, const std::string& table_name) { - int32_t seqid = send_SearchVector(table_name, query_record_array, topk); - recv_SearchVector(_return, seqid); + int32_t seqid = send_DescribeTable(table_name); + recv_DescribeTable(_return, seqid); } -int32_t MegasearchServiceConcurrentClient::send_SearchVector(const std::string& table_name, const std::vector & query_record_array, const int64_t topk) +int32_t MegasearchServiceConcurrentClient::send_DescribeTable(const std::string& table_name) { int32_t cseqid = this->sync_.generateSeqId(); ::apache::thrift::async::TConcurrentSendSentry sentry(&this->sync_); - oprot_->writeMessageBegin("SearchVector", ::apache::thrift::protocol::T_CALL, cseqid); + oprot_->writeMessageBegin("DescribeTable", ::apache::thrift::protocol::T_CALL, cseqid); - MegasearchService_SearchVector_pargs args; + MegasearchService_DescribeTable_pargs args; args.table_name = &table_name; - args.query_record_array = &query_record_array; - args.topk = &topk; args.write(oprot_); oprot_->writeMessageEnd(); @@ -3479,7 +3176,7 @@ int32_t MegasearchServiceConcurrentClient::send_SearchVector(const std::string& return cseqid; } -void MegasearchServiceConcurrentClient::recv_SearchVector(std::vector & _return, const int32_t seqid) +void MegasearchServiceConcurrentClient::recv_DescribeTable(TableSchema& _return, const int32_t seqid) { int32_t rseqid = 0; @@ -3508,7 +3205,7 @@ void MegasearchServiceConcurrentClient::recv_SearchVector(std::vectorreadMessageEnd(); iprot_->getTransport()->readEnd(); } - if (fname.compare("SearchVector") != 0) { + if (fname.compare("DescribeTable") != 0) { iprot_->skip(::apache::thrift::protocol::T_STRUCT); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); @@ -3517,7 +3214,7 @@ void MegasearchServiceConcurrentClient::recv_SearchVector(std::vectorreadMessageEnd(); @@ -3533,7 +3230,7 @@ void MegasearchServiceConcurrentClient::recv_SearchVector(std::vectorsync_.updatePending(fname, mtype, rseqid); @@ -3543,19 +3240,19 @@ void MegasearchServiceConcurrentClient::recv_SearchVector(std::vectorsync_.generateSeqId(); ::apache::thrift::async::TConcurrentSendSentry sentry(&this->sync_); - oprot_->writeMessageBegin("DescribeTable", ::apache::thrift::protocol::T_CALL, cseqid); + oprot_->writeMessageBegin("GetTableRowCount", ::apache::thrift::protocol::T_CALL, cseqid); - MegasearchService_DescribeTable_pargs args; + MegasearchService_GetTableRowCount_pargs args; args.table_name = &table_name; args.write(oprot_); @@ -3567,7 +3264,7 @@ int32_t MegasearchServiceConcurrentClient::send_DescribeTable(const std::string& return cseqid; } -void MegasearchServiceConcurrentClient::recv_DescribeTable(TableSchema& _return, const int32_t seqid) +int64_t MegasearchServiceConcurrentClient::recv_GetTableRowCount(const int32_t seqid) { int32_t rseqid = 0; @@ -3596,7 +3293,7 @@ void MegasearchServiceConcurrentClient::recv_DescribeTable(TableSchema& _return, iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); } - if (fname.compare("DescribeTable") != 0) { + if (fname.compare("GetTableRowCount") != 0) { iprot_->skip(::apache::thrift::protocol::T_STRUCT); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); @@ -3605,23 +3302,23 @@ void MegasearchServiceConcurrentClient::recv_DescribeTable(TableSchema& _return, using ::apache::thrift::protocol::TProtocolException; throw TProtocolException(TProtocolException::INVALID_DATA); } - MegasearchService_DescribeTable_presult result; + int64_t _return; + MegasearchService_GetTableRowCount_presult result; result.success = &_return; result.read(iprot_); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); if (result.__isset.success) { - // _return pointer has now been filled sentry.commit(); - return; + return _return; } if (result.__isset.e) { sentry.commit(); throw result.e; } // in a bad state, don't commit - throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "DescribeTable failed: unknown result"); + throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "GetTableRowCount failed: unknown result"); } // seqid != rseqid this->sync_.updatePending(fname, mtype, rseqid); diff --git a/cpp/src/thrift/gen-cpp/MegasearchService.h b/cpp/src/thrift/gen-cpp/MegasearchService.h index 50e773970a7b3f90c427982b6da30d28fc0f7a34..60f1ddff624308b31bb032cb73c6cfa30b1a0890 100644 --- a/cpp/src/thrift/gen-cpp/MegasearchService.h +++ b/cpp/src/thrift/gen-cpp/MegasearchService.h @@ -46,30 +46,6 @@ class MegasearchServiceIf { */ virtual void DeleteTable(const std::string& table_name) = 0; - /** - * @brief Create table partition - * - * This method is used to create table partition. - * - * @param param, use to provide partition information to be created. - * - * - * @param param - */ - virtual void CreateTablePartition(const CreateTablePartitionParam& param) = 0; - - /** - * @brief Delete table partition - * - * This method is used to delete table partition. - * - * @param param, use to provide partition information to be deleted. - * - * - * @param param - */ - virtual void DeleteTablePartition(const DeleteTablePartitionParam& param) = 0; - /** * @brief Add vector array to table * @@ -92,22 +68,24 @@ class MegasearchServiceIf { * * @param table_name, table_name is queried. * @param query_record_array, all vector are going to be queried. + * @param query_range_array, optional ranges for conditional search. If not specified, search whole table * @param topk, how many similarity vectors will be searched. * * @return query result array. * * @param table_name * @param query_record_array + * @param query_range_array * @param topk */ - virtual void SearchVector(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const int64_t topk) = 0; + virtual void SearchVector(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) = 0; /** - * @brief Show table information + * @brief Get table schema * - * This method is used to show table information. + * This method is used to get table schema. * - * @param table_name, which table is show. + * @param table_name, target table name. * * @return table schema * @@ -115,6 +93,19 @@ class MegasearchServiceIf { */ virtual void DescribeTable(TableSchema& _return, const std::string& table_name) = 0; + /** + * @brief Get table row count + * + * This method is used to get table row count. + * + * @param table_name, target table name. + * + * @return table row count + * + * @param table_name + */ + virtual int64_t GetTableRowCount(const std::string& table_name) = 0; + /** * @brief List all tables in database * @@ -170,21 +161,19 @@ class MegasearchServiceNull : virtual public MegasearchServiceIf { void DeleteTable(const std::string& /* table_name */) { return; } - void CreateTablePartition(const CreateTablePartitionParam& /* param */) { - return; - } - void DeleteTablePartition(const DeleteTablePartitionParam& /* param */) { - return; - } void AddVector(std::vector & /* _return */, const std::string& /* table_name */, const std::vector & /* record_array */) { return; } - void SearchVector(std::vector & /* _return */, const std::string& /* table_name */, const std::vector & /* query_record_array */, const int64_t /* topk */) { + void SearchVector(std::vector & /* _return */, const std::string& /* table_name */, const std::vector & /* query_record_array */, const std::vector & /* query_range_array */, const int64_t /* topk */) { return; } void DescribeTable(TableSchema& /* _return */, const std::string& /* table_name */) { return; } + int64_t GetTableRowCount(const std::string& /* table_name */) { + int64_t _return = 0; + return _return; + } void ShowTables(std::vector & /* _return */) { return; } @@ -401,214 +390,6 @@ class MegasearchService_DeleteTable_presult { }; -typedef struct _MegasearchService_CreateTablePartition_args__isset { - _MegasearchService_CreateTablePartition_args__isset() : param(false) {} - bool param :1; -} _MegasearchService_CreateTablePartition_args__isset; - -class MegasearchService_CreateTablePartition_args { - public: - - MegasearchService_CreateTablePartition_args(const MegasearchService_CreateTablePartition_args&); - MegasearchService_CreateTablePartition_args& operator=(const MegasearchService_CreateTablePartition_args&); - MegasearchService_CreateTablePartition_args() { - } - - virtual ~MegasearchService_CreateTablePartition_args() throw(); - CreateTablePartitionParam param; - - _MegasearchService_CreateTablePartition_args__isset __isset; - - void __set_param(const CreateTablePartitionParam& val); - - bool operator == (const MegasearchService_CreateTablePartition_args & rhs) const - { - if (!(param == rhs.param)) - return false; - return true; - } - bool operator != (const MegasearchService_CreateTablePartition_args &rhs) const { - return !(*this == rhs); - } - - bool operator < (const MegasearchService_CreateTablePartition_args & ) const; - - uint32_t read(::apache::thrift::protocol::TProtocol* iprot); - uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; - -}; - - -class MegasearchService_CreateTablePartition_pargs { - public: - - - virtual ~MegasearchService_CreateTablePartition_pargs() throw(); - const CreateTablePartitionParam* param; - - uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; - -}; - -typedef struct _MegasearchService_CreateTablePartition_result__isset { - _MegasearchService_CreateTablePartition_result__isset() : e(false) {} - bool e :1; -} _MegasearchService_CreateTablePartition_result__isset; - -class MegasearchService_CreateTablePartition_result { - public: - - MegasearchService_CreateTablePartition_result(const MegasearchService_CreateTablePartition_result&); - MegasearchService_CreateTablePartition_result& operator=(const MegasearchService_CreateTablePartition_result&); - MegasearchService_CreateTablePartition_result() { - } - - virtual ~MegasearchService_CreateTablePartition_result() throw(); - Exception e; - - _MegasearchService_CreateTablePartition_result__isset __isset; - - void __set_e(const Exception& val); - - bool operator == (const MegasearchService_CreateTablePartition_result & rhs) const - { - if (!(e == rhs.e)) - return false; - return true; - } - bool operator != (const MegasearchService_CreateTablePartition_result &rhs) const { - return !(*this == rhs); - } - - bool operator < (const MegasearchService_CreateTablePartition_result & ) const; - - uint32_t read(::apache::thrift::protocol::TProtocol* iprot); - uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; - -}; - -typedef struct _MegasearchService_CreateTablePartition_presult__isset { - _MegasearchService_CreateTablePartition_presult__isset() : e(false) {} - bool e :1; -} _MegasearchService_CreateTablePartition_presult__isset; - -class MegasearchService_CreateTablePartition_presult { - public: - - - virtual ~MegasearchService_CreateTablePartition_presult() throw(); - Exception e; - - _MegasearchService_CreateTablePartition_presult__isset __isset; - - uint32_t read(::apache::thrift::protocol::TProtocol* iprot); - -}; - -typedef struct _MegasearchService_DeleteTablePartition_args__isset { - _MegasearchService_DeleteTablePartition_args__isset() : param(false) {} - bool param :1; -} _MegasearchService_DeleteTablePartition_args__isset; - -class MegasearchService_DeleteTablePartition_args { - public: - - MegasearchService_DeleteTablePartition_args(const MegasearchService_DeleteTablePartition_args&); - MegasearchService_DeleteTablePartition_args& operator=(const MegasearchService_DeleteTablePartition_args&); - MegasearchService_DeleteTablePartition_args() { - } - - virtual ~MegasearchService_DeleteTablePartition_args() throw(); - DeleteTablePartitionParam param; - - _MegasearchService_DeleteTablePartition_args__isset __isset; - - void __set_param(const DeleteTablePartitionParam& val); - - bool operator == (const MegasearchService_DeleteTablePartition_args & rhs) const - { - if (!(param == rhs.param)) - return false; - return true; - } - bool operator != (const MegasearchService_DeleteTablePartition_args &rhs) const { - return !(*this == rhs); - } - - bool operator < (const MegasearchService_DeleteTablePartition_args & ) const; - - uint32_t read(::apache::thrift::protocol::TProtocol* iprot); - uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; - -}; - - -class MegasearchService_DeleteTablePartition_pargs { - public: - - - virtual ~MegasearchService_DeleteTablePartition_pargs() throw(); - const DeleteTablePartitionParam* param; - - uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; - -}; - -typedef struct _MegasearchService_DeleteTablePartition_result__isset { - _MegasearchService_DeleteTablePartition_result__isset() : e(false) {} - bool e :1; -} _MegasearchService_DeleteTablePartition_result__isset; - -class MegasearchService_DeleteTablePartition_result { - public: - - MegasearchService_DeleteTablePartition_result(const MegasearchService_DeleteTablePartition_result&); - MegasearchService_DeleteTablePartition_result& operator=(const MegasearchService_DeleteTablePartition_result&); - MegasearchService_DeleteTablePartition_result() { - } - - virtual ~MegasearchService_DeleteTablePartition_result() throw(); - Exception e; - - _MegasearchService_DeleteTablePartition_result__isset __isset; - - void __set_e(const Exception& val); - - bool operator == (const MegasearchService_DeleteTablePartition_result & rhs) const - { - if (!(e == rhs.e)) - return false; - return true; - } - bool operator != (const MegasearchService_DeleteTablePartition_result &rhs) const { - return !(*this == rhs); - } - - bool operator < (const MegasearchService_DeleteTablePartition_result & ) const; - - uint32_t read(::apache::thrift::protocol::TProtocol* iprot); - uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; - -}; - -typedef struct _MegasearchService_DeleteTablePartition_presult__isset { - _MegasearchService_DeleteTablePartition_presult__isset() : e(false) {} - bool e :1; -} _MegasearchService_DeleteTablePartition_presult__isset; - -class MegasearchService_DeleteTablePartition_presult { - public: - - - virtual ~MegasearchService_DeleteTablePartition_presult() throw(); - Exception e; - - _MegasearchService_DeleteTablePartition_presult__isset __isset; - - uint32_t read(::apache::thrift::protocol::TProtocol* iprot); - -}; - typedef struct _MegasearchService_AddVector_args__isset { _MegasearchService_AddVector_args__isset() : table_name(false), record_array(false) {} bool table_name :1; @@ -729,9 +510,10 @@ class MegasearchService_AddVector_presult { }; typedef struct _MegasearchService_SearchVector_args__isset { - _MegasearchService_SearchVector_args__isset() : table_name(false), query_record_array(false), topk(false) {} + _MegasearchService_SearchVector_args__isset() : table_name(false), query_record_array(false), query_range_array(false), topk(false) {} bool table_name :1; bool query_record_array :1; + bool query_range_array :1; bool topk :1; } _MegasearchService_SearchVector_args__isset; @@ -745,14 +527,17 @@ class MegasearchService_SearchVector_args { virtual ~MegasearchService_SearchVector_args() throw(); std::string table_name; - std::vector query_record_array; + std::vector query_record_array; + std::vector query_range_array; int64_t topk; _MegasearchService_SearchVector_args__isset __isset; void __set_table_name(const std::string& val); - void __set_query_record_array(const std::vector & val); + void __set_query_record_array(const std::vector & val); + + void __set_query_range_array(const std::vector & val); void __set_topk(const int64_t val); @@ -762,6 +547,8 @@ class MegasearchService_SearchVector_args { return false; if (!(query_record_array == rhs.query_record_array)) return false; + if (!(query_range_array == rhs.query_range_array)) + return false; if (!(topk == rhs.topk)) return false; return true; @@ -784,7 +571,8 @@ class MegasearchService_SearchVector_pargs { virtual ~MegasearchService_SearchVector_pargs() throw(); const std::string* table_name; - const std::vector * query_record_array; + const std::vector * query_record_array; + const std::vector * query_range_array; const int64_t* topk; uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; @@ -966,6 +754,118 @@ class MegasearchService_DescribeTable_presult { }; +typedef struct _MegasearchService_GetTableRowCount_args__isset { + _MegasearchService_GetTableRowCount_args__isset() : table_name(false) {} + bool table_name :1; +} _MegasearchService_GetTableRowCount_args__isset; + +class MegasearchService_GetTableRowCount_args { + public: + + MegasearchService_GetTableRowCount_args(const MegasearchService_GetTableRowCount_args&); + MegasearchService_GetTableRowCount_args& operator=(const MegasearchService_GetTableRowCount_args&); + MegasearchService_GetTableRowCount_args() : table_name() { + } + + virtual ~MegasearchService_GetTableRowCount_args() throw(); + std::string table_name; + + _MegasearchService_GetTableRowCount_args__isset __isset; + + void __set_table_name(const std::string& val); + + bool operator == (const MegasearchService_GetTableRowCount_args & rhs) const + { + if (!(table_name == rhs.table_name)) + return false; + return true; + } + bool operator != (const MegasearchService_GetTableRowCount_args &rhs) const { + return !(*this == rhs); + } + + bool operator < (const MegasearchService_GetTableRowCount_args & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + + +class MegasearchService_GetTableRowCount_pargs { + public: + + + virtual ~MegasearchService_GetTableRowCount_pargs() throw(); + const std::string* table_name; + + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + +typedef struct _MegasearchService_GetTableRowCount_result__isset { + _MegasearchService_GetTableRowCount_result__isset() : success(false), e(false) {} + bool success :1; + bool e :1; +} _MegasearchService_GetTableRowCount_result__isset; + +class MegasearchService_GetTableRowCount_result { + public: + + MegasearchService_GetTableRowCount_result(const MegasearchService_GetTableRowCount_result&); + MegasearchService_GetTableRowCount_result& operator=(const MegasearchService_GetTableRowCount_result&); + MegasearchService_GetTableRowCount_result() : success(0) { + } + + virtual ~MegasearchService_GetTableRowCount_result() throw(); + int64_t success; + Exception e; + + _MegasearchService_GetTableRowCount_result__isset __isset; + + void __set_success(const int64_t val); + + void __set_e(const Exception& val); + + bool operator == (const MegasearchService_GetTableRowCount_result & rhs) const + { + if (!(success == rhs.success)) + return false; + if (!(e == rhs.e)) + return false; + return true; + } + bool operator != (const MegasearchService_GetTableRowCount_result &rhs) const { + return !(*this == rhs); + } + + bool operator < (const MegasearchService_GetTableRowCount_result & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + +typedef struct _MegasearchService_GetTableRowCount_presult__isset { + _MegasearchService_GetTableRowCount_presult__isset() : success(false), e(false) {} + bool success :1; + bool e :1; +} _MegasearchService_GetTableRowCount_presult__isset; + +class MegasearchService_GetTableRowCount_presult { + public: + + + virtual ~MegasearchService_GetTableRowCount_presult() throw(); + int64_t* success; + Exception e; + + _MegasearchService_GetTableRowCount_presult__isset __isset; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + +}; + class MegasearchService_ShowTables_args { public: @@ -1209,21 +1109,18 @@ class MegasearchServiceClient : virtual public MegasearchServiceIf { void DeleteTable(const std::string& table_name); void send_DeleteTable(const std::string& table_name); void recv_DeleteTable(); - void CreateTablePartition(const CreateTablePartitionParam& param); - void send_CreateTablePartition(const CreateTablePartitionParam& param); - void recv_CreateTablePartition(); - void DeleteTablePartition(const DeleteTablePartitionParam& param); - void send_DeleteTablePartition(const DeleteTablePartitionParam& param); - void recv_DeleteTablePartition(); void AddVector(std::vector & _return, const std::string& table_name, const std::vector & record_array); void send_AddVector(const std::string& table_name, const std::vector & record_array); void recv_AddVector(std::vector & _return); - void SearchVector(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const int64_t topk); - void send_SearchVector(const std::string& table_name, const std::vector & query_record_array, const int64_t topk); + void SearchVector(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); + void send_SearchVector(const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); void recv_SearchVector(std::vector & _return); void DescribeTable(TableSchema& _return, const std::string& table_name); void send_DescribeTable(const std::string& table_name); void recv_DescribeTable(TableSchema& _return); + int64_t GetTableRowCount(const std::string& table_name); + void send_GetTableRowCount(const std::string& table_name); + int64_t recv_GetTableRowCount(); void ShowTables(std::vector & _return); void send_ShowTables(); void recv_ShowTables(std::vector & _return); @@ -1247,11 +1144,10 @@ class MegasearchServiceProcessor : public ::apache::thrift::TDispatchProcessor { ProcessMap processMap_; void process_CreateTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_DeleteTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); - void process_CreateTablePartition(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); - void process_DeleteTablePartition(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_AddVector(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_SearchVector(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_DescribeTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); + void process_GetTableRowCount(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_ShowTables(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_Ping(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); public: @@ -1259,11 +1155,10 @@ class MegasearchServiceProcessor : public ::apache::thrift::TDispatchProcessor { iface_(iface) { processMap_["CreateTable"] = &MegasearchServiceProcessor::process_CreateTable; processMap_["DeleteTable"] = &MegasearchServiceProcessor::process_DeleteTable; - processMap_["CreateTablePartition"] = &MegasearchServiceProcessor::process_CreateTablePartition; - processMap_["DeleteTablePartition"] = &MegasearchServiceProcessor::process_DeleteTablePartition; processMap_["AddVector"] = &MegasearchServiceProcessor::process_AddVector; processMap_["SearchVector"] = &MegasearchServiceProcessor::process_SearchVector; processMap_["DescribeTable"] = &MegasearchServiceProcessor::process_DescribeTable; + processMap_["GetTableRowCount"] = &MegasearchServiceProcessor::process_GetTableRowCount; processMap_["ShowTables"] = &MegasearchServiceProcessor::process_ShowTables; processMap_["Ping"] = &MegasearchServiceProcessor::process_Ping; } @@ -1312,24 +1207,6 @@ class MegasearchServiceMultiface : virtual public MegasearchServiceIf { ifaces_[i]->DeleteTable(table_name); } - void CreateTablePartition(const CreateTablePartitionParam& param) { - size_t sz = ifaces_.size(); - size_t i = 0; - for (; i < (sz - 1); ++i) { - ifaces_[i]->CreateTablePartition(param); - } - ifaces_[i]->CreateTablePartition(param); - } - - void DeleteTablePartition(const DeleteTablePartitionParam& param) { - size_t sz = ifaces_.size(); - size_t i = 0; - for (; i < (sz - 1); ++i) { - ifaces_[i]->DeleteTablePartition(param); - } - ifaces_[i]->DeleteTablePartition(param); - } - void AddVector(std::vector & _return, const std::string& table_name, const std::vector & record_array) { size_t sz = ifaces_.size(); size_t i = 0; @@ -1340,13 +1217,13 @@ class MegasearchServiceMultiface : virtual public MegasearchServiceIf { return; } - void SearchVector(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const int64_t topk) { + void SearchVector(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) { size_t sz = ifaces_.size(); size_t i = 0; for (; i < (sz - 1); ++i) { - ifaces_[i]->SearchVector(_return, table_name, query_record_array, topk); + ifaces_[i]->SearchVector(_return, table_name, query_record_array, query_range_array, topk); } - ifaces_[i]->SearchVector(_return, table_name, query_record_array, topk); + ifaces_[i]->SearchVector(_return, table_name, query_record_array, query_range_array, topk); return; } @@ -1360,6 +1237,15 @@ class MegasearchServiceMultiface : virtual public MegasearchServiceIf { return; } + int64_t GetTableRowCount(const std::string& table_name) { + size_t sz = ifaces_.size(); + size_t i = 0; + for (; i < (sz - 1); ++i) { + ifaces_[i]->GetTableRowCount(table_name); + } + return ifaces_[i]->GetTableRowCount(table_name); + } + void ShowTables(std::vector & _return) { size_t sz = ifaces_.size(); size_t i = 0; @@ -1416,21 +1302,18 @@ class MegasearchServiceConcurrentClient : virtual public MegasearchServiceIf { void DeleteTable(const std::string& table_name); int32_t send_DeleteTable(const std::string& table_name); void recv_DeleteTable(const int32_t seqid); - void CreateTablePartition(const CreateTablePartitionParam& param); - int32_t send_CreateTablePartition(const CreateTablePartitionParam& param); - void recv_CreateTablePartition(const int32_t seqid); - void DeleteTablePartition(const DeleteTablePartitionParam& param); - int32_t send_DeleteTablePartition(const DeleteTablePartitionParam& param); - void recv_DeleteTablePartition(const int32_t seqid); void AddVector(std::vector & _return, const std::string& table_name, const std::vector & record_array); int32_t send_AddVector(const std::string& table_name, const std::vector & record_array); void recv_AddVector(std::vector & _return, const int32_t seqid); - void SearchVector(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const int64_t topk); - int32_t send_SearchVector(const std::string& table_name, const std::vector & query_record_array, const int64_t topk); + void SearchVector(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); + int32_t send_SearchVector(const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); void recv_SearchVector(std::vector & _return, const int32_t seqid); void DescribeTable(TableSchema& _return, const std::string& table_name); int32_t send_DescribeTable(const std::string& table_name); void recv_DescribeTable(TableSchema& _return, const int32_t seqid); + int64_t GetTableRowCount(const std::string& table_name); + int32_t send_GetTableRowCount(const std::string& table_name); + int64_t recv_GetTableRowCount(const int32_t seqid); void ShowTables(std::vector & _return); int32_t send_ShowTables(); void recv_ShowTables(std::vector & _return, const int32_t seqid); diff --git a/cpp/src/thrift/gen-cpp/MegasearchService_server.skeleton.cpp b/cpp/src/thrift/gen-cpp/MegasearchService_server.skeleton.cpp index f58e5556e7e55ae0365e8c9a6bc0960c7882ac7a..14b9e6f39a5d9c35be1475ee1cdecb0cb1583362 100644 --- a/cpp/src/thrift/gen-cpp/MegasearchService_server.skeleton.cpp +++ b/cpp/src/thrift/gen-cpp/MegasearchService_server.skeleton.cpp @@ -50,36 +50,6 @@ class MegasearchServiceHandler : virtual public MegasearchServiceIf { printf("DeleteTable\n"); } - /** - * @brief Create table partition - * - * This method is used to create table partition. - * - * @param param, use to provide partition information to be created. - * - * - * @param param - */ - void CreateTablePartition(const CreateTablePartitionParam& param) { - // Your implementation goes here - printf("CreateTablePartition\n"); - } - - /** - * @brief Delete table partition - * - * This method is used to delete table partition. - * - * @param param, use to provide partition information to be deleted. - * - * - * @param param - */ - void DeleteTablePartition(const DeleteTablePartitionParam& param) { - // Your implementation goes here - printf("DeleteTablePartition\n"); - } - /** * @brief Add vector array to table * @@ -105,25 +75,27 @@ class MegasearchServiceHandler : virtual public MegasearchServiceIf { * * @param table_name, table_name is queried. * @param query_record_array, all vector are going to be queried. + * @param query_range_array, optional ranges for conditional search. If not specified, search whole table * @param topk, how many similarity vectors will be searched. * * @return query result array. * * @param table_name * @param query_record_array + * @param query_range_array * @param topk */ - void SearchVector(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const int64_t topk) { + void SearchVector(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) { // Your implementation goes here printf("SearchVector\n"); } /** - * @brief Show table information + * @brief Get table schema * - * This method is used to show table information. + * This method is used to get table schema. * - * @param table_name, which table is show. + * @param table_name, target table name. * * @return table schema * @@ -134,6 +106,22 @@ class MegasearchServiceHandler : virtual public MegasearchServiceIf { printf("DescribeTable\n"); } + /** + * @brief Get table row count + * + * This method is used to get table row count. + * + * @param table_name, target table name. + * + * @return table row count + * + * @param table_name + */ + int64_t GetTableRowCount(const std::string& table_name) { + // Your implementation goes here + printf("GetTableRowCount\n"); + } + /** * @brief List all tables in database * diff --git a/cpp/src/thrift/gen-cpp/megasearch_types.cpp b/cpp/src/thrift/gen-cpp/megasearch_types.cpp index 5731477b11ef74522b08bd7b8e19952fee6029b8..13e0bed5fb522101e248c566341576b408ba9597 100644 --- a/cpp/src/thrift/gen-cpp/megasearch_types.cpp +++ b/cpp/src/thrift/gen-cpp/megasearch_types.cpp @@ -18,7 +18,6 @@ int _kErrorCodeValues[] = { ErrorCode::CONNECT_FAILED, ErrorCode::PERMISSION_DENIED, ErrorCode::TABLE_NOT_EXISTS, - ErrorCode::PARTITION_NOT_EXIST, ErrorCode::ILLEGAL_ARGUMENT, ErrorCode::ILLEGAL_RANGE, ErrorCode::ILLEGAL_DIMENSION @@ -28,12 +27,11 @@ const char* _kErrorCodeNames[] = { "CONNECT_FAILED", "PERMISSION_DENIED", "TABLE_NOT_EXISTS", - "PARTITION_NOT_EXIST", "ILLEGAL_ARGUMENT", "ILLEGAL_RANGE", "ILLEGAL_DIMENSION" }; -const std::map _ErrorCode_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(8, _kErrorCodeValues, _kErrorCodeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL)); +const std::map _ErrorCode_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(7, _kErrorCodeValues, _kErrorCodeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL)); std::ostream& operator<<(std::ostream& out, const ErrorCode::type& val) { std::map::const_iterator it = _ErrorCode_VALUES_TO_NAMES.find(val); @@ -171,642 +169,33 @@ const char* Exception::what() const throw() { } -Column::~Column() throw() { -} - - -void Column::__set_type(const int32_t val) { - this->type = val; -} - -void Column::__set_name(const std::string& val) { - this->name = val; -} -std::ostream& operator<<(std::ostream& out, const Column& obj) -{ - obj.printTo(out); - return out; -} - - -uint32_t Column::read(::apache::thrift::protocol::TProtocol* iprot) { - - ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); - uint32_t xfer = 0; - std::string fname; - ::apache::thrift::protocol::TType ftype; - int16_t fid; - - xfer += iprot->readStructBegin(fname); - - using ::apache::thrift::protocol::TProtocolException; - - bool isset_type = false; - bool isset_name = false; - - while (true) - { - xfer += iprot->readFieldBegin(fname, ftype, fid); - if (ftype == ::apache::thrift::protocol::T_STOP) { - break; - } - switch (fid) - { - case 1: - if (ftype == ::apache::thrift::protocol::T_I32) { - xfer += iprot->readI32(this->type); - isset_type = true; - } else { - xfer += iprot->skip(ftype); - } - break; - case 2: - if (ftype == ::apache::thrift::protocol::T_STRING) { - xfer += iprot->readString(this->name); - isset_name = true; - } else { - xfer += iprot->skip(ftype); - } - break; - default: - xfer += iprot->skip(ftype); - break; - } - xfer += iprot->readFieldEnd(); - } - - xfer += iprot->readStructEnd(); - - if (!isset_type) - throw TProtocolException(TProtocolException::INVALID_DATA); - if (!isset_name) - throw TProtocolException(TProtocolException::INVALID_DATA); - return xfer; -} - -uint32_t Column::write(::apache::thrift::protocol::TProtocol* oprot) const { - uint32_t xfer = 0; - ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); - xfer += oprot->writeStructBegin("Column"); - - xfer += oprot->writeFieldBegin("type", ::apache::thrift::protocol::T_I32, 1); - xfer += oprot->writeI32(this->type); - xfer += oprot->writeFieldEnd(); - - xfer += oprot->writeFieldBegin("name", ::apache::thrift::protocol::T_STRING, 2); - xfer += oprot->writeString(this->name); - xfer += oprot->writeFieldEnd(); - - xfer += oprot->writeFieldStop(); - xfer += oprot->writeStructEnd(); - return xfer; -} - -void swap(Column &a, Column &b) { - using ::std::swap; - swap(a.type, b.type); - swap(a.name, b.name); -} - -Column::Column(const Column& other3) { - type = other3.type; - name = other3.name; -} -Column& Column::operator=(const Column& other4) { - type = other4.type; - name = other4.name; - return *this; -} -void Column::printTo(std::ostream& out) const { - using ::apache::thrift::to_string; - out << "Column("; - out << "type=" << to_string(type); - out << ", " << "name=" << to_string(name); - out << ")"; -} - - -VectorColumn::~VectorColumn() throw() { -} - - -void VectorColumn::__set_base(const Column& val) { - this->base = val; -} - -void VectorColumn::__set_dimension(const int64_t val) { - this->dimension = val; -} - -void VectorColumn::__set_index_type(const std::string& val) { - this->index_type = val; -} - -void VectorColumn::__set_store_raw_vector(const bool val) { - this->store_raw_vector = val; -} -std::ostream& operator<<(std::ostream& out, const VectorColumn& obj) -{ - obj.printTo(out); - return out; -} - - -uint32_t VectorColumn::read(::apache::thrift::protocol::TProtocol* iprot) { - - ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); - uint32_t xfer = 0; - std::string fname; - ::apache::thrift::protocol::TType ftype; - int16_t fid; - - xfer += iprot->readStructBegin(fname); - - using ::apache::thrift::protocol::TProtocolException; - - bool isset_base = false; - bool isset_dimension = false; - bool isset_index_type = false; - - while (true) - { - xfer += iprot->readFieldBegin(fname, ftype, fid); - if (ftype == ::apache::thrift::protocol::T_STOP) { - break; - } - switch (fid) - { - case 1: - if (ftype == ::apache::thrift::protocol::T_STRUCT) { - xfer += this->base.read(iprot); - isset_base = true; - } else { - xfer += iprot->skip(ftype); - } - break; - case 2: - if (ftype == ::apache::thrift::protocol::T_I64) { - xfer += iprot->readI64(this->dimension); - isset_dimension = true; - } else { - xfer += iprot->skip(ftype); - } - break; - case 3: - if (ftype == ::apache::thrift::protocol::T_STRING) { - xfer += iprot->readString(this->index_type); - isset_index_type = true; - } else { - xfer += iprot->skip(ftype); - } - break; - case 4: - if (ftype == ::apache::thrift::protocol::T_BOOL) { - xfer += iprot->readBool(this->store_raw_vector); - this->__isset.store_raw_vector = true; - } else { - xfer += iprot->skip(ftype); - } - break; - default: - xfer += iprot->skip(ftype); - break; - } - xfer += iprot->readFieldEnd(); - } - - xfer += iprot->readStructEnd(); - - if (!isset_base) - throw TProtocolException(TProtocolException::INVALID_DATA); - if (!isset_dimension) - throw TProtocolException(TProtocolException::INVALID_DATA); - if (!isset_index_type) - throw TProtocolException(TProtocolException::INVALID_DATA); - return xfer; -} - -uint32_t VectorColumn::write(::apache::thrift::protocol::TProtocol* oprot) const { - uint32_t xfer = 0; - ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); - xfer += oprot->writeStructBegin("VectorColumn"); - - xfer += oprot->writeFieldBegin("base", ::apache::thrift::protocol::T_STRUCT, 1); - xfer += this->base.write(oprot); - xfer += oprot->writeFieldEnd(); - - xfer += oprot->writeFieldBegin("dimension", ::apache::thrift::protocol::T_I64, 2); - xfer += oprot->writeI64(this->dimension); - xfer += oprot->writeFieldEnd(); - - xfer += oprot->writeFieldBegin("index_type", ::apache::thrift::protocol::T_STRING, 3); - xfer += oprot->writeString(this->index_type); - xfer += oprot->writeFieldEnd(); - - xfer += oprot->writeFieldBegin("store_raw_vector", ::apache::thrift::protocol::T_BOOL, 4); - xfer += oprot->writeBool(this->store_raw_vector); - xfer += oprot->writeFieldEnd(); - - xfer += oprot->writeFieldStop(); - xfer += oprot->writeStructEnd(); - return xfer; -} - -void swap(VectorColumn &a, VectorColumn &b) { - using ::std::swap; - swap(a.base, b.base); - swap(a.dimension, b.dimension); - swap(a.index_type, b.index_type); - swap(a.store_raw_vector, b.store_raw_vector); - swap(a.__isset, b.__isset); -} - -VectorColumn::VectorColumn(const VectorColumn& other5) { - base = other5.base; - dimension = other5.dimension; - index_type = other5.index_type; - store_raw_vector = other5.store_raw_vector; - __isset = other5.__isset; -} -VectorColumn& VectorColumn::operator=(const VectorColumn& other6) { - base = other6.base; - dimension = other6.dimension; - index_type = other6.index_type; - store_raw_vector = other6.store_raw_vector; - __isset = other6.__isset; - return *this; -} -void VectorColumn::printTo(std::ostream& out) const { - using ::apache::thrift::to_string; - out << "VectorColumn("; - out << "base=" << to_string(base); - out << ", " << "dimension=" << to_string(dimension); - out << ", " << "index_type=" << to_string(index_type); - out << ", " << "store_raw_vector=" << to_string(store_raw_vector); - out << ")"; -} - - -TableSchema::~TableSchema() throw() { -} - - -void TableSchema::__set_table_name(const std::string& val) { - this->table_name = val; -} - -void TableSchema::__set_vector_column_array(const std::vector & val) { - this->vector_column_array = val; -} - -void TableSchema::__set_attribute_column_array(const std::vector & val) { - this->attribute_column_array = val; -__isset.attribute_column_array = true; -} - -void TableSchema::__set_partition_column_name_array(const std::vector & val) { - this->partition_column_name_array = val; -__isset.partition_column_name_array = true; -} -std::ostream& operator<<(std::ostream& out, const TableSchema& obj) -{ - obj.printTo(out); - return out; -} - - -uint32_t TableSchema::read(::apache::thrift::protocol::TProtocol* iprot) { - - ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); - uint32_t xfer = 0; - std::string fname; - ::apache::thrift::protocol::TType ftype; - int16_t fid; - - xfer += iprot->readStructBegin(fname); - - using ::apache::thrift::protocol::TProtocolException; - - bool isset_table_name = false; - bool isset_vector_column_array = false; - - while (true) - { - xfer += iprot->readFieldBegin(fname, ftype, fid); - if (ftype == ::apache::thrift::protocol::T_STOP) { - break; - } - switch (fid) - { - case 1: - if (ftype == ::apache::thrift::protocol::T_STRING) { - xfer += iprot->readString(this->table_name); - isset_table_name = true; - } else { - xfer += iprot->skip(ftype); - } - break; - case 2: - if (ftype == ::apache::thrift::protocol::T_LIST) { - { - this->vector_column_array.clear(); - uint32_t _size7; - ::apache::thrift::protocol::TType _etype10; - xfer += iprot->readListBegin(_etype10, _size7); - this->vector_column_array.resize(_size7); - uint32_t _i11; - for (_i11 = 0; _i11 < _size7; ++_i11) - { - xfer += this->vector_column_array[_i11].read(iprot); - } - xfer += iprot->readListEnd(); - } - isset_vector_column_array = true; - } else { - xfer += iprot->skip(ftype); - } - break; - case 3: - if (ftype == ::apache::thrift::protocol::T_LIST) { - { - this->attribute_column_array.clear(); - uint32_t _size12; - ::apache::thrift::protocol::TType _etype15; - xfer += iprot->readListBegin(_etype15, _size12); - this->attribute_column_array.resize(_size12); - uint32_t _i16; - for (_i16 = 0; _i16 < _size12; ++_i16) - { - xfer += this->attribute_column_array[_i16].read(iprot); - } - xfer += iprot->readListEnd(); - } - this->__isset.attribute_column_array = true; - } else { - xfer += iprot->skip(ftype); - } - break; - case 4: - if (ftype == ::apache::thrift::protocol::T_LIST) { - { - this->partition_column_name_array.clear(); - uint32_t _size17; - ::apache::thrift::protocol::TType _etype20; - xfer += iprot->readListBegin(_etype20, _size17); - this->partition_column_name_array.resize(_size17); - uint32_t _i21; - for (_i21 = 0; _i21 < _size17; ++_i21) - { - xfer += iprot->readString(this->partition_column_name_array[_i21]); - } - xfer += iprot->readListEnd(); - } - this->__isset.partition_column_name_array = true; - } else { - xfer += iprot->skip(ftype); - } - break; - default: - xfer += iprot->skip(ftype); - break; - } - xfer += iprot->readFieldEnd(); - } - - xfer += iprot->readStructEnd(); - - if (!isset_table_name) - throw TProtocolException(TProtocolException::INVALID_DATA); - if (!isset_vector_column_array) - throw TProtocolException(TProtocolException::INVALID_DATA); - return xfer; -} - -uint32_t TableSchema::write(::apache::thrift::protocol::TProtocol* oprot) const { - uint32_t xfer = 0; - ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); - xfer += oprot->writeStructBegin("TableSchema"); - - xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 1); - xfer += oprot->writeString(this->table_name); - xfer += oprot->writeFieldEnd(); - - xfer += oprot->writeFieldBegin("vector_column_array", ::apache::thrift::protocol::T_LIST, 2); - { - xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->vector_column_array.size())); - std::vector ::const_iterator _iter22; - for (_iter22 = this->vector_column_array.begin(); _iter22 != this->vector_column_array.end(); ++_iter22) - { - xfer += (*_iter22).write(oprot); - } - xfer += oprot->writeListEnd(); - } - xfer += oprot->writeFieldEnd(); - - if (this->__isset.attribute_column_array) { - xfer += oprot->writeFieldBegin("attribute_column_array", ::apache::thrift::protocol::T_LIST, 3); - { - xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->attribute_column_array.size())); - std::vector ::const_iterator _iter23; - for (_iter23 = this->attribute_column_array.begin(); _iter23 != this->attribute_column_array.end(); ++_iter23) - { - xfer += (*_iter23).write(oprot); - } - xfer += oprot->writeListEnd(); - } - xfer += oprot->writeFieldEnd(); - } - if (this->__isset.partition_column_name_array) { - xfer += oprot->writeFieldBegin("partition_column_name_array", ::apache::thrift::protocol::T_LIST, 4); - { - xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->partition_column_name_array.size())); - std::vector ::const_iterator _iter24; - for (_iter24 = this->partition_column_name_array.begin(); _iter24 != this->partition_column_name_array.end(); ++_iter24) - { - xfer += oprot->writeString((*_iter24)); - } - xfer += oprot->writeListEnd(); - } - xfer += oprot->writeFieldEnd(); - } - xfer += oprot->writeFieldStop(); - xfer += oprot->writeStructEnd(); - return xfer; -} - -void swap(TableSchema &a, TableSchema &b) { - using ::std::swap; - swap(a.table_name, b.table_name); - swap(a.vector_column_array, b.vector_column_array); - swap(a.attribute_column_array, b.attribute_column_array); - swap(a.partition_column_name_array, b.partition_column_name_array); - swap(a.__isset, b.__isset); -} - -TableSchema::TableSchema(const TableSchema& other25) { - table_name = other25.table_name; - vector_column_array = other25.vector_column_array; - attribute_column_array = other25.attribute_column_array; - partition_column_name_array = other25.partition_column_name_array; - __isset = other25.__isset; -} -TableSchema& TableSchema::operator=(const TableSchema& other26) { - table_name = other26.table_name; - vector_column_array = other26.vector_column_array; - attribute_column_array = other26.attribute_column_array; - partition_column_name_array = other26.partition_column_name_array; - __isset = other26.__isset; - return *this; -} -void TableSchema::printTo(std::ostream& out) const { - using ::apache::thrift::to_string; - out << "TableSchema("; - out << "table_name=" << to_string(table_name); - out << ", " << "vector_column_array=" << to_string(vector_column_array); - out << ", " << "attribute_column_array="; (__isset.attribute_column_array ? (out << to_string(attribute_column_array)) : (out << "")); - out << ", " << "partition_column_name_array="; (__isset.partition_column_name_array ? (out << to_string(partition_column_name_array)) : (out << "")); - out << ")"; -} - - -Range::~Range() throw() { -} - - -void Range::__set_start_value(const std::string& val) { - this->start_value = val; -} - -void Range::__set_end_value(const std::string& val) { - this->end_value = val; -} -std::ostream& operator<<(std::ostream& out, const Range& obj) -{ - obj.printTo(out); - return out; -} - - -uint32_t Range::read(::apache::thrift::protocol::TProtocol* iprot) { - - ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); - uint32_t xfer = 0; - std::string fname; - ::apache::thrift::protocol::TType ftype; - int16_t fid; - - xfer += iprot->readStructBegin(fname); - - using ::apache::thrift::protocol::TProtocolException; - - bool isset_start_value = false; - bool isset_end_value = false; - - while (true) - { - xfer += iprot->readFieldBegin(fname, ftype, fid); - if (ftype == ::apache::thrift::protocol::T_STOP) { - break; - } - switch (fid) - { - case 1: - if (ftype == ::apache::thrift::protocol::T_STRING) { - xfer += iprot->readString(this->start_value); - isset_start_value = true; - } else { - xfer += iprot->skip(ftype); - } - break; - case 2: - if (ftype == ::apache::thrift::protocol::T_STRING) { - xfer += iprot->readString(this->end_value); - isset_end_value = true; - } else { - xfer += iprot->skip(ftype); - } - break; - default: - xfer += iprot->skip(ftype); - break; - } - xfer += iprot->readFieldEnd(); - } - - xfer += iprot->readStructEnd(); - - if (!isset_start_value) - throw TProtocolException(TProtocolException::INVALID_DATA); - if (!isset_end_value) - throw TProtocolException(TProtocolException::INVALID_DATA); - return xfer; -} - -uint32_t Range::write(::apache::thrift::protocol::TProtocol* oprot) const { - uint32_t xfer = 0; - ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); - xfer += oprot->writeStructBegin("Range"); - - xfer += oprot->writeFieldBegin("start_value", ::apache::thrift::protocol::T_STRING, 1); - xfer += oprot->writeString(this->start_value); - xfer += oprot->writeFieldEnd(); - - xfer += oprot->writeFieldBegin("end_value", ::apache::thrift::protocol::T_STRING, 2); - xfer += oprot->writeString(this->end_value); - xfer += oprot->writeFieldEnd(); - - xfer += oprot->writeFieldStop(); - xfer += oprot->writeStructEnd(); - return xfer; -} - -void swap(Range &a, Range &b) { - using ::std::swap; - swap(a.start_value, b.start_value); - swap(a.end_value, b.end_value); -} - -Range::Range(const Range& other27) { - start_value = other27.start_value; - end_value = other27.end_value; -} -Range& Range::operator=(const Range& other28) { - start_value = other28.start_value; - end_value = other28.end_value; - return *this; -} -void Range::printTo(std::ostream& out) const { - using ::apache::thrift::to_string; - out << "Range("; - out << "start_value=" << to_string(start_value); - out << ", " << "end_value=" << to_string(end_value); - out << ")"; +TableSchema::~TableSchema() throw() { } -CreateTablePartitionParam::~CreateTablePartitionParam() throw() { +void TableSchema::__set_table_name(const std::string& val) { + this->table_name = val; } - -void CreateTablePartitionParam::__set_table_name(const std::string& val) { - this->table_name = val; +void TableSchema::__set_index_type(const int32_t val) { + this->index_type = val; } -void CreateTablePartitionParam::__set_partition_name(const std::string& val) { - this->partition_name = val; +void TableSchema::__set_dimension(const int64_t val) { + this->dimension = val; } -void CreateTablePartitionParam::__set_range_map(const std::map & val) { - this->range_map = val; +void TableSchema::__set_store_raw_vector(const bool val) { + this->store_raw_vector = val; } -std::ostream& operator<<(std::ostream& out, const CreateTablePartitionParam& obj) +std::ostream& operator<<(std::ostream& out, const TableSchema& obj) { obj.printTo(out); return out; } -uint32_t CreateTablePartitionParam::read(::apache::thrift::protocol::TProtocol* iprot) { +uint32_t TableSchema::read(::apache::thrift::protocol::TProtocol* iprot) { ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); uint32_t xfer = 0; @@ -819,8 +208,6 @@ uint32_t CreateTablePartitionParam::read(::apache::thrift::protocol::TProtocol* using ::apache::thrift::protocol::TProtocolException; bool isset_table_name = false; - bool isset_partition_name = false; - bool isset_range_map = false; while (true) { @@ -839,32 +226,25 @@ uint32_t CreateTablePartitionParam::read(::apache::thrift::protocol::TProtocol* } break; case 2: - if (ftype == ::apache::thrift::protocol::T_STRING) { - xfer += iprot->readString(this->partition_name); - isset_partition_name = true; + if (ftype == ::apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->index_type); + this->__isset.index_type = true; } else { xfer += iprot->skip(ftype); } break; case 3: - if (ftype == ::apache::thrift::protocol::T_MAP) { - { - this->range_map.clear(); - uint32_t _size29; - ::apache::thrift::protocol::TType _ktype30; - ::apache::thrift::protocol::TType _vtype31; - xfer += iprot->readMapBegin(_ktype30, _vtype31, _size29); - uint32_t _i33; - for (_i33 = 0; _i33 < _size29; ++_i33) - { - std::string _key34; - xfer += iprot->readString(_key34); - Range& _val35 = this->range_map[_key34]; - xfer += _val35.read(iprot); - } - xfer += iprot->readMapEnd(); - } - isset_range_map = true; + if (ftype == ::apache::thrift::protocol::T_I64) { + xfer += iprot->readI64(this->dimension); + this->__isset.dimension = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 4: + if (ftype == ::apache::thrift::protocol::T_BOOL) { + xfer += iprot->readBool(this->store_raw_vector); + this->__isset.store_raw_vector = true; } else { xfer += iprot->skip(ftype); } @@ -880,37 +260,28 @@ uint32_t CreateTablePartitionParam::read(::apache::thrift::protocol::TProtocol* if (!isset_table_name) throw TProtocolException(TProtocolException::INVALID_DATA); - if (!isset_partition_name) - throw TProtocolException(TProtocolException::INVALID_DATA); - if (!isset_range_map) - throw TProtocolException(TProtocolException::INVALID_DATA); return xfer; } -uint32_t CreateTablePartitionParam::write(::apache::thrift::protocol::TProtocol* oprot) const { +uint32_t TableSchema::write(::apache::thrift::protocol::TProtocol* oprot) const { uint32_t xfer = 0; ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); - xfer += oprot->writeStructBegin("CreateTablePartitionParam"); + xfer += oprot->writeStructBegin("TableSchema"); xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 1); xfer += oprot->writeString(this->table_name); xfer += oprot->writeFieldEnd(); - xfer += oprot->writeFieldBegin("partition_name", ::apache::thrift::protocol::T_STRING, 2); - xfer += oprot->writeString(this->partition_name); + xfer += oprot->writeFieldBegin("index_type", ::apache::thrift::protocol::T_I32, 2); + xfer += oprot->writeI32(this->index_type); xfer += oprot->writeFieldEnd(); - xfer += oprot->writeFieldBegin("range_map", ::apache::thrift::protocol::T_MAP, 3); - { - xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_STRUCT, static_cast(this->range_map.size())); - std::map ::const_iterator _iter36; - for (_iter36 = this->range_map.begin(); _iter36 != this->range_map.end(); ++_iter36) - { - xfer += oprot->writeString(_iter36->first); - xfer += _iter36->second.write(oprot); - } - xfer += oprot->writeMapEnd(); - } + xfer += oprot->writeFieldBegin("dimension", ::apache::thrift::protocol::T_I64, 3); + xfer += oprot->writeI64(this->dimension); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("store_raw_vector", ::apache::thrift::protocol::T_BOOL, 4); + xfer += oprot->writeBool(this->store_raw_vector); xfer += oprot->writeFieldEnd(); xfer += oprot->writeFieldStop(); @@ -918,53 +289,60 @@ uint32_t CreateTablePartitionParam::write(::apache::thrift::protocol::TProtocol* return xfer; } -void swap(CreateTablePartitionParam &a, CreateTablePartitionParam &b) { +void swap(TableSchema &a, TableSchema &b) { using ::std::swap; swap(a.table_name, b.table_name); - swap(a.partition_name, b.partition_name); - swap(a.range_map, b.range_map); + swap(a.index_type, b.index_type); + swap(a.dimension, b.dimension); + swap(a.store_raw_vector, b.store_raw_vector); + swap(a.__isset, b.__isset); } -CreateTablePartitionParam::CreateTablePartitionParam(const CreateTablePartitionParam& other37) { - table_name = other37.table_name; - partition_name = other37.partition_name; - range_map = other37.range_map; +TableSchema::TableSchema(const TableSchema& other3) { + table_name = other3.table_name; + index_type = other3.index_type; + dimension = other3.dimension; + store_raw_vector = other3.store_raw_vector; + __isset = other3.__isset; } -CreateTablePartitionParam& CreateTablePartitionParam::operator=(const CreateTablePartitionParam& other38) { - table_name = other38.table_name; - partition_name = other38.partition_name; - range_map = other38.range_map; +TableSchema& TableSchema::operator=(const TableSchema& other4) { + table_name = other4.table_name; + index_type = other4.index_type; + dimension = other4.dimension; + store_raw_vector = other4.store_raw_vector; + __isset = other4.__isset; return *this; } -void CreateTablePartitionParam::printTo(std::ostream& out) const { +void TableSchema::printTo(std::ostream& out) const { using ::apache::thrift::to_string; - out << "CreateTablePartitionParam("; + out << "TableSchema("; out << "table_name=" << to_string(table_name); - out << ", " << "partition_name=" << to_string(partition_name); - out << ", " << "range_map=" << to_string(range_map); + out << ", " << "index_type=" << to_string(index_type); + out << ", " << "dimension=" << to_string(dimension); + out << ", " << "store_raw_vector=" << to_string(store_raw_vector); out << ")"; } -DeleteTablePartitionParam::~DeleteTablePartitionParam() throw() { +Range::~Range() throw() { } -void DeleteTablePartitionParam::__set_table_name(const std::string& val) { - this->table_name = val; +void Range::__set_start_value(const std::string& val) { + this->start_value = val; } -void DeleteTablePartitionParam::__set_partition_name_array(const std::vector & val) { - this->partition_name_array = val; +void Range::__set_end_value(const std::string& val) { + this->end_value = val; } -std::ostream& operator<<(std::ostream& out, const DeleteTablePartitionParam& obj) +std::ostream& operator<<(std::ostream& out, const Range& obj) { obj.printTo(out); return out; } -uint32_t DeleteTablePartitionParam::read(::apache::thrift::protocol::TProtocol* iprot) { +uint32_t Range::read(::apache::thrift::protocol::TProtocol* iprot) { ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); uint32_t xfer = 0; @@ -976,8 +354,6 @@ uint32_t DeleteTablePartitionParam::read(::apache::thrift::protocol::TProtocol* using ::apache::thrift::protocol::TProtocolException; - bool isset_table_name = false; - bool isset_partition_name_array = false; while (true) { @@ -989,28 +365,16 @@ uint32_t DeleteTablePartitionParam::read(::apache::thrift::protocol::TProtocol* { case 1: if (ftype == ::apache::thrift::protocol::T_STRING) { - xfer += iprot->readString(this->table_name); - isset_table_name = true; + xfer += iprot->readString(this->start_value); + this->__isset.start_value = true; } else { xfer += iprot->skip(ftype); } break; case 2: - if (ftype == ::apache::thrift::protocol::T_LIST) { - { - this->partition_name_array.clear(); - uint32_t _size39; - ::apache::thrift::protocol::TType _etype42; - xfer += iprot->readListBegin(_etype42, _size39); - this->partition_name_array.resize(_size39); - uint32_t _i43; - for (_i43 = 0; _i43 < _size39; ++_i43) - { - xfer += iprot->readString(this->partition_name_array[_i43]); - } - xfer += iprot->readListEnd(); - } - isset_partition_name_array = true; + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(this->end_value); + this->__isset.end_value = true; } else { xfer += iprot->skip(ftype); } @@ -1024,32 +388,20 @@ uint32_t DeleteTablePartitionParam::read(::apache::thrift::protocol::TProtocol* xfer += iprot->readStructEnd(); - if (!isset_table_name) - throw TProtocolException(TProtocolException::INVALID_DATA); - if (!isset_partition_name_array) - throw TProtocolException(TProtocolException::INVALID_DATA); return xfer; } -uint32_t DeleteTablePartitionParam::write(::apache::thrift::protocol::TProtocol* oprot) const { +uint32_t Range::write(::apache::thrift::protocol::TProtocol* oprot) const { uint32_t xfer = 0; ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); - xfer += oprot->writeStructBegin("DeleteTablePartitionParam"); + xfer += oprot->writeStructBegin("Range"); - xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 1); - xfer += oprot->writeString(this->table_name); + xfer += oprot->writeFieldBegin("start_value", ::apache::thrift::protocol::T_STRING, 1); + xfer += oprot->writeString(this->start_value); xfer += oprot->writeFieldEnd(); - xfer += oprot->writeFieldBegin("partition_name_array", ::apache::thrift::protocol::T_LIST, 2); - { - xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->partition_name_array.size())); - std::vector ::const_iterator _iter44; - for (_iter44 = this->partition_name_array.begin(); _iter44 != this->partition_name_array.end(); ++_iter44) - { - xfer += oprot->writeString((*_iter44)); - } - xfer += oprot->writeListEnd(); - } + xfer += oprot->writeFieldBegin("end_value", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeString(this->end_value); xfer += oprot->writeFieldEnd(); xfer += oprot->writeFieldStop(); @@ -1057,26 +409,29 @@ uint32_t DeleteTablePartitionParam::write(::apache::thrift::protocol::TProtocol* return xfer; } -void swap(DeleteTablePartitionParam &a, DeleteTablePartitionParam &b) { +void swap(Range &a, Range &b) { using ::std::swap; - swap(a.table_name, b.table_name); - swap(a.partition_name_array, b.partition_name_array); + swap(a.start_value, b.start_value); + swap(a.end_value, b.end_value); + swap(a.__isset, b.__isset); } -DeleteTablePartitionParam::DeleteTablePartitionParam(const DeleteTablePartitionParam& other45) { - table_name = other45.table_name; - partition_name_array = other45.partition_name_array; +Range::Range(const Range& other5) { + start_value = other5.start_value; + end_value = other5.end_value; + __isset = other5.__isset; } -DeleteTablePartitionParam& DeleteTablePartitionParam::operator=(const DeleteTablePartitionParam& other46) { - table_name = other46.table_name; - partition_name_array = other46.partition_name_array; +Range& Range::operator=(const Range& other6) { + start_value = other6.start_value; + end_value = other6.end_value; + __isset = other6.__isset; return *this; } -void DeleteTablePartitionParam::printTo(std::ostream& out) const { +void Range::printTo(std::ostream& out) const { using ::apache::thrift::to_string; - out << "DeleteTablePartitionParam("; - out << "table_name=" << to_string(table_name); - out << ", " << "partition_name_array=" << to_string(partition_name_array); + out << "Range("; + out << "start_value=" << to_string(start_value); + out << ", " << "end_value=" << to_string(end_value); out << ")"; } @@ -1085,12 +440,8 @@ RowRecord::~RowRecord() throw() { } -void RowRecord::__set_vector_map(const std::map & val) { - this->vector_map = val; -} - -void RowRecord::__set_attribute_map(const std::map & val) { - this->attribute_map = val; +void RowRecord::__set_vector_data(const std::string& val) { + this->vector_data = val; } std::ostream& operator<<(std::ostream& out, const RowRecord& obj) { @@ -1111,7 +462,7 @@ uint32_t RowRecord::read(::apache::thrift::protocol::TProtocol* iprot) { using ::apache::thrift::protocol::TProtocolException; - bool isset_vector_map = false; + bool isset_vector_data = false; while (true) { @@ -1122,47 +473,9 @@ uint32_t RowRecord::read(::apache::thrift::protocol::TProtocol* iprot) { switch (fid) { case 1: - if (ftype == ::apache::thrift::protocol::T_MAP) { - { - this->vector_map.clear(); - uint32_t _size47; - ::apache::thrift::protocol::TType _ktype48; - ::apache::thrift::protocol::TType _vtype49; - xfer += iprot->readMapBegin(_ktype48, _vtype49, _size47); - uint32_t _i51; - for (_i51 = 0; _i51 < _size47; ++_i51) - { - std::string _key52; - xfer += iprot->readString(_key52); - std::string& _val53 = this->vector_map[_key52]; - xfer += iprot->readBinary(_val53); - } - xfer += iprot->readMapEnd(); - } - isset_vector_map = true; - } else { - xfer += iprot->skip(ftype); - } - break; - case 2: - if (ftype == ::apache::thrift::protocol::T_MAP) { - { - this->attribute_map.clear(); - uint32_t _size54; - ::apache::thrift::protocol::TType _ktype55; - ::apache::thrift::protocol::TType _vtype56; - xfer += iprot->readMapBegin(_ktype55, _vtype56, _size54); - uint32_t _i58; - for (_i58 = 0; _i58 < _size54; ++_i58) - { - std::string _key59; - xfer += iprot->readString(_key59); - std::string& _val60 = this->attribute_map[_key59]; - xfer += iprot->readString(_val60); - } - xfer += iprot->readMapEnd(); - } - this->__isset.attribute_map = true; + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readBinary(this->vector_data); + isset_vector_data = true; } else { xfer += iprot->skip(ftype); } @@ -1176,7 +489,7 @@ uint32_t RowRecord::read(::apache::thrift::protocol::TProtocol* iprot) { xfer += iprot->readStructEnd(); - if (!isset_vector_map) + if (!isset_vector_data) throw TProtocolException(TProtocolException::INVALID_DATA); return xfer; } @@ -1186,30 +499,8 @@ uint32_t RowRecord::write(::apache::thrift::protocol::TProtocol* oprot) const { ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); xfer += oprot->writeStructBegin("RowRecord"); - xfer += oprot->writeFieldBegin("vector_map", ::apache::thrift::protocol::T_MAP, 1); - { - xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_STRING, static_cast(this->vector_map.size())); - std::map ::const_iterator _iter61; - for (_iter61 = this->vector_map.begin(); _iter61 != this->vector_map.end(); ++_iter61) - { - xfer += oprot->writeString(_iter61->first); - xfer += oprot->writeBinary(_iter61->second); - } - xfer += oprot->writeMapEnd(); - } - xfer += oprot->writeFieldEnd(); - - xfer += oprot->writeFieldBegin("attribute_map", ::apache::thrift::protocol::T_MAP, 2); - { - xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_STRING, static_cast(this->attribute_map.size())); - std::map ::const_iterator _iter62; - for (_iter62 = this->attribute_map.begin(); _iter62 != this->attribute_map.end(); ++_iter62) - { - xfer += oprot->writeString(_iter62->first); - xfer += oprot->writeString(_iter62->second); - } - xfer += oprot->writeMapEnd(); - } + xfer += oprot->writeFieldBegin("vector_data", ::apache::thrift::protocol::T_STRING, 1); + xfer += oprot->writeBinary(this->vector_data); xfer += oprot->writeFieldEnd(); xfer += oprot->writeFieldStop(); @@ -1219,254 +510,20 @@ uint32_t RowRecord::write(::apache::thrift::protocol::TProtocol* oprot) const { void swap(RowRecord &a, RowRecord &b) { using ::std::swap; - swap(a.vector_map, b.vector_map); - swap(a.attribute_map, b.attribute_map); - swap(a.__isset, b.__isset); + swap(a.vector_data, b.vector_data); } -RowRecord::RowRecord(const RowRecord& other63) { - vector_map = other63.vector_map; - attribute_map = other63.attribute_map; - __isset = other63.__isset; +RowRecord::RowRecord(const RowRecord& other7) { + vector_data = other7.vector_data; } -RowRecord& RowRecord::operator=(const RowRecord& other64) { - vector_map = other64.vector_map; - attribute_map = other64.attribute_map; - __isset = other64.__isset; +RowRecord& RowRecord::operator=(const RowRecord& other8) { + vector_data = other8.vector_data; return *this; } void RowRecord::printTo(std::ostream& out) const { using ::apache::thrift::to_string; out << "RowRecord("; - out << "vector_map=" << to_string(vector_map); - out << ", " << "attribute_map=" << to_string(attribute_map); - out << ")"; -} - - -QueryRecord::~QueryRecord() throw() { -} - - -void QueryRecord::__set_vector_map(const std::map & val) { - this->vector_map = val; -} - -void QueryRecord::__set_selected_column_array(const std::vector & val) { - this->selected_column_array = val; -__isset.selected_column_array = true; -} - -void QueryRecord::__set_partition_filter_column_map(const std::map > & val) { - this->partition_filter_column_map = val; -__isset.partition_filter_column_map = true; -} -std::ostream& operator<<(std::ostream& out, const QueryRecord& obj) -{ - obj.printTo(out); - return out; -} - - -uint32_t QueryRecord::read(::apache::thrift::protocol::TProtocol* iprot) { - - ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); - uint32_t xfer = 0; - std::string fname; - ::apache::thrift::protocol::TType ftype; - int16_t fid; - - xfer += iprot->readStructBegin(fname); - - using ::apache::thrift::protocol::TProtocolException; - - bool isset_vector_map = false; - - while (true) - { - xfer += iprot->readFieldBegin(fname, ftype, fid); - if (ftype == ::apache::thrift::protocol::T_STOP) { - break; - } - switch (fid) - { - case 1: - if (ftype == ::apache::thrift::protocol::T_MAP) { - { - this->vector_map.clear(); - uint32_t _size65; - ::apache::thrift::protocol::TType _ktype66; - ::apache::thrift::protocol::TType _vtype67; - xfer += iprot->readMapBegin(_ktype66, _vtype67, _size65); - uint32_t _i69; - for (_i69 = 0; _i69 < _size65; ++_i69) - { - std::string _key70; - xfer += iprot->readString(_key70); - std::string& _val71 = this->vector_map[_key70]; - xfer += iprot->readBinary(_val71); - } - xfer += iprot->readMapEnd(); - } - isset_vector_map = true; - } else { - xfer += iprot->skip(ftype); - } - break; - case 2: - if (ftype == ::apache::thrift::protocol::T_LIST) { - { - this->selected_column_array.clear(); - uint32_t _size72; - ::apache::thrift::protocol::TType _etype75; - xfer += iprot->readListBegin(_etype75, _size72); - this->selected_column_array.resize(_size72); - uint32_t _i76; - for (_i76 = 0; _i76 < _size72; ++_i76) - { - xfer += iprot->readString(this->selected_column_array[_i76]); - } - xfer += iprot->readListEnd(); - } - this->__isset.selected_column_array = true; - } else { - xfer += iprot->skip(ftype); - } - break; - case 3: - if (ftype == ::apache::thrift::protocol::T_MAP) { - { - this->partition_filter_column_map.clear(); - uint32_t _size77; - ::apache::thrift::protocol::TType _ktype78; - ::apache::thrift::protocol::TType _vtype79; - xfer += iprot->readMapBegin(_ktype78, _vtype79, _size77); - uint32_t _i81; - for (_i81 = 0; _i81 < _size77; ++_i81) - { - std::string _key82; - xfer += iprot->readString(_key82); - std::vector & _val83 = this->partition_filter_column_map[_key82]; - { - _val83.clear(); - uint32_t _size84; - ::apache::thrift::protocol::TType _etype87; - xfer += iprot->readListBegin(_etype87, _size84); - _val83.resize(_size84); - uint32_t _i88; - for (_i88 = 0; _i88 < _size84; ++_i88) - { - xfer += _val83[_i88].read(iprot); - } - xfer += iprot->readListEnd(); - } - } - xfer += iprot->readMapEnd(); - } - this->__isset.partition_filter_column_map = true; - } else { - xfer += iprot->skip(ftype); - } - break; - default: - xfer += iprot->skip(ftype); - break; - } - xfer += iprot->readFieldEnd(); - } - - xfer += iprot->readStructEnd(); - - if (!isset_vector_map) - throw TProtocolException(TProtocolException::INVALID_DATA); - return xfer; -} - -uint32_t QueryRecord::write(::apache::thrift::protocol::TProtocol* oprot) const { - uint32_t xfer = 0; - ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); - xfer += oprot->writeStructBegin("QueryRecord"); - - xfer += oprot->writeFieldBegin("vector_map", ::apache::thrift::protocol::T_MAP, 1); - { - xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_STRING, static_cast(this->vector_map.size())); - std::map ::const_iterator _iter89; - for (_iter89 = this->vector_map.begin(); _iter89 != this->vector_map.end(); ++_iter89) - { - xfer += oprot->writeString(_iter89->first); - xfer += oprot->writeBinary(_iter89->second); - } - xfer += oprot->writeMapEnd(); - } - xfer += oprot->writeFieldEnd(); - - if (this->__isset.selected_column_array) { - xfer += oprot->writeFieldBegin("selected_column_array", ::apache::thrift::protocol::T_LIST, 2); - { - xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->selected_column_array.size())); - std::vector ::const_iterator _iter90; - for (_iter90 = this->selected_column_array.begin(); _iter90 != this->selected_column_array.end(); ++_iter90) - { - xfer += oprot->writeString((*_iter90)); - } - xfer += oprot->writeListEnd(); - } - xfer += oprot->writeFieldEnd(); - } - if (this->__isset.partition_filter_column_map) { - xfer += oprot->writeFieldBegin("partition_filter_column_map", ::apache::thrift::protocol::T_MAP, 3); - { - xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_LIST, static_cast(this->partition_filter_column_map.size())); - std::map > ::const_iterator _iter91; - for (_iter91 = this->partition_filter_column_map.begin(); _iter91 != this->partition_filter_column_map.end(); ++_iter91) - { - xfer += oprot->writeString(_iter91->first); - { - xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(_iter91->second.size())); - std::vector ::const_iterator _iter92; - for (_iter92 = _iter91->second.begin(); _iter92 != _iter91->second.end(); ++_iter92) - { - xfer += (*_iter92).write(oprot); - } - xfer += oprot->writeListEnd(); - } - } - xfer += oprot->writeMapEnd(); - } - xfer += oprot->writeFieldEnd(); - } - xfer += oprot->writeFieldStop(); - xfer += oprot->writeStructEnd(); - return xfer; -} - -void swap(QueryRecord &a, QueryRecord &b) { - using ::std::swap; - swap(a.vector_map, b.vector_map); - swap(a.selected_column_array, b.selected_column_array); - swap(a.partition_filter_column_map, b.partition_filter_column_map); - swap(a.__isset, b.__isset); -} - -QueryRecord::QueryRecord(const QueryRecord& other93) { - vector_map = other93.vector_map; - selected_column_array = other93.selected_column_array; - partition_filter_column_map = other93.partition_filter_column_map; - __isset = other93.__isset; -} -QueryRecord& QueryRecord::operator=(const QueryRecord& other94) { - vector_map = other94.vector_map; - selected_column_array = other94.selected_column_array; - partition_filter_column_map = other94.partition_filter_column_map; - __isset = other94.__isset; - return *this; -} -void QueryRecord::printTo(std::ostream& out) const { - using ::apache::thrift::to_string; - out << "QueryRecord("; - out << "vector_map=" << to_string(vector_map); - out << ", " << "selected_column_array="; (__isset.selected_column_array ? (out << to_string(selected_column_array)) : (out << "")); - out << ", " << "partition_filter_column_map="; (__isset.partition_filter_column_map ? (out << to_string(partition_filter_column_map)) : (out << "")); + out << "vector_data=" << to_string(vector_data); out << ")"; } @@ -1482,10 +539,6 @@ void QueryResult::__set_id(const int64_t val) { void QueryResult::__set_score(const double val) { this->score = val; } - -void QueryResult::__set_column_map(const std::map & val) { - this->column_map = val; -} std::ostream& operator<<(std::ostream& out, const QueryResult& obj) { obj.printTo(out); @@ -1530,29 +583,6 @@ uint32_t QueryResult::read(::apache::thrift::protocol::TProtocol* iprot) { xfer += iprot->skip(ftype); } break; - case 3: - if (ftype == ::apache::thrift::protocol::T_MAP) { - { - this->column_map.clear(); - uint32_t _size95; - ::apache::thrift::protocol::TType _ktype96; - ::apache::thrift::protocol::TType _vtype97; - xfer += iprot->readMapBegin(_ktype96, _vtype97, _size95); - uint32_t _i99; - for (_i99 = 0; _i99 < _size95; ++_i99) - { - std::string _key100; - xfer += iprot->readString(_key100); - std::string& _val101 = this->column_map[_key100]; - xfer += iprot->readString(_val101); - } - xfer += iprot->readMapEnd(); - } - this->__isset.column_map = true; - } else { - xfer += iprot->skip(ftype); - } - break; default: xfer += iprot->skip(ftype); break; @@ -1578,19 +608,6 @@ uint32_t QueryResult::write(::apache::thrift::protocol::TProtocol* oprot) const xfer += oprot->writeDouble(this->score); xfer += oprot->writeFieldEnd(); - xfer += oprot->writeFieldBegin("column_map", ::apache::thrift::protocol::T_MAP, 3); - { - xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_STRING, static_cast(this->column_map.size())); - std::map ::const_iterator _iter102; - for (_iter102 = this->column_map.begin(); _iter102 != this->column_map.end(); ++_iter102) - { - xfer += oprot->writeString(_iter102->first); - xfer += oprot->writeString(_iter102->second); - } - xfer += oprot->writeMapEnd(); - } - xfer += oprot->writeFieldEnd(); - xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; @@ -1600,21 +617,18 @@ void swap(QueryResult &a, QueryResult &b) { using ::std::swap; swap(a.id, b.id); swap(a.score, b.score); - swap(a.column_map, b.column_map); swap(a.__isset, b.__isset); } -QueryResult::QueryResult(const QueryResult& other103) { - id = other103.id; - score = other103.score; - column_map = other103.column_map; - __isset = other103.__isset; +QueryResult::QueryResult(const QueryResult& other9) { + id = other9.id; + score = other9.score; + __isset = other9.__isset; } -QueryResult& QueryResult::operator=(const QueryResult& other104) { - id = other104.id; - score = other104.score; - column_map = other104.column_map; - __isset = other104.__isset; +QueryResult& QueryResult::operator=(const QueryResult& other10) { + id = other10.id; + score = other10.score; + __isset = other10.__isset; return *this; } void QueryResult::printTo(std::ostream& out) const { @@ -1622,7 +636,6 @@ void QueryResult::printTo(std::ostream& out) const { out << "QueryResult("; out << "id=" << to_string(id); out << ", " << "score=" << to_string(score); - out << ", " << "column_map=" << to_string(column_map); out << ")"; } @@ -1666,14 +679,14 @@ uint32_t TopKQueryResult::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->query_result_arrays.clear(); - uint32_t _size105; - ::apache::thrift::protocol::TType _etype108; - xfer += iprot->readListBegin(_etype108, _size105); - this->query_result_arrays.resize(_size105); - uint32_t _i109; - for (_i109 = 0; _i109 < _size105; ++_i109) + uint32_t _size11; + ::apache::thrift::protocol::TType _etype14; + xfer += iprot->readListBegin(_etype14, _size11); + this->query_result_arrays.resize(_size11); + uint32_t _i15; + for (_i15 = 0; _i15 < _size11; ++_i15) { - xfer += this->query_result_arrays[_i109].read(iprot); + xfer += this->query_result_arrays[_i15].read(iprot); } xfer += iprot->readListEnd(); } @@ -1702,10 +715,10 @@ uint32_t TopKQueryResult::write(::apache::thrift::protocol::TProtocol* oprot) co xfer += oprot->writeFieldBegin("query_result_arrays", ::apache::thrift::protocol::T_LIST, 1); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->query_result_arrays.size())); - std::vector ::const_iterator _iter110; - for (_iter110 = this->query_result_arrays.begin(); _iter110 != this->query_result_arrays.end(); ++_iter110) + std::vector ::const_iterator _iter16; + for (_iter16 = this->query_result_arrays.begin(); _iter16 != this->query_result_arrays.end(); ++_iter16) { - xfer += (*_iter110).write(oprot); + xfer += (*_iter16).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1722,13 +735,13 @@ void swap(TopKQueryResult &a, TopKQueryResult &b) { swap(a.__isset, b.__isset); } -TopKQueryResult::TopKQueryResult(const TopKQueryResult& other111) { - query_result_arrays = other111.query_result_arrays; - __isset = other111.__isset; +TopKQueryResult::TopKQueryResult(const TopKQueryResult& other17) { + query_result_arrays = other17.query_result_arrays; + __isset = other17.__isset; } -TopKQueryResult& TopKQueryResult::operator=(const TopKQueryResult& other112) { - query_result_arrays = other112.query_result_arrays; - __isset = other112.__isset; +TopKQueryResult& TopKQueryResult::operator=(const TopKQueryResult& other18) { + query_result_arrays = other18.query_result_arrays; + __isset = other18.__isset; return *this; } void TopKQueryResult::printTo(std::ostream& out) const { diff --git a/cpp/src/thrift/gen-cpp/megasearch_types.h b/cpp/src/thrift/gen-cpp/megasearch_types.h index 8f8f03b47d4b312ea38b08e4d57284c323c57aca..8ef19d02c817ff3bb4634ac150d9ec29a3a95dee 100644 --- a/cpp/src/thrift/gen-cpp/megasearch_types.h +++ b/cpp/src/thrift/gen-cpp/megasearch_types.h @@ -26,10 +26,9 @@ struct ErrorCode { CONNECT_FAILED = 1, PERMISSION_DENIED = 2, TABLE_NOT_EXISTS = 3, - PARTITION_NOT_EXIST = 4, - ILLEGAL_ARGUMENT = 5, - ILLEGAL_RANGE = 6, - ILLEGAL_DIMENSION = 7 + ILLEGAL_ARGUMENT = 4, + ILLEGAL_RANGE = 5, + ILLEGAL_DIMENSION = 6 }; }; @@ -39,22 +38,12 @@ std::ostream& operator<<(std::ostream& out, const ErrorCode::type& val); class Exception; -class Column; - -class VectorColumn; - class TableSchema; class Range; -class CreateTablePartitionParam; - -class DeleteTablePartitionParam; - class RowRecord; -class QueryRecord; - class QueryResult; class TopKQueryResult; @@ -109,108 +98,11 @@ void swap(Exception &a, Exception &b); std::ostream& operator<<(std::ostream& out, const Exception& obj); - -class Column : public virtual ::apache::thrift::TBase { - public: - - Column(const Column&); - Column& operator=(const Column&); - Column() : type(0), name() { - } - - virtual ~Column() throw(); - int32_t type; - std::string name; - - void __set_type(const int32_t val); - - void __set_name(const std::string& val); - - bool operator == (const Column & rhs) const - { - if (!(type == rhs.type)) - return false; - if (!(name == rhs.name)) - return false; - return true; - } - bool operator != (const Column &rhs) const { - return !(*this == rhs); - } - - bool operator < (const Column & ) const; - - uint32_t read(::apache::thrift::protocol::TProtocol* iprot); - uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; - - virtual void printTo(std::ostream& out) const; -}; - -void swap(Column &a, Column &b); - -std::ostream& operator<<(std::ostream& out, const Column& obj); - -typedef struct _VectorColumn__isset { - _VectorColumn__isset() : store_raw_vector(true) {} - bool store_raw_vector :1; -} _VectorColumn__isset; - -class VectorColumn : public virtual ::apache::thrift::TBase { - public: - - VectorColumn(const VectorColumn&); - VectorColumn& operator=(const VectorColumn&); - VectorColumn() : dimension(0), index_type(), store_raw_vector(false) { - } - - virtual ~VectorColumn() throw(); - Column base; - int64_t dimension; - std::string index_type; - bool store_raw_vector; - - _VectorColumn__isset __isset; - - void __set_base(const Column& val); - - void __set_dimension(const int64_t val); - - void __set_index_type(const std::string& val); - - void __set_store_raw_vector(const bool val); - - bool operator == (const VectorColumn & rhs) const - { - if (!(base == rhs.base)) - return false; - if (!(dimension == rhs.dimension)) - return false; - if (!(index_type == rhs.index_type)) - return false; - if (!(store_raw_vector == rhs.store_raw_vector)) - return false; - return true; - } - bool operator != (const VectorColumn &rhs) const { - return !(*this == rhs); - } - - bool operator < (const VectorColumn & ) const; - - uint32_t read(::apache::thrift::protocol::TProtocol* iprot); - uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; - - virtual void printTo(std::ostream& out) const; -}; - -void swap(VectorColumn &a, VectorColumn &b); - -std::ostream& operator<<(std::ostream& out, const VectorColumn& obj); - typedef struct _TableSchema__isset { - _TableSchema__isset() : attribute_column_array(false), partition_column_name_array(false) {} - bool attribute_column_array :1; - bool partition_column_name_array :1; + _TableSchema__isset() : index_type(true), dimension(true), store_raw_vector(true) {} + bool index_type :1; + bool dimension :1; + bool store_raw_vector :1; } _TableSchema__isset; class TableSchema : public virtual ::apache::thrift::TBase { @@ -218,38 +110,34 @@ class TableSchema : public virtual ::apache::thrift::TBase { TableSchema(const TableSchema&); TableSchema& operator=(const TableSchema&); - TableSchema() : table_name() { + TableSchema() : table_name(), index_type(0), dimension(0LL), store_raw_vector(false) { } virtual ~TableSchema() throw(); std::string table_name; - std::vector vector_column_array; - std::vector attribute_column_array; - std::vector partition_column_name_array; + int32_t index_type; + int64_t dimension; + bool store_raw_vector; _TableSchema__isset __isset; void __set_table_name(const std::string& val); - void __set_vector_column_array(const std::vector & val); + void __set_index_type(const int32_t val); - void __set_attribute_column_array(const std::vector & val); + void __set_dimension(const int64_t val); - void __set_partition_column_name_array(const std::vector & val); + void __set_store_raw_vector(const bool val); bool operator == (const TableSchema & rhs) const { if (!(table_name == rhs.table_name)) return false; - if (!(vector_column_array == rhs.vector_column_array)) - return false; - if (__isset.attribute_column_array != rhs.__isset.attribute_column_array) - return false; - else if (__isset.attribute_column_array && !(attribute_column_array == rhs.attribute_column_array)) + if (!(index_type == rhs.index_type)) return false; - if (__isset.partition_column_name_array != rhs.__isset.partition_column_name_array) + if (!(dimension == rhs.dimension)) return false; - else if (__isset.partition_column_name_array && !(partition_column_name_array == rhs.partition_column_name_array)) + if (!(store_raw_vector == rhs.store_raw_vector)) return false; return true; } @@ -269,6 +157,11 @@ void swap(TableSchema &a, TableSchema &b); std::ostream& operator<<(std::ostream& out, const TableSchema& obj); +typedef struct _Range__isset { + _Range__isset() : start_value(false), end_value(false) {} + bool start_value :1; + bool end_value :1; +} _Range__isset; class Range : public virtual ::apache::thrift::TBase { public: @@ -282,6 +175,8 @@ class Range : public virtual ::apache::thrift::TBase { std::string start_value; std::string end_value; + _Range__isset __isset; + void __set_start_value(const std::string& val); void __set_end_value(const std::string& val); @@ -311,120 +206,22 @@ void swap(Range &a, Range &b); std::ostream& operator<<(std::ostream& out, const Range& obj); -class CreateTablePartitionParam : public virtual ::apache::thrift::TBase { - public: - - CreateTablePartitionParam(const CreateTablePartitionParam&); - CreateTablePartitionParam& operator=(const CreateTablePartitionParam&); - CreateTablePartitionParam() : table_name(), partition_name() { - } - - virtual ~CreateTablePartitionParam() throw(); - std::string table_name; - std::string partition_name; - std::map range_map; - - void __set_table_name(const std::string& val); - - void __set_partition_name(const std::string& val); - - void __set_range_map(const std::map & val); - - bool operator == (const CreateTablePartitionParam & rhs) const - { - if (!(table_name == rhs.table_name)) - return false; - if (!(partition_name == rhs.partition_name)) - return false; - if (!(range_map == rhs.range_map)) - return false; - return true; - } - bool operator != (const CreateTablePartitionParam &rhs) const { - return !(*this == rhs); - } - - bool operator < (const CreateTablePartitionParam & ) const; - - uint32_t read(::apache::thrift::protocol::TProtocol* iprot); - uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; - - virtual void printTo(std::ostream& out) const; -}; - -void swap(CreateTablePartitionParam &a, CreateTablePartitionParam &b); - -std::ostream& operator<<(std::ostream& out, const CreateTablePartitionParam& obj); - - -class DeleteTablePartitionParam : public virtual ::apache::thrift::TBase { - public: - - DeleteTablePartitionParam(const DeleteTablePartitionParam&); - DeleteTablePartitionParam& operator=(const DeleteTablePartitionParam&); - DeleteTablePartitionParam() : table_name() { - } - - virtual ~DeleteTablePartitionParam() throw(); - std::string table_name; - std::vector partition_name_array; - - void __set_table_name(const std::string& val); - - void __set_partition_name_array(const std::vector & val); - - bool operator == (const DeleteTablePartitionParam & rhs) const - { - if (!(table_name == rhs.table_name)) - return false; - if (!(partition_name_array == rhs.partition_name_array)) - return false; - return true; - } - bool operator != (const DeleteTablePartitionParam &rhs) const { - return !(*this == rhs); - } - - bool operator < (const DeleteTablePartitionParam & ) const; - - uint32_t read(::apache::thrift::protocol::TProtocol* iprot); - uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; - - virtual void printTo(std::ostream& out) const; -}; - -void swap(DeleteTablePartitionParam &a, DeleteTablePartitionParam &b); - -std::ostream& operator<<(std::ostream& out, const DeleteTablePartitionParam& obj); - -typedef struct _RowRecord__isset { - _RowRecord__isset() : attribute_map(false) {} - bool attribute_map :1; -} _RowRecord__isset; - class RowRecord : public virtual ::apache::thrift::TBase { public: RowRecord(const RowRecord&); RowRecord& operator=(const RowRecord&); - RowRecord() { + RowRecord() : vector_data() { } virtual ~RowRecord() throw(); - std::map vector_map; - std::map attribute_map; - - _RowRecord__isset __isset; + std::string vector_data; - void __set_vector_map(const std::map & val); - - void __set_attribute_map(const std::map & val); + void __set_vector_data(const std::string& val); bool operator == (const RowRecord & rhs) const { - if (!(vector_map == rhs.vector_map)) - return false; - if (!(attribute_map == rhs.attribute_map)) + if (!(vector_data == rhs.vector_data)) return false; return true; } @@ -444,68 +241,10 @@ void swap(RowRecord &a, RowRecord &b); std::ostream& operator<<(std::ostream& out, const RowRecord& obj); -typedef struct _QueryRecord__isset { - _QueryRecord__isset() : selected_column_array(false), partition_filter_column_map(false) {} - bool selected_column_array :1; - bool partition_filter_column_map :1; -} _QueryRecord__isset; - -class QueryRecord : public virtual ::apache::thrift::TBase { - public: - - QueryRecord(const QueryRecord&); - QueryRecord& operator=(const QueryRecord&); - QueryRecord() { - } - - virtual ~QueryRecord() throw(); - std::map vector_map; - std::vector selected_column_array; - std::map > partition_filter_column_map; - - _QueryRecord__isset __isset; - - void __set_vector_map(const std::map & val); - - void __set_selected_column_array(const std::vector & val); - - void __set_partition_filter_column_map(const std::map > & val); - - bool operator == (const QueryRecord & rhs) const - { - if (!(vector_map == rhs.vector_map)) - return false; - if (__isset.selected_column_array != rhs.__isset.selected_column_array) - return false; - else if (__isset.selected_column_array && !(selected_column_array == rhs.selected_column_array)) - return false; - if (__isset.partition_filter_column_map != rhs.__isset.partition_filter_column_map) - return false; - else if (__isset.partition_filter_column_map && !(partition_filter_column_map == rhs.partition_filter_column_map)) - return false; - return true; - } - bool operator != (const QueryRecord &rhs) const { - return !(*this == rhs); - } - - bool operator < (const QueryRecord & ) const; - - uint32_t read(::apache::thrift::protocol::TProtocol* iprot); - uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; - - virtual void printTo(std::ostream& out) const; -}; - -void swap(QueryRecord &a, QueryRecord &b); - -std::ostream& operator<<(std::ostream& out, const QueryRecord& obj); - typedef struct _QueryResult__isset { - _QueryResult__isset() : id(false), score(false), column_map(false) {} + _QueryResult__isset() : id(false), score(false) {} bool id :1; bool score :1; - bool column_map :1; } _QueryResult__isset; class QueryResult : public virtual ::apache::thrift::TBase { @@ -519,7 +258,6 @@ class QueryResult : public virtual ::apache::thrift::TBase { virtual ~QueryResult() throw(); int64_t id; double score; - std::map column_map; _QueryResult__isset __isset; @@ -527,16 +265,12 @@ class QueryResult : public virtual ::apache::thrift::TBase { void __set_score(const double val); - void __set_column_map(const std::map & val); - bool operator == (const QueryResult & rhs) const { if (!(id == rhs.id)) return false; if (!(score == rhs.score)) return false; - if (!(column_map == rhs.column_map)) - return false; return true; } bool operator != (const QueryResult &rhs) const { diff --git a/cpp/src/thrift/megasearch.thrift b/cpp/src/thrift/megasearch.thrift index 07fcfedfe8513217bd630fb16703fb38c50a1a95..0dc7230934e912480e0cb9dddd7b1ebe4741e98c 100644 --- a/cpp/src/thrift/megasearch.thrift +++ b/cpp/src/thrift/megasearch.thrift @@ -17,104 +17,56 @@ enum ErrorCode { SUCCESS = 0, CONNECT_FAILED, PERMISSION_DENIED, - TABLE_NOT_EXISTS, - PARTITION_NOT_EXIST, - ILLEGAL_ARGUMENT, - ILLEGAL_RANGE, - ILLEGAL_DIMENSION, + TABLE_NOT_EXISTS, + ILLEGAL_ARGUMENT, + ILLEGAL_RANGE, + ILLEGAL_DIMENSION, } exception Exception { - 1: ErrorCode code; - 2: string reason; + 1: ErrorCode code; + 2: string reason; } -/** - * @brief Table column description - */ -struct Column { - 1: required i32 type; ///< Column Type: 0:invalid/1:int8/2:int16/3:int32/4:int64/5:float32/6:float64/7:date/8:vector - 2: required string name; ///< Column name -} -/** - * @brief Table vector column description - */ -struct VectorColumn { - 1: required Column base; ///< Base column schema - 2: required i64 dimension; ///< Vector dimension - 3: required string index_type; ///< Index type, optional: raw, ivf - 4: bool store_raw_vector = false; ///< Is vector self stored in the table -} - -/** * @brief Table Schema */ struct TableSchema { - 1: required string table_name; ///< Table name - 2: required list vector_column_array; ///< Vector column description - 3: optional list attribute_column_array; ///< Columns description - 4: optional list partition_column_name_array; ///< Partition column name + 1: required string table_name; ///< Table name + 2: i32 index_type = 0; ///< Index type, optional: 0-invalid, 1-idmap, 2-ivflat + 3: i64 dimension = 0; ///< Vector dimension + 4: bool store_raw_vector = false; ///< Store raw data } /** * @brief Range Schema */ struct Range { - 1: required string start_value; ///< Range start - 2: required string end_value; ///< Range stop -} - -/** - * @brief Create table partition parameters - */ -struct CreateTablePartitionParam { - 1: required string table_name; ///< Table name, vector/float32/float64 type column is not allowed for partition - 2: required string partition_name; ///< Partition name, created partition name - 3: required map range_map; ///< Column name to Range map -} - - -/** - * @brief Delete table partition parameters - */ -struct DeleteTablePartitionParam { - 1: required string table_name; ///< Table name - 2: required list partition_name_array; ///< Partition name array + 1: string start_value; ///< Range start + 2: string end_value; ///< Range stop } /** * @brief Record inserted */ struct RowRecord { - 1: required map vector_map; ///< Vector columns - 2: map attribute_map; ///< Other attribute columns -} - -/** - * @brief Query record - */ -struct QueryRecord { - 1: required map vector_map; ///< Query vectors - 2: optional list selected_column_array; ///< Output column array - 3: optional map> partition_filter_column_map; ///< Range used to select partitions + 1: required binary vector_data; ///< Vector data, double array } /** * @brief Query result */ struct QueryResult { - 1: i64 id; ///< Output result - 2: double score; ///< Vector similarity score: 0 ~ 100 - 3: map column_map; ///< Other column + 1: i64 id; ///< Output result + 2: double score; ///< Vector similarity score: 0 ~ 100 } /** * @brief TopK query result */ struct TopKQueryResult { - 1: list query_result_arrays; ///< TopK query result + 1: list query_result_arrays; ///< TopK query result } service MegasearchService { @@ -140,28 +92,6 @@ service MegasearchService { void DeleteTable(2: string table_name) throws(1: Exception e); - /** - * @brief Create table partition - * - * This method is used to create table partition. - * - * @param param, use to provide partition information to be created. - * - */ - void CreateTablePartition(2: CreateTablePartitionParam param) throws(1: Exception e); - - - /** - * @brief Delete table partition - * - * This method is used to delete table partition. - * - * @param param, use to provide partition information to be deleted. - * - */ - void DeleteTablePartition(2: DeleteTablePartitionParam param) throws(1: Exception e); - - /** * @brief Add vector array to table * @@ -183,25 +113,40 @@ service MegasearchService { * * @param table_name, table_name is queried. * @param query_record_array, all vector are going to be queried. + * @param query_range_array, optional ranges for conditional search. If not specified, search whole table * @param topk, how many similarity vectors will be searched. * * @return query result array. */ list SearchVector(2: string table_name, - 3: list query_record_array, - 4: i64 topk) throws(1: Exception e); + 3: list query_record_array, + 4: list query_range_array, + 5: i64 topk) throws(1: Exception e); + /** - * @brief Show table information + * @brief Get table schema * - * This method is used to show table information. + * This method is used to get table schema. * - * @param table_name, which table is show. + * @param table_name, target table name. * * @return table schema */ TableSchema DescribeTable(2: string table_name) throws(1: Exception e); + + /** + * @brief Get table row count + * + * This method is used to get table row count. + * + * @param table_name, target table name. + * + * @return table row count + */ + i64 GetTableRowCount(2: string table_name) throws(1: Exception e); + /** * @brief List all tables in database * @@ -212,6 +157,7 @@ service MegasearchService { */ list ShowTables() throws(1: Exception e); + /** * @brief Give the server status * diff --git a/cpp/src/utils/Error.h b/cpp/src/utils/Error.h index d7904585d991504376ba06d310bbdb8c8e126037..eb8d820d0471344e34c31765cfb49490f055d56e 100644 --- a/cpp/src/utils/Error.h +++ b/cpp/src/utils/Error.h @@ -31,7 +31,7 @@ constexpr ServerError SERVER_INVALID_ARGUMENT = ToGlobalServerErrorCode(0x004); constexpr ServerError SERVER_FILE_NOT_FOUND = ToGlobalServerErrorCode(0x005); constexpr ServerError SERVER_NOT_IMPLEMENT = ToGlobalServerErrorCode(0x006); constexpr ServerError SERVER_BLOCKING_QUEUE_EMPTY = ToGlobalServerErrorCode(0x007); -constexpr ServerError SERVER_GROUP_NOT_EXIST = ToGlobalServerErrorCode(0x008); +constexpr ServerError SERVER_TABLE_NOT_EXIST = ToGlobalServerErrorCode(0x008); constexpr ServerError SERVER_INVALID_TIME_RANGE = ToGlobalServerErrorCode(0x009); constexpr ServerError SERVER_INVALID_VECTOR_DIMENSION = ToGlobalServerErrorCode(0x00a); constexpr ServerError SERVER_LICENSE_VALIDATION_FAIL = ToGlobalServerErrorCode(0x00b); diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index 4ae327a6a81d901d4e91caf4af7df21cb41ddee6..c903a7b9573933c2606c86e1bc1db0c12067e041 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -67,15 +67,15 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) { long size; engine::meta::TableSchema group_info; - group_info.dimension = group_dim; - group_info.table_id = group_name; + group_info.dimension_ = group_dim; + group_info.table_id_ = group_name; engine::Status stat = db_->CreateTable(group_info); engine::meta::TableSchema group_info_get; - group_info_get.table_id = group_name; + group_info_get.table_id_ = group_name; stat = db_->DescribeTable(group_info_get); ASSERT_STATS(stat); - ASSERT_EQ(group_info_get.dimension, group_dim); + ASSERT_EQ(group_info_get.dimension_, group_dim); engine::IDNumbers vector_ids; engine::IDNumbers target_ids; @@ -114,15 +114,15 @@ TEST_F(DBTest, DB_TEST) { static const int group_dim = 256; engine::meta::TableSchema group_info; - group_info.dimension = group_dim; - group_info.table_id = group_name; + group_info.dimension_ = group_dim; + group_info.table_id_ = group_name; engine::Status stat = db_->CreateTable(group_info); engine::meta::TableSchema group_info_get; - group_info_get.table_id = group_name; + group_info_get.table_id_ = group_name; stat = db_->DescribeTable(group_info_get); ASSERT_STATS(stat); - ASSERT_EQ(group_info_get.dimension, group_dim); + ASSERT_EQ(group_info_get.dimension_, group_dim); engine::IDNumbers vector_ids; engine::IDNumbers target_ids; @@ -164,11 +164,11 @@ TEST_F(DBTest, DB_TEST) { ASSERT_STATS(stat); for (auto k=0; kCreateTable(group_info); engine::meta::TableSchema group_info_get; - group_info_get.table_id = group_name; + group_info_get.table_id_ = group_name; stat = db_->DescribeTable(group_info_get); ASSERT_STATS(stat); - ASSERT_EQ(group_info_get.dimension, group_dim); + ASSERT_EQ(group_info_get.dimension_, group_dim); // prepare raw data size_t nb = 250000; diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index abd03d30579798798e82ffe775bd9a98d01b0ac5..0345f85e9552726de6facf3f0017134cc36f2512 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -21,22 +21,22 @@ TEST_F(MetaTest, GROUP_TEST) { auto table_id = "meta_test_group"; meta::TableSchema group; - group.table_id = table_id; + group.table_id_ = table_id; auto status = impl_->CreateTable(group); ASSERT_TRUE(status.ok()); - auto gid = group.id; - group.id = -1; + auto gid = group.id_; + group.id_ = -1; status = impl_->DescribeTable(group); ASSERT_TRUE(status.ok()); - ASSERT_EQ(group.id, gid); - ASSERT_EQ(group.table_id, table_id); + ASSERT_EQ(group.id_, gid); + ASSERT_EQ(group.table_id_, table_id); - group.table_id = "not_found"; + group.table_id_ = "not_found"; status = impl_->DescribeTable(group); ASSERT_TRUE(!status.ok()); - group.table_id = table_id; + group.table_id_ = table_id; status = impl_->CreateTable(group); ASSERT_TRUE(!status.ok()); } @@ -45,49 +45,49 @@ TEST_F(MetaTest, table_file_TEST) { auto table_id = "meta_test_group"; meta::TableSchema group; - group.table_id = table_id; + group.table_id_ = table_id; auto status = impl_->CreateTable(group); meta::TableFileSchema table_file; - table_file.table_id = group.table_id; + table_file.table_id_ = group.table_id_; status = impl_->CreateTableFile(table_file); ASSERT_TRUE(status.ok()); - ASSERT_EQ(table_file.file_type, meta::TableFileSchema::NEW); + ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW); - auto file_id = table_file.file_id; + auto file_id = table_file.file_id_; auto new_file_type = meta::TableFileSchema::INDEX; - table_file.file_type = new_file_type; + table_file.file_type_ = new_file_type; status = impl_->UpdateTableFile(table_file); ASSERT_TRUE(status.ok()); - ASSERT_EQ(table_file.file_type, new_file_type); + ASSERT_EQ(table_file.file_type_, new_file_type); meta::DatesT dates; dates.push_back(meta::Meta::GetDate()); - status = impl_->DropPartitionsByDates(table_file.table_id, dates); + status = impl_->DropPartitionsByDates(table_file.table_id_, dates); ASSERT_FALSE(status.ok()); dates.clear(); for (auto i=2; i < 10; ++i) { dates.push_back(meta::Meta::GetDateWithDelta(-1*i)); } - status = impl_->DropPartitionsByDates(table_file.table_id, dates); + status = impl_->DropPartitionsByDates(table_file.table_id_, dates); ASSERT_TRUE(status.ok()); - table_file.date = meta::Meta::GetDateWithDelta(-2); + table_file.date_ = meta::Meta::GetDateWithDelta(-2); status = impl_->UpdateTableFile(table_file); ASSERT_TRUE(status.ok()); - ASSERT_EQ(table_file.date, meta::Meta::GetDateWithDelta(-2)); - ASSERT_FALSE(table_file.file_type == meta::TableFileSchema::TO_DELETE); + ASSERT_EQ(table_file.date_, meta::Meta::GetDateWithDelta(-2)); + ASSERT_FALSE(table_file.file_type_ == meta::TableFileSchema::TO_DELETE); dates.clear(); - dates.push_back(table_file.date); - status = impl_->DropPartitionsByDates(table_file.table_id, dates); + dates.push_back(table_file.date_); + status = impl_->DropPartitionsByDates(table_file.table_id_, dates); ASSERT_TRUE(status.ok()); status = impl_->GetTableFile(table_file); ASSERT_TRUE(status.ok()); - ASSERT_TRUE(table_file.file_type == meta::TableFileSchema::TO_DELETE); + ASSERT_TRUE(table_file.file_type_ == meta::TableFileSchema::TO_DELETE); } TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { @@ -103,21 +103,21 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { auto table_id = "meta_test_group"; meta::TableSchema group; - group.table_id = table_id; + group.table_id_ = table_id; auto status = impl.CreateTable(group); meta::TableFilesSchema files; meta::TableFileSchema table_file; - table_file.table_id = group.table_id; + table_file.table_id_ = group.table_id_; auto cnt = 100; long ts = utils::GetMicroSecTimeStamp(); std::vector days; for (auto i=0; iCreateTable(group); int new_files_cnt = 4; @@ -196,29 +196,29 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { int index_files_cnt = 7; meta::TableFileSchema table_file; - table_file.table_id = group.table_id; + table_file.table_id_ = group.table_id_; for (auto i=0; iCreateTableFile(table_file); - table_file.file_type = meta::TableFileSchema::NEW; + table_file.file_type_ = meta::TableFileSchema::NEW; status = impl_->UpdateTableFile(table_file); } for (auto i=0; iCreateTableFile(table_file); - table_file.file_type = meta::TableFileSchema::RAW; + table_file.file_type_ = meta::TableFileSchema::RAW; status = impl_->UpdateTableFile(table_file); } for (auto i=0; iCreateTableFile(table_file); - table_file.file_type = meta::TableFileSchema::TO_INDEX; + table_file.file_type_ = meta::TableFileSchema::TO_INDEX; status = impl_->UpdateTableFile(table_file); } for (auto i=0; iCreateTableFile(table_file); - table_file.file_type = meta::TableFileSchema::INDEX; + table_file.file_type_ = meta::TableFileSchema::INDEX; status = impl_->UpdateTableFile(table_file); } @@ -229,17 +229,17 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { ASSERT_EQ(files.size(), to_index_files_cnt); meta::DatePartionedTableFilesSchema dated_files; - status = impl_->FilesToMerge(group.table_id, dated_files); + status = impl_->FilesToMerge(group.table_id_, dated_files); ASSERT_TRUE(status.ok()); - ASSERT_EQ(dated_files[table_file.date].size(), raw_files_cnt); + ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt); status = impl_->FilesToIndex(files); ASSERT_TRUE(status.ok()); ASSERT_EQ(files.size(), to_index_files_cnt); - meta::DatesT dates = {table_file.date}; + meta::DatesT dates = {table_file.date_}; status = impl_->FilesToSearch(table_id, dates, dated_files); ASSERT_TRUE(status.ok()); - ASSERT_EQ(dated_files[table_file.date].size(), + ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt+raw_files_cnt+index_files_cnt); } diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index aa3ea560f3d2228aca0ec3535724c743f6bfb808..3e7c588ef6dcebf7338841d1fced9f7539b129bc 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -38,7 +38,7 @@ engine::Options DBTest::GetOptions() { void DBTest::SetUp() { InitLog(); auto options = GetOptions(); - db_ = engine::DBFactory::Build(options, "Faiss,IDMap"); + db_ = engine::DBFactory::Build(options); } void DBTest::TearDown() { diff --git a/cpp/unittest/faiss_wrapper/CMakeLists.txt b/cpp/unittest/faiss_wrapper/CMakeLists.txt index 1ea5431586b1c2e47d5e344764a8d3d3491ffcbe..49fdd7d6059266c69646f0ee9e0bb1808c3c2e95 100644 --- a/cpp/unittest/faiss_wrapper/CMakeLists.txt +++ b/cpp/unittest/faiss_wrapper/CMakeLists.txt @@ -13,8 +13,6 @@ include_directories(/usr/local/cuda/include) link_directories("/usr/local/cuda/lib64") set(require_files - ../../src/server/VecIdMapper.cpp - ../../src/server/RocksIdMapper.cpp ../../src/server/ServerConfig.cpp ../../src/utils/CommonUtil.cpp ../../src/utils/TimeRecorder.cpp @@ -38,7 +36,6 @@ set(wrapper_libs cudart cublas sqlite3 - rocksdb snappy bz2 z diff --git a/cpp/unittest/metrics/metrics_test.cpp b/cpp/unittest/metrics/metrics_test.cpp index ce09c9a4da45cf83783f2779afea501ea7320cca..f6d7f02feb18690bb6c1d37ab79fd51d585a343c 100644 --- a/cpp/unittest/metrics/metrics_test.cpp +++ b/cpp/unittest/metrics/metrics_test.cpp @@ -38,12 +38,12 @@ TEST_F(DBTest, Metric_Tes) { static const int group_dim = 256; engine::meta::TableSchema group_info; - group_info.dimension = group_dim; - group_info.table_id = group_name; + group_info.dimension_ = group_dim; + group_info.table_id_ = group_name; engine::Status stat = db_->CreateTable(group_info); engine::meta::TableSchema group_info_get; - group_info_get.table_id = group_name; + group_info_get.table_id_ = group_name; stat = db_->DescribeTable(group_info_get); @@ -87,11 +87,11 @@ TEST_F(DBTest, Metric_Tes) { ASSERT_STATS(stat); for (auto k=0; k -#include "server/ServerConfig.h" -#include "server/VecIdMapper.h" -#include "utils/TimeRecorder.h" -#include "utils/CommonUtil.h" - -#include - -using namespace zilliz::vecwise; - -namespace { - std::string CurrentTime() { - time_t tt; - time(&tt); - tt = tt + 8 * 3600; - tm *t = gmtime(&tt); - - std::string str = std::to_string(t->tm_year + 1900) + "_" + std::to_string(t->tm_mon + 1) - + "_" + std::to_string(t->tm_mday) + "_" + std::to_string(t->tm_hour) - + "_" + std::to_string(t->tm_min) + "_" + std::to_string(t->tm_sec); - - return str; - } - - std::string GetGroupID() { - static std::string s_id(CurrentTime()); - return s_id; - } -} - -TEST(IdMapperTest, IDMAPPER_TEST) { - server::IVecIdMapper* mapper = server::IVecIdMapper::GetInstance(); - - std::string group_id = GetGroupID(); - - std::vector nid = {"1", "50", "900", "10000"}; - std::vector sid = {"one", "fifty", "nine zero zero", "many"}; - server::ServerError err = mapper->Put(nid, sid, group_id); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - err = mapper->Put(nid, std::vector(), group_id); - ASSERT_NE(err, server::SERVER_SUCCESS); - - std::vector res; - err = mapper->Get(nid, res, group_id); - ASSERT_EQ(res.size(), nid.size()); - for(size_t i = 0; i < res.size(); i++) { - ASSERT_EQ(res[i], sid[i]); - } - - std::string str_id; - err = mapper->Get(nid[1], str_id, group_id); - ASSERT_EQ(str_id, "fifty"); - - err = mapper->Get(nid[1], str_id); - ASSERT_EQ(str_id, ""); - - err = mapper->Get(nid[2], str_id, group_id); - ASSERT_EQ(str_id, "nine zero zero"); - - err = mapper->Delete(nid[2], group_id); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - err = mapper->Get(nid[2], str_id, group_id); - ASSERT_EQ(str_id, ""); - - err = mapper->Get(nid[3], str_id, group_id); - ASSERT_EQ(str_id, "many"); - - err = mapper->DeleteGroup(group_id); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - err = mapper->Get(nid[3], str_id, group_id); - ASSERT_EQ(str_id, ""); - - std::string ct = CurrentTime(); - err = mapper->Put("current_time", ct, "time"); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - err = mapper->Get("current_time", str_id, "time"); - ASSERT_EQ(str_id, ct); - - //test performance - nid.clear(); - sid.clear(); - const int64_t count = 1000000; - { - server::TimeRecorder rc("prepare id data"); - for (int64_t i = 0; i < count; i++) { - nid.push_back(std::to_string(i + 100000)); - sid.push_back("val_" + std::to_string(i)); - } - rc.Record("done!"); - } - - { - std::string str_info = "Insert " + std::to_string(count) + " k/v into mapper"; - server::TimeRecorder rc(str_info); - err = mapper->Put(nid, sid); - ASSERT_EQ(err, server::SERVER_SUCCESS); - rc.Record("done!"); - } - - { - std::string str_info = "Get " + std::to_string(count) + " k/v from mapper"; - server::TimeRecorder rc(str_info); - std::vector res; - err = mapper->Get(nid, res); - ASSERT_EQ(res.size(), nid.size()); - rc.Record("done!"); - } -} - diff --git a/cpp/version.h.macro b/cpp/version.h.macro new file mode 100644 index 0000000000000000000000000000000000000000..a1d61cb271fcd3728d67dbabc9568ac5d60873fe --- /dev/null +++ b/cpp/version.h.macro @@ -0,0 +1,5 @@ +#pragma once + +#define MEGASEARCH_VERSION "@MEGASEARCH_VERSION@" +#define BUILD_TYPE "@BUILD_TYPE@" +#define BUILD_TIME @BUILD_TIME@ \ No newline at end of file