diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index acbe60489329b3638f65912fc9fd73a5a6b8bf68..bcc6f90f263ef605fb842efe387a6117a4ce6ad7 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -30,6 +30,8 @@ public: virtual Status search(const std::string& group_id, size_t k, size_t nq, const float* vectors, QueryResults& results) = 0; + virtual Status drop_all() = 0; + DB() = default; DB(const DB&) = delete; DB& operator=(const DB&) = delete; diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 18e77366323868f78124cdc4e13eb19c5263970c..89859e57581116228dc08e2685ca051450b0d5e1 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -246,9 +246,9 @@ Status DBImpl::background_merge_files(const std::string& group_id) { return status; } - if (raw_files.size() == 0) { - return Status::OK(); - } + /* if (raw_files.size() == 0) { */ + /* return Status::OK(); */ + /* } */ bool has_merge = false; @@ -261,9 +261,7 @@ Status DBImpl::background_merge_files(const std::string& group_id) { merge_files(group_id, kv.first, kv.second); } - if (has_merge) { - try_build_index(); - } + try_build_index(); _pMeta->cleanup_ttl_files(1); @@ -346,6 +344,10 @@ void DBImpl::background_compaction() { } } +Status DBImpl::drop_all() { + return _pMeta->drop_all(); +} + DBImpl::~DBImpl() { { std::unique_lock lock(_mutex); diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 6ecc287dbe102460b60b8118fc6c8393472a6f7f..bb0de8ac5d45d05d50a572ab4f15dddcf953bea0 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -36,6 +36,8 @@ public: virtual Status search(const std::string& group_id, size_t k, size_t nq, const float* vectors, QueryResults& results) override; + virtual Status drop_all() override; + virtual ~DBImpl(); private: diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 3652046e44f218d7e4b70b69f52659eda9ec82e1..ca92504fac2c50d46aac1cacf6de32071eba0747 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -427,6 +427,13 @@ Status DBMetaImpl::cleanup() { return Status::OK(); } +Status DBMetaImpl::drop_all() { + if (boost::filesystem::is_directory(_options.path)) { + boost::filesystem::remove_all(_options.path); + } + return Status::OK(); +} + DBMetaImpl::~DBMetaImpl() { cleanup(); } diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index 7ac04433438b17ea2796c38a2d44ecae6b13d183..e4510597b36225e96d2e810c72ba477295ec50e7 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -48,6 +48,8 @@ public: virtual Status cleanup_ttl_files(uint16_t seconds) override; + virtual Status drop_all() override; + virtual ~DBMetaImpl(); private: diff --git a/cpp/src/db/LocalMetaImpl.cpp b/cpp/src/db/LocalMetaImpl.cpp index f7192a05b7c72267df231b0114878c1ed681961a..07b5c1086502a4c404a63a67cb130f75af19152c 100644 --- a/cpp/src/db/LocalMetaImpl.cpp +++ b/cpp/src/db/LocalMetaImpl.cpp @@ -246,6 +246,11 @@ Status LocalMetaImpl::cleanup_ttl_files(uint16_t seconds) { return Status::OK(); } +Status LocalMetaImpl::drop_all() { + // PXU TODO + return Status::OK(); +} + } // namespace meta } // namespace engine } // namespace vecwise diff --git a/cpp/src/db/LocalMetaImpl.h b/cpp/src/db/LocalMetaImpl.h index 6c687500e1c5393e1061d3639f5114edec2040ce..2175379a01d702e7b28aa281d79a2c16eaedd0a9 100644 --- a/cpp/src/db/LocalMetaImpl.h +++ b/cpp/src/db/LocalMetaImpl.h @@ -43,6 +43,8 @@ public: virtual Status cleanup_ttl_files(uint16_t seconds) override; + virtual Status drop_all() override; + private: Status GetGroupMetaInfoByPath(const std::string& path, GroupSchema& group_info); diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index 4dbc275f656a3804ba34b7d8eb2c2aa3eba4b443..82c7900e47af464fb66bc7153c51ddeff0b9f73c 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -83,6 +83,8 @@ public: virtual Status cleanup() = 0; virtual Status cleanup_ttl_files(uint16_t) = 0; + virtual Status drop_all() = 0; + static DateT GetDate(const std::time_t& t); static DateT GetDate(); diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index b8106452c6f88775b75382d4978786b8a9df8d16..97ae94172233f506e8337a620313bd53188bf549 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include "db/DB.h" #include "faiss/Index.h" @@ -27,6 +29,7 @@ TEST(DBTest, DB_TEST) { static const int group_dim = 256; engine::Options opt; + opt.memory_sync_interval = 1; opt.meta.backend_uri = "http://127.0.0.1"; opt.meta.path = "/tmp/vecwise_test/db_test"; @@ -38,7 +41,6 @@ TEST(DBTest, DB_TEST) { group_info.dimension = group_dim; group_info.group_id = group_name; engine::Status stat = db->add_group(group_info); - //ASSERT_STATS(stat); engine::meta::GroupSchema group_info_get; group_info_get.group_id = group_name; @@ -47,21 +49,48 @@ TEST(DBTest, DB_TEST) { ASSERT_EQ(group_info_get.dimension, group_dim); engine::IDNumbers vector_ids; - std::vector vec_f; - for(int i = 0; i < group_dim; i++){ - vec_f.push_back((float)i); - vector_ids.push_back(i+1); + engine::IDNumbers target_ids; + + int d = 256; + int nb = 10; + float *xb = new float[d * nb]; + for(int i = 0; i < nb; i++) { + for(int j = 0; j < d; j++) xb[d * i + j] = drand48(); + xb[d * i] += i / 2000.; } - stat = db->add_vectors(group_name, 1, vec_f.data(), vector_ids); - ASSERT_STATS(stat); - //engine::QueryResults results; - //std::vector vec_s = vec_f; - //stat = db->search(group_name, 1, 1, vec_f.data(), results); - //ASSERT_STATS(stat); - //ASSERT_EQ(results.size(), 1); - //ASSERT_EQ(results[0][0], vector_ids[0]); + int qb = 1; + float *qxb = new float[d * qb]; + for(int i = 0; i < qb; i++) { + for(int j = 0; j < d; j++) qxb[d * i + j] = drand48(); + qxb[d * i] += i / 2000.; + } + + int loop = 500000; + + for (auto i=0; iadd_vectors(group_name, qb, qxb, target_ids); + } else { + db->add_vectors(group_name, nb, xb, vector_ids); + } + } + std::this_thread::sleep_for(std::chrono::seconds(2)); + + engine::QueryResults results; + int k = 10; + LOG(DEBUG) << "PRE"; + stat = db->search(group_name, k, qb, qxb, results); + LOG(DEBUG) << "POST"; + ASSERT_STATS(stat); + ASSERT_EQ(results[0][0], target_ids[0]); + + delete [] xb; + delete [] qxb; + delete db; + engine::DB::Open(opt, &db); + db->drop_all(); delete db; } @@ -141,4 +170,7 @@ TEST(SearchTest, DB_TEST) { // TODO(linxj): add groundTruth assert delete db; -} \ No newline at end of file + engine::DB::Open(opt, &db); + db->drop_all(); + delete db; +}