diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 6b432056c5e37827fd82869942ff2f8affc01e69..c2ed775601f875ed8c35bda5ff30ba9c4002756a 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -10,7 +10,7 @@ db_config: # URI format: dialect://username:password@host:port/database # All parts except dialect are optional, but you MUST include the delimiters # Currently dialect supports mysql or sqlite - db_backend_url: dialect://username:password@host:port/database # meta database uri + db_backend_url: sqlite://:@:/ index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index a3db0bf1103822d5d0babb5d5894719ff2b2d905..bf859b3b4f689bc4a0237399fa79ed07fafff439 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -70,9 +70,15 @@ DBWrapper::DBWrapper() { kill(0, SIGUSR1); } - zilliz::milvus::engine::DB::Open(opt, &db_); + std::string msg = opt.meta.path; + try { + zilliz::milvus::engine::DB::Open(opt, &db_); + } catch(std::exception& ex) { + msg = ex.what(); + } + if(db_ == nullptr) { - std::cout << "ERROR! Failed to open database" << std::endl; + std::cout << "ERROR! Failed to open database: " << msg << std::endl; kill(0, SIGUSR1); } } diff --git a/cpp/unittest/db/MySQLMetaImpl_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp similarity index 91% rename from cpp/unittest/db/MySQLMetaImpl_test.cpp rename to cpp/unittest/db/mysql_meta_test.cpp index 93e0ce9b28ed198fbbe11351b84a41af241274f1..436086acb3651af46d4c744dae422199bcc743a8 100644 --- a/cpp/unittest/db/MySQLMetaImpl_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -32,12 +32,19 @@ using namespace zilliz::milvus::engine; //} TEST_F(MySQLTest, core) { -// DBMetaOptions options; + DBMetaOptions options; // //dialect+driver://username:password@host:port/database // options.backend_uri = "mysql://root:1234@:/test"; // options.path = "/tmp/vecwise_test"; + try { + options = getDBMetaOptions(); + } catch(std::exception& ex) { + ASSERT_TRUE(false); + return; + } + int mode = Options::MODE::SINGLE; - meta::MySQLMetaImpl impl(getDBMetaOptions(), mode); + meta::MySQLMetaImpl impl(options, mode); // auto status = impl.Initialize(); // ASSERT_TRUE(status.ok()); @@ -192,9 +199,16 @@ TEST_F(MySQLTest, core) { } TEST_F(MySQLTest, GROUP_TEST) { + DBMetaOptions options; + try { + options = getDBMetaOptions(); + } catch(std::exception& ex) { + ASSERT_TRUE(false); + return; + } int mode = Options::MODE::SINGLE; - meta::MySQLMetaImpl impl(getDBMetaOptions(), mode); + meta::MySQLMetaImpl impl(options, mode); auto table_id = "meta_test_group"; @@ -228,9 +242,16 @@ TEST_F(MySQLTest, GROUP_TEST) { } TEST_F(MySQLTest, table_file_TEST) { + DBMetaOptions options; + try { + options = getDBMetaOptions(); + } catch(std::exception& ex) { + ASSERT_TRUE(false); + return; + } int mode = Options::MODE::SINGLE; - meta::MySQLMetaImpl impl(getDBMetaOptions(), mode); + meta::MySQLMetaImpl impl(options, mode); auto table_id = "meta_test_group"; @@ -296,7 +317,14 @@ TEST_F(MySQLTest, table_file_TEST) { TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { srand(time(0)); - DBMetaOptions options = getDBMetaOptions(); + DBMetaOptions options; + try { + options = getDBMetaOptions(); + } catch(std::exception& ex) { + ASSERT_TRUE(false); + return; + } + int days_num = rand() % 100; std::stringstream ss; ss << "days:" << days_num; @@ -350,7 +378,14 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { } TEST_F(MySQLTest, ARCHIVE_TEST_DISK) { - DBMetaOptions options = getDBMetaOptions(); + DBMetaOptions options; + try { + options = getDBMetaOptions(); + } catch(std::exception& ex) { + ASSERT_TRUE(false); + return; + } + options.archive_conf = ArchiveConf("delete", "disk:11"); int mode = Options::MODE::SINGLE; auto impl = meta::MySQLMetaImpl(options, mode); @@ -397,9 +432,16 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DISK) { } TEST_F(MySQLTest, TABLE_FILES_TEST) { + DBMetaOptions options; + try { + options = getDBMetaOptions(); + } catch(std::exception& ex) { + ASSERT_TRUE(false); + return; + } int mode = Options::MODE::SINGLE; - auto impl = meta::MySQLMetaImpl(getDBMetaOptions(), mode); + auto impl = meta::MySQLMetaImpl(options, mode); auto table_id = "meta_test_group"; @@ -460,6 +502,11 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) { ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt+raw_files_cnt+index_files_cnt); + status = impl.FilesToSearch(table_id, meta::DatesT(), dated_files); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(dated_files[table_file.date_].size(), + to_index_files_cnt+raw_files_cnt+index_files_cnt); + status = impl.DropAll(); ASSERT_TRUE(status.ok()); } diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index 6b1fc1e407d7eea883cc5b7e2a79e1ac9486a2f1..70c071254907a2c21aab24d276cc070c1d91d5e3 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -91,6 +91,11 @@ zilliz::milvus::engine::DBMetaOptions MySQLTest::getDBMetaOptions() { zilliz::milvus::engine::DBMetaOptions options; options.path = "/tmp/milvus_test"; options.backend_uri = DBTestEnvironment::getURI(); + + if(options.backend_uri.empty()) { + throw std::exception(); + } + return options; } diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index 80210772a827fbf042a14e3a464d47ca98c4f76d..d31e44c05626d19cf145fb1fe4e73d810d6588d0 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -17,6 +17,7 @@ aux_source_directory(../../src/config config_files) aux_source_directory(../../src/cache cache_srcs) aux_source_directory(../../src/wrapper wrapper_src) aux_source_directory(../../src/metrics metrics_src) +aux_source_directory(./ test_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files) @@ -35,6 +36,7 @@ link_directories("/usr/local/cuda/lib64") #include_directories(../db/utils.h) include_directories(../../src/metrics) +include_directories(/usr/include/mysql) #set(metrics_src_files # ../../src/metrics/Metrics.cpp @@ -47,17 +49,14 @@ include_directories(../../src/metrics) # ) set(count_test_src - ${unittest_srcs} ${config_files} ${cache_srcs} ${db_srcs} ${db_scheduler_srcs} ${wrapper_src} ${metrics_src} - metrics_test.cpp - prometheus_test.cpp - ../db/utils.cpp - metricbase_test.cpp) + ${test_srcs} + ) add_executable(metrics_test ${count_test_src} ${require_files} ) @@ -75,6 +74,7 @@ target_link_libraries(metrics_test gtest pthread z + mysqlpp ${unittest_libs} ) diff --git a/cpp/unittest/metrics/metricbase_test.cpp b/cpp/unittest/metrics/metricbase_test.cpp index ac850c7b48f118fb30e8c13dd0317c783fd5369b..1997748fddf3aa6e4618de8b9b1567fe09045976 100644 --- a/cpp/unittest/metrics/metricbase_test.cpp +++ b/cpp/unittest/metrics/metricbase_test.cpp @@ -11,7 +11,7 @@ using namespace zilliz::milvus; -TEST(MetricbaseTest, Metricbase_Test){ +TEST(MetricbaseTest, METRICBASE_TEST){ server::MetricsBase instance = server::MetricsBase::GetInstance(); instance.Init(); server::SystemInfo::GetInstance().Init(); diff --git a/cpp/unittest/metrics/metrics_test.cpp b/cpp/unittest/metrics/metrics_test.cpp index 923c7b717baa26d48c5b2a7aa9fe7ccc5eb93229..883e63ed03298d3184d5d84bb07074b81aa08a18 100644 --- a/cpp/unittest/metrics/metrics_test.cpp +++ b/cpp/unittest/metrics/metrics_test.cpp @@ -15,7 +15,7 @@ #include #include "metrics/Metrics.h" -#include "../db/utils.h" +#include "utils.h" #include "db/DB.h" #include "db/DBMetaImpl.h" #include "db/Factories.h" @@ -24,7 +24,7 @@ using namespace zilliz::milvus; -TEST_F(DBTest, Metric_Tes) { +TEST_F(MetricTest, Metric_Tes) { server::SystemInfo::GetInstance().Init(); // server::Metrics::GetInstance().Init(); diff --git a/cpp/unittest/metrics/prometheus_test.cpp b/cpp/unittest/metrics/prometheus_test.cpp index 885abed56605ccba783210035ee3def90cc97c54..521e00fc5c91401b90cde3115ca7f11bd59803bf 100644 --- a/cpp/unittest/metrics/prometheus_test.cpp +++ b/cpp/unittest/metrics/prometheus_test.cpp @@ -11,7 +11,7 @@ using namespace zilliz::milvus; -TEST(PrometheusTest, Prometheus_Test){ +TEST(PrometheusTest, PROMETHEUS_TEST){ server::PrometheusMetrics instance = server::PrometheusMetrics::GetInstance(); instance.Init(); instance.SetStartup(true); diff --git a/cpp/unittest/metrics/utils.cpp b/cpp/unittest/metrics/utils.cpp new file mode 100644 index 0000000000000000000000000000000000000000..81e924a87ee1e7e480781d2610baf6027b1ea7ec --- /dev/null +++ b/cpp/unittest/metrics/utils.cpp @@ -0,0 +1,79 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// + +#include +#include +#include +#include + +#include "utils.h" +#include "db/Factories.h" +#include "db/Options.h" + +INITIALIZE_EASYLOGGINGPP + +using namespace zilliz::milvus; + +static std::string uri; + +class DBTestEnvironment : public ::testing::Environment { +public: + +// explicit DBTestEnvironment(std::string uri) : uri_(uri) {} + + static std::string getURI() { + return uri; + } + + void SetUp() override { + getURI(); + } + +}; + +void ASSERT_STATS(engine::Status& stat) { + ASSERT_TRUE(stat.ok()); + if(!stat.ok()) { + std::cout << stat.ToString() << std::endl; + } +} + + +void MetricTest::InitLog() { + el::Configurations defaultConf; + defaultConf.setToDefault(); + defaultConf.set(el::Level::Debug, + el::ConfigurationType::Format, "[%thread-%datetime-%level]: %msg (%fbase:%line)"); + el::Loggers::reconfigureLogger("default", defaultConf); +} + +engine::Options MetricTest::GetOptions() { + auto options = engine::OptionsFactory::Build(); + options.meta.path = "/tmp/milvus_test"; + options.meta.backend_uri = "sqlite://:@:/"; + return options; +} + +void MetricTest::SetUp() { + InitLog(); + auto options = GetOptions(); + db_ = engine::DBFactory::Build(options); +} + +void MetricTest::TearDown() { + delete db_; + boost::filesystem::remove_all("/tmp/milvus_test"); +} + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + if (argc > 1) { + uri = argv[1]; + } +// std::cout << uri << std::endl; + ::testing::AddGlobalTestEnvironment(new DBTestEnvironment); + return RUN_ALL_TESTS(); +} diff --git a/cpp/unittest/metrics/utils.h b/cpp/unittest/metrics/utils.h new file mode 100644 index 0000000000000000000000000000000000000000..1badce00f217698f73bac53bb8f794952654f4a1 --- /dev/null +++ b/cpp/unittest/metrics/utils.h @@ -0,0 +1,64 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include +#include +//#include + +#include "db/DB.h" +#include "db/DBMetaImpl.h" +#include "db/MySQLMetaImpl.h" + + +#define TIMING + +#ifdef TIMING +#define INIT_TIMER auto start = std::chrono::high_resolution_clock::now(); +#define START_TIMER start = std::chrono::high_resolution_clock::now(); +#define STOP_TIMER(name) LOG(DEBUG) << "RUNTIME of " << name << ": " << \ + std::chrono::duration_cast( \ + std::chrono::high_resolution_clock::now()-start \ + ).count() << " ms "; +#else +#define INIT_TIMER +#define START_TIMER +#define STOP_TIMER(name) +#endif + +void ASSERT_STATS(zilliz::milvus::engine::Status& stat); + +//class TestEnv : public ::testing::Environment { +//public: +// +// static std::string getURI() { +// if (const char* uri = std::getenv("MILVUS_DBMETA_URI")) { +// return uri; +// } +// else { +// return ""; +// } +// } +// +// void SetUp() override { +// getURI(); +// } +// +//}; +// +//::testing::Environment* const test_env = +// ::testing::AddGlobalTestEnvironment(new TestEnv); + +class MetricTest : public ::testing::Test { +protected: + zilliz::milvus::engine::DB* db_; + + void InitLog(); + virtual void SetUp() override; + virtual void TearDown() override; + virtual zilliz::milvus::engine::Options GetOptions(); +}; \ No newline at end of file