Merge branch 'branch-0.3.0' into refact

Former-commit-id: 7dc970c19bec9073be1c1e001be5e3c45b025d81

cpp/CHANGELOG.md

@@ -35,6 +35,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-84 - cmake: add arrow, jemalloc and jsoncons third party; default build option OFF
 - MS-85 - add NetIO metric
 - MS-96 - add new query interface for specified files
+- MS-97 - Add S3 SDK for MinIO Storage
 
 ## Task
 - MS-74 - Change README.md in cpp

cpp/cmake/DefineOptions.cmake

@@ -107,6 +107,8 @@ else()
 endif()
 define_option(MILVUS_WITH_ZSTD "Build with zstd compression" ${MILVUS_WITH_ZSTD_DEFAULT})
 
+define_option(MILVUS_WITH_AWS "Build with AWS SDK" ON)
+
 #----------------------------------------------------------------------
 if(MSVC)
     set_option_category("MSVC")

cpp/cmake/ThirdPartyPackages.cmake

@@ -35,7 +35,8 @@ set(MILVUS_THIRDPARTY_DEPENDENCIES
         Thrift
         yaml-cpp
         ZLIB
-        ZSTD)
+        ZSTD
+        AWS)
 
 message(STATUS "Using ${MILVUS_DEPENDENCY_SOURCE} approach to find dependencies")
 
@@ -83,6 +84,8 @@ macro(build_dependency DEPENDENCY_NAME)
         build_zlib()
     elseif("${DEPENDENCY_NAME}" STREQUAL "ZSTD")
         build_zstd()
+    elseif("${DEPENDENCY_NAME}" STREQUAL "AWS")
+        build_aws()
     else()
         message(FATAL_ERROR "Unknown thirdparty dependency to build: ${DEPENDENCY_NAME}")
     endif ()
@@ -330,6 +333,11 @@ else()
     set(ZSTD_SOURCE_URL "https://github.com/facebook/zstd/archive/${ZSTD_VERSION}.tar.gz")
 endif()
 
+if(DEFINED ENV{MILVUS_AWS_URL})
+    set(AWS_SOURCE_URL "$ENV{MILVUS_AWS_URL}")
+else()
+    set(AWS_SOURCE_URL "https://github.com/aws/aws-sdk-cpp/archive/${AWS_VERSION}.tar.gz")
+endif()
 # ----------------------------------------------------------------------
 # ARROW
 
@@ -1736,3 +1744,68 @@ if(MILVUS_WITH_ZSTD)
     link_directories(SYSTEM ${ZSTD_PREFIX}/lib)
     include_directories(SYSTEM ${ZSTD_INCLUDE_DIR})
 endif()
+
+# ----------------------------------------------------------------------
+# aws
+macro(build_aws)
+    message(STATUS "Building aws-${AWS_VERSION} from source")
+    set(AWS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/aws_ep-prefix/src/aws_ep")
+
+    set(AWS_CMAKE_ARGS
+            ${EP_COMMON_TOOLCHAIN}
+            "-DCMAKE_INSTALL_PREFIX=${AWS_PREFIX}"
+            -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+            -DCMAKE_INSTALL_LIBDIR=lib #${CMAKE_INSTALL_LIBDIR}
+            -DBUILD_ONLY=s3
+            -DBUILD_SHARED_LIBS=off
+            -DENABLE_TESTING=off
+            -DENABLE_UNITY_BUILD=on
+            -DNO_ENCRYPTION=off)
+
+    set(AWS_STATIC_LIB "${AWS_PREFIX}/lib/libs3.a")
+    # Only pass our C flags on Unix as on MSVC it leads to a
+    # "incompatible command-line options" error
+    set(AWS_CMAKE_ARGS
+            ${AWS_CMAKE_ARGS}
+            -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+            -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+            -DCMAKE_C_FLAGS=${EP_C_FLAGS}
+            -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS})
+
+    if(CMAKE_VERSION VERSION_LESS 3.7)
+        message(FATAL_ERROR "Building AWS using ExternalProject requires at least CMake 3.7")
+    endif()
+
+    externalproject_add(aws_ep
+            ${EP_LOG_OPTIONS}
+            CMAKE_ARGS
+            ${AWS_CMAKE_ARGS}
+            BUILD_COMMAND
+            ${MAKE}
+            ${MAKE_BUILD_ARGS}
+            INSTALL_DIR
+            ${AWS_PREFIX}
+            URL
+            ${AWS_SOURCE_URL}
+            BUILD_BYPRODUCTS
+            "${AWS_STATIC_LIB}")
+
+
+    file(MAKE_DIRECTORY "${AWS_PREFIX}/include")
+
+    add_library(aws STATIC IMPORTED)
+    set_target_properties(aws
+            PROPERTIES IMPORTED_LOCATION "${AWS_STATIC_LIB}"
+            INTERFACE_INCLUDE_DIRECTORIES "${AWS_PREFIX}/include")
+
+    add_dependencies(aws aws_ep)
+endmacro()
+
+if(MILVUS_WITH_AWS)
+    resolve_dependency(AWS)
+
+    # TODO: Don't use global includes but rather target_include_directories
+    get_target_property(AWS_INCLUDE_DIR aws INTERFACE_INCLUDE_DIRECTORIES)
+    link_directories(SYSTEM ${AWS_PREFIX}/lib)
+    include_directories(SYSTEM ${AWS_INCLUDE_DIR})
+endif()

cpp/src/CMakeLists.txt

@@ -14,12 +14,6 @@ aux_source_directory(db/scheduler db_scheduler_files)
 aux_source_directory(wrapper wrapper_files)
 aux_source_directory(metrics metrics_files)
 
-#set(metrics_files
-#        metrics/Metrics.cpp
-#        metrics/MetricBase.h
-#)
-
-
 set(license_check_files
         license/LicenseLibrary.cpp
         license/LicenseCheck.cpp
@@ -53,6 +47,10 @@ set(engine_files
 set(get_sys_info_files
         license/GetSysInfo.cpp)
 
+set(s3_client_files
+        storage/s3/S3ClientWrapper.cpp
+        storage/s3/S3ClientWrapper.h)
+
 include_directories(/usr/include)
 include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include")
 include_directories(thrift/gen-cpp)

cpp/src/db/Status.h

@@ -7,39 +7,44 @@
 
 #include <string>
 
+
 namespace zilliz {
 namespace milvus {
 namespace engine {
 
 class Status {
-public:
+ public:
     Status() noexcept : state_(nullptr) {}
     ~Status() { delete[] state_; }
 
-    Status(const Status& rhs);
-    Status& operator=(const Status& rhs);
+    Status(const Status &rhs);
+    Status &operator=(const Status &rhs);
 
-    Status(Status&& rhs) noexcept : state_(rhs.state_) { rhs.state_ = nullptr; }
-    Status& operator=(Status&& rhs_) noexcept;
+    Status(Status &&rhs) noexcept : state_(rhs.state_) { rhs.state_ = nullptr; }
+    Status &operator=(Status &&rhs_) noexcept;
 
     static Status OK() { return Status(); }
-    static Status NotFound(const std::string& msg, const std::string& msg2="") {
+    static Status NotFound(const std::string &msg, const std::string &msg2 = "") {
         return Status(kNotFound, msg, msg2);
     }
-    static Status Error(const std::string& msg, const std::string& msg2="") {
+    static Status Error(const std::string &msg, const std::string &msg2 = "") {
         return Status(kError, msg, msg2);
     }
 
-    static Status InvalidDBPath(const std::string& msg, const std::string& msg2="") {
+    static Status InvalidDBPath(const std::string &msg, const std::string &msg2 = "") {
         return Status(kInvalidDBPath, msg, msg2);
     }
-    static Status GroupError(const std::string& msg, const std::string& msg2="") {
+    static Status GroupError(const std::string &msg, const std::string &msg2 = "") {
         return Status(kGroupError, msg, msg2);
     }
-    static Status DBTransactionError(const std::string& msg, const std::string& msg2="") {
+    static Status DBTransactionError(const std::string &msg, const std::string &msg2 = "") {
         return Status(kDBTransactionError, msg, msg2);
     }
 
+    static Status AlreadyExist(const std::string &msg, const std::string &msg2 = "") {
+        return Status(kAlreadyExist, msg, msg2);
+    }
+
     bool ok() const { return state_ == nullptr; }
 
     bool IsNotFound() const { return code() == kNotFound; }
@@ -48,11 +53,12 @@ public:
     bool IsInvalidDBPath() const { return code() == kInvalidDBPath; }
     bool IsGroupError() const { return code() == kGroupError; }
     bool IsDBTransactionError() const { return code() == kDBTransactionError; }
+    bool IsAlreadyExist() const { return code() == kAlreadyExist; }
 
     std::string ToString() const;
 
-private:
-    const char* state_;
+ private:
+    const char *state_ = nullptr;
 
     enum Code {
         kOK = 0,
@@ -62,21 +68,23 @@ private:
         kInvalidDBPath,
         kGroupError,
         kDBTransactionError,
+
+        kAlreadyExist,
     };
 
     Code code() const {
         return (state_ == nullptr) ? kOK : static_cast<Code>(state_[4]);
     }
-    Status(Code code, const std::string& msg, const std::string& msg2);
-    static const char* CopyState(const char* s);
+    Status(Code code, const std::string &msg, const std::string &msg2);
+    static const char *CopyState(const char *s);
 
 }; // Status
 
-inline Status::Status(const Status& rhs) {
+inline Status::Status(const Status &rhs) {
     state_ = (rhs.state_ == nullptr) ? nullptr : CopyState(rhs.state_);
 }
 
-inline Status& Status::operator=(const Status& rhs) {
+inline Status &Status::operator=(const Status &rhs) {
     if (state_ != rhs.state_) {
         delete[] state_;
         state_ = (rhs.state_ == nullptr) ? nullptr : CopyState(rhs.state_);
@@ -84,7 +92,7 @@ inline Status& Status::operator=(const Status& rhs) {
     return *this;
 }
 
-inline Status& Status::operator=(Status&& rhs) noexcept {
+inline Status &Status::operator=(Status &&rhs) noexcept {
     std::swap(state_, rhs.state_);
     return *this;
 }

cpp/src/storage/IStorage.h

+#pragma once
+
+#include "db/Status.h"
+
+#include <string>
+
+
+namespace zilliz {
+namespace milvus {
+namespace engine {
+namespace storage {
+
+class IStorage {
+ public:
+    virtual Status Create(const std::string &ip_address,
+                          const std::string &port,
+                          const std::string &access_key,
+                          const std::string &secret_key) = 0;
+
+    virtual Status Close() = 0;
+
+    virtual Status CreateBucket(std::string& bucket_name) = 0;
+    virtual Status DeleteBucket(std::string& bucket_name) = 0;
+    virtual Status UploadFile(std::string &bucket_name, std::string &object_key, std::string &path_key) = 0;
+    virtual Status DownloadFile(std::string &bucket_name, std::string &object_key, std::string &path_key) = 0;
+    virtual Status DeleteFile(std::string &bucket_name, std::string &object_key) = 0;
+};
+
+}
+}
+}
+}
\ No newline at end of file

cpp/src/storage/s3/S3ClientWrapper.cpp

+#include "S3ClientWrapper.h"
+
+#include <aws/s3/model/CreateBucketRequest.h>
+#include <aws/s3/model/DeleteBucketRequest.h>
+#include <aws/s3/model/PutObjectRequest.h>
+#include <aws/s3/model/GetObjectRequest.h>
+#include <aws/s3/model/DeleteObjectRequest.h>
+
+#include <iostream>
+#include <fstream>
+
+
+namespace zilliz {
+namespace milvus {
+namespace engine {
+namespace storage {
+
+Status
+S3ClientWrapper::Create(const std::string &ip_address,
+                        const std::string &port,
+                        const std::string &access_key,
+                        const std::string &secret_key) {
+    Aws::InitAPI(options_);
+    Aws::Client::ClientConfiguration cfg;
+
+    // TODO: ip_address need to be validated.
+
+    cfg.endpointOverride = ip_address + ":" + port; // S3 server ip address and port
+    cfg.scheme = Aws::Http::Scheme::HTTP;
+    cfg.verifySSL =
+        false; //Aws::Auth::AWSCredentials cred("RPW421T9GSIO4A45Y9ZR", "2owKYy9emSS90Q0pXuyqpX1OxBCyEDYodsiBemcq"); // 认证的Key
+    client_ =
+        new S3Client(Aws::Auth::AWSCredentials(access_key, secret_key),
+                     cfg,
+                     Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always,
+                     false);
+    if (client_ == nullptr) {
+        std::string error = "Can't connect server.";
+        return Status::Error(error);
+    } else {
+        return Status::OK();
+    }
+}
+
+
+Status
+S3ClientWrapper::Close() {
+    if (client_ != nullptr) {
+        delete client_;
+        client_ = nullptr;
+    }
+    Aws::ShutdownAPI(options_);
+    return Status::OK();
+}
+
+Status
+S3ClientWrapper::CreateBucket(std::string& bucket_name) {
+    Aws::S3::Model::CreateBucketRequest request;
+    request.SetBucket(bucket_name);
+
+    auto outcome = client_->CreateBucket(request);
+
+    if (outcome.IsSuccess())
+    {
+        return Status::OK();
+    }
+    else
+    {
+        std::cout << "CreateBucket error: "
+                  << outcome.GetError().GetExceptionName() << std::endl
+                  << outcome.GetError().GetMessage() << std::endl;
+        switch(outcome.GetError().GetErrorType()) {
+            case Aws::S3::S3Errors::BUCKET_ALREADY_EXISTS:
+            case Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU:
+                return Status::AlreadyExist(outcome.GetError().GetMessage());
+            default:
+                return Status::Error(outcome.GetError().GetMessage());
+        }
+    }
+}
+
+Status
+S3ClientWrapper::DeleteBucket(std::string& bucket_name) {
+    Aws::S3::Model::DeleteBucketRequest bucket_request;
+    bucket_request.SetBucket(bucket_name);
+
+    auto outcome = client_->DeleteBucket(bucket_request);
+
+    if (outcome.IsSuccess())
+    {
+        return Status::OK();
+    }
+    else
+    {
+        std::cout << "DeleteBucket error: "
+                  << outcome.GetError().GetExceptionName() << " - "
+                  << outcome.GetError().GetMessage() << std::endl;
+        return Status::Error(outcome.GetError().GetMessage());
+    }
+}
+
+Status
+S3ClientWrapper::UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) {
+
+    PutObjectRequest putObjectRequest;
+    putObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str());
+
+    auto input_data = Aws::MakeShared<Aws::FStream>("PutObjectInputStream",
+                                                    pathkey.c_str(),
+                                                    std::ios_base::in | std::ios_base::binary);
+    putObjectRequest.SetBody(input_data);
+    auto put_object_result = client_->PutObject(putObjectRequest);
+    if (put_object_result.IsSuccess()) {
+        return Status::OK();
+    } else {
+        std::cout << "PutObject error: " << put_object_result.GetError().GetExceptionName() << " "
+                  << put_object_result.GetError().GetMessage() << std::endl;
+        return Status::Error(put_object_result.GetError().GetMessage());
+    }
+}
+
+Status
+S3ClientWrapper::DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) {
+    GetObjectRequest object_request;
+    object_request.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str());
+    auto get_object_outcome = client_->GetObject(object_request);
+    if (get_object_outcome.IsSuccess()) {
+        Aws::OFStream local_file(pathkey.c_str(), std::ios::out | std::ios::binary);
+        local_file << get_object_outcome.GetResult().GetBody().rdbuf();
+        return Status::OK();
+    } else {
+        std::cout << "GetObject error: " << get_object_outcome.GetError().GetExceptionName() << " "
+                  << get_object_outcome.GetError().GetMessage() << std::endl;
+        return Status::Error(get_object_outcome.GetError().GetMessage());
+    }
+}
+
+Status
+S3ClientWrapper::DeleteFile(std::string &bucket_name, std::string &object_key) {
+    Aws::S3::Model::DeleteObjectRequest object_request;
+    object_request.WithBucket(bucket_name).WithKey(object_key);
+
+    auto delete_object_outcome = client_->DeleteObject(object_request);
+
+    if (delete_object_outcome.IsSuccess()) {
+        return Status::OK();
+    } else {
+        std::cout << "DeleteObject error: " <<
+                  delete_object_outcome.GetError().GetExceptionName() << " " <<
+                  delete_object_outcome.GetError().GetMessage() << std::endl;
+
+        return Status::Error(delete_object_outcome.GetError().GetMessage());
+    }
+}
+
+}
+}
+}
+}
\ No newline at end of file

cpp/src/storage/s3/S3ClientWrapper.h

+#pragma once
+
+#include "storage/IStorage.h"
+
+
+#include <aws/s3/S3Client.h>
+#include <aws/core/Aws.h>
+#include <aws/core/auth/AWSCredentialsProvider.h>
+
+
+using namespace Aws::S3;
+using namespace Aws::S3::Model;
+
+namespace zilliz {
+namespace milvus {
+namespace engine {
+namespace storage {
+
+class S3ClientWrapper : public IStorage {
+ public:
+
+    S3ClientWrapper() = default;
+    ~S3ClientWrapper() = default;
+
+    Status Create(const std::string &ip_address,
+                const std::string &port,
+                const std::string &access_key,
+                const std::string &secret_key) override;
+    Status Close() override;
+
+    Status CreateBucket(std::string& bucket_name) override;
+    Status DeleteBucket(std::string& bucket_name) override;
+    Status UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override;
+    Status DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override;
+    Status DeleteFile(std::string &bucket_name, std::string &object_key) override;
+
+ private:
+    S3Client *client_ = nullptr;
+    Aws::SDKOptions options_;
+};
+
+}
+}
+}
+}
\ No newline at end of file

cpp/thirdparty/versions.txt

@@ -17,5 +17,6 @@ THRIFT_VERSION=v0.12.0
 YAMLCPP_VERSION=0.6.2
 ZLIB_VERSION=v1.2.11
 ZSTD_VERSION=v1.4.0
+AWS_VERSION=1.7.125
 
 # vim: set filetype=sh:

cpp/unittest/CMakeLists.txt

@@ -8,12 +8,22 @@ link_directories(
         "${GTEST_PREFIX}/lib/"
 
 )
+
+aux_source_directory(${MILVUS_ENGINE_SRC}/db db_srcs)
+aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
+
 message(STATUS "GTEST LIB: ${GTEST_PREFIX}/lib")
 
 set(unittest_srcs
         ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp)
         #${EASYLOGGINGPP_INCLUDE_DIR}/easylogging++.cc)
 
+set(require_files
+        ${MILVUS_ENGINE_SRC}/server/ServerConfig.cpp
+        ${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp
+        ${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp
+        )
+
 set(unittest_libs
         yaml-cpp
         gtest
@@ -37,4 +47,5 @@ add_subdirectory(server)
 add_subdirectory(db)
 add_subdirectory(faiss_wrapper)
 add_subdirectory(license)
-add_subdirectory(metrics)
\ No newline at end of file
+add_subdirectory(metrics)
+add_subdirectory(storage)
\ No newline at end of file

cpp/unittest/db/CMakeLists.txt

@@ -3,22 +3,16 @@
 # Unauthorized copying of this file, via any medium is strictly prohibited.
 # Proprietary and confidential.
 #-------------------------------------------------------------------------------
-include_directories(../../src)
-
-aux_source_directory(../../src/db db_srcs)
-aux_source_directory(../../src/db/scheduler db_scheduler_srcs)
-aux_source_directory(../../src/config config_files)
-aux_source_directory(../../src/cache cache_srcs)
-aux_source_directory(../../src/wrapper wrapper_src)
+aux_source_directory(${MILVUS_ENGINE_SRC}/db db_srcs)
+aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler db_scheduler_srcs)
+aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
+aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs)
+aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src)
 
 include_directories(/usr/local/cuda/include)
 link_directories("/usr/local/cuda/lib64")
 
-set(require_files
-        ../../src/server/ServerConfig.cpp
-        ../../src/utils/CommonUtil.cpp
-        ../../src/utils/TimeRecorder.cpp
-        )
+
 
 set(db_test_src
     ${unittest_srcs}

cpp/unittest/faiss_wrapper/CMakeLists.txt

@@ -3,21 +3,13 @@
 # Unauthorized copying of this file, via any medium is strictly prohibited.
 # Proprietary and confidential.
 #-------------------------------------------------------------------------------
-include_directories(../../src)
-
-aux_source_directory(../../src/wrapper wrapper_src)
-aux_source_directory(../../src/config config_files)
+aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src)
+aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
 
 # Make sure that your call to link_directories takes place before your call to the relevant add_executable.
 include_directories(/usr/local/cuda/include)
 link_directories("/usr/local/cuda/lib64")
 
-set(require_files
-        ../../src/server/ServerConfig.cpp
-        ../../src/utils/CommonUtil.cpp
-        ../../src/utils/TimeRecorder.cpp
-        )
-
 set(wrapper_test_src
         ${unittest_srcs}
         ${wrapper_src}

cpp/unittest/license/CMakeLists.txt

@@ -3,12 +3,10 @@
 # Unauthorized copying of this file, via any medium is strictly prohibited.
 # Proprietary and confidential.
 #-------------------------------------------------------------------------------
-include_directories(../../src)
-
-aux_source_directory(../../src/db db_srcs)
-aux_source_directory(../../src/config config_files)
-aux_source_directory(../../src/cache cache_srcs)
-aux_source_directory(../../src/wrapper wrapper_src)
+aux_source_directory(${MILVUS_ENGINE_SRC}/db db_srcs)
+aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
+aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs)
+aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src)
 
 include_directories(/usr/local/cuda/include)
 link_directories(/usr/local/cuda)
@@ -18,19 +16,16 @@ link_directories(/usr/lib/nvidia-415)
 link_directories(/usr/local/cuda/targets/x86_64-linux/lib/stubs/)
 link_directories(/usr/local/cuda/lib64/stubs/)
 
-set(require_files
-        ../../src/license/LicenseLibrary.cpp
-        ../../src/license/LicenseCheck.cpp
+set(license_src_files
+        ${MILVUS_ENGINE_SRC}/license/LicenseLibrary.cpp
+        ${MILVUS_ENGINE_SRC}/license/LicenseCheck.cpp
     )
 
 
 set(db_test_src
-
     license_library_tests.cpp
     license_check_test.cpp
-
-
-        ${require_files})
+    ${license_src_files})
 
 cuda_add_executable(license_test ${db_test_src})
 

cpp/unittest/metrics/CMakeLists.txt

@@ -17,6 +17,7 @@ aux_source_directory(../../src/db/scheduler db_scheduler_srcs)
 aux_source_directory(../../src/config config_files)
 aux_source_directory(../../src/cache cache_srcs)
 aux_source_directory(../../src/wrapper wrapper_src)
+aux_source_directory(../../src/metrics metrics_src)
 
 
 include_directories(/usr/include)
@@ -28,32 +29,26 @@ link_directories("/usr/local/cuda/lib64")
 include_directories(../../src/metrics)
 
 
-set(require_files
-        ../../src/metrics/Metrics.cpp
-
-#        ../../src/cache/CacheMgr.cpp
-        ../../src/metrics/PrometheusMetrics.cpp
-        ../../src/metrics/MetricBase.h
-        ../../src/server/ServerConfig.cpp
-        ../../src/utils/CommonUtil.cpp
-        ../../src/utils/TimeRecorder.cpp
+#set(metrics_src_files
+#        ../../src/metrics/Metrics.cpp
+#        ../../src/metrics/Metrics.h
 #        ../../src/metrics/PrometheusMetrics.cpp
-        )
+#        ../../src/metrics/MetricBase.h
+#        ../../src/server/ServerConfig.cpp
+#        ../../src/utils/CommonUtil.cpp
+#        ../../src/utils/TimeRecorder.cpp
+#        )
 
 set(count_test_src
-        #        ${unittest_srcs}
-        #        ${config_files}
-        #        ${require_files}
         ${unittest_srcs}
         ${config_files}
         ${cache_srcs}
         ${db_srcs}
         ${db_scheduler_srcs}
         ${wrapper_src}
-        ${require_files}
+        ${metrics_src}
         metrics_test.cpp
         ../db/utils.cpp
-        ../../src/metrics/Metrics.h
         )
 
 

cpp/unittest/metrics/metrics_test.cpp

@@ -102,7 +102,7 @@ TEST_F(DBTest, Metric_Tes) {
         }
     });
 
-    int loop = 100000;
+    int loop = 10;
 
     for (auto i=0; i<loop; ++i) {
         if (i==40) {

cpp/unittest/server/CMakeLists.txt

@@ -14,12 +14,9 @@ aux_source_directory(../../src/cache cache_srcs)
 aux_source_directory(../../src/wrapper wrapper_src)
 aux_source_directory(./ test_srcs)
 
-set(require_files
-        ../../src/server/ServerConfig.cpp
-        ../../src/utils/CommonUtil.cpp
-        ../../src/utils/TimeRecorder.cpp
-        ../../src/utils/StringHelpFunctions.cpp
-        ../../src/utils/AttributeSerializer.cpp
+set(server_src_files
+        ${MILVUS_ENGINE_SRC}/utils/StringHelpFunctions.cpp
+        ${MILVUS_ENGINE_SRC}/utils/AttributeSerializer.cpp
         )
 
 cuda_add_executable(server_test
@@ -28,6 +25,7 @@ cuda_add_executable(server_test
         ${cache_srcs}
         ${wrapper_src}
         ${test_srcs}
+        ${server_src_files}
         ${require_files}
         )
 

cpp/unittest/storage/CMakeLists.txt

+#-------------------------------------------------------------------------------
+# Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
+# Unauthorized copying of this file, via any medium is strictly prohibited.
+# Proprietary and confidential.
+#-------------------------------------------------------------------------------
+aux_source_directory(${MILVUS_ENGINE_SRC}/storage/s3 s3_client_src)
+
+# Make sure that your call to link_directories takes place before your call to the relevant add_executable.
+include_directories(/usr/local/cuda/include)
+link_directories("/usr/local/cuda/lib64")
+
+set(s3_client_test_src
+        ${unittest_srcs}
+        ${s3_client_src}
+        ${require_files}
+        s3_client_test.cpp
+        ${MILVUS_ENGINE_SRC}/db/Status.cpp
+        )
+
+add_executable(s3_test
+        ${s3_client_test_src}
+        ${config_files})
+
+set(s3_client_libs
+        stdc++
+        libaws-cpp-sdk-s3.a
+        libaws-cpp-sdk-core.a
+        libaws-c-event-stream.a
+        libaws-checksums.a
+        libaws-c-common.a
+        boost_filesystem
+        )
+target_link_libraries(s3_test
+        ${s3_client_libs}
+        ${unittest_libs}
+        curl
+        crypto)

cpp/unittest/storage/s3_client_test.cpp

+////////////////////////////////////////////////////////////////////////////////
+// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
+// Unauthorized copying of this file, via any medium is strictly prohibited.
+// Proprietary and confidential.
+////////////////////////////////////////////////////////////////////////////////
+
+#include "storage/IStorage.h"
+#include "storage/s3/S3ClientWrapper.h"
+#include <gtest/gtest.h>
+#include <memory.h>
+#include <cmake-build-debug/boost_ep-prefix/src/boost_ep/boost/exception/detail/shared_ptr.hpp>
+#include <fstream>
+
+
+using namespace zilliz::milvus::engine;
+
+TEST(s3_client_wrapper, CLIENT_WRAPPER_TEST) {
+
+    std::shared_ptr<storage::IStorage> storage_ptr = std::make_shared<storage::S3ClientWrapper>();
+
+    std::string ip_address = "127.0.0.1";
+    std::string port = "9000";
+    std::string access_key = "AKIAIOSFODNN7EXAMPLE";
+    std::string secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY";
+
+    Status status = storage_ptr->Create(ip_address, port, access_key, secret_key);
+    ASSERT_TRUE(status.ok());
+
+    std::string filename = "/tmp/s3_test_file";
+    std::string bucket_name = "bucktname";
+    std::string object_name = "test_file";
+
+    status = storage_ptr->CreateBucket(bucket_name);
+    std::cout << status.IsAlreadyExist() << std::endl;
+    if (status.IsAlreadyExist()) {
+        status = storage_ptr->DeleteBucket(bucket_name);
+        status = storage_ptr->CreateBucket(bucket_name);
+    }
+
+    ASSERT_TRUE(status.ok());
+
+    std::ofstream ofile(filename);
+    std::stringstream ss;
+    for (int i = 0; i < 1024; ++i) {
+        ss << i;
+    }
+    ofile << ss.str() << std::endl;
+    ofile.close();
+    status = storage_ptr->UploadFile(bucket_name, object_name, filename);
+    ASSERT_TRUE(status.ok());
+
+    status = storage_ptr->DownloadFile(bucket_name, object_name, filename);
+    std::ifstream infile(filename);
+    std::string in_buffer;
+    infile >> in_buffer;
+    ASSERT_STREQ(in_buffer.c_str(), ss.str().c_str());
+
+    status = storage_ptr->DeleteFile(bucket_name, object_name);
+    ASSERT_TRUE(status.ok());
+
+    status = storage_ptr->DeleteBucket(bucket_name);
+    ASSERT_TRUE(status.ok());
+
+    status = storage_ptr->Close();
+    ASSERT_TRUE(status.ok());
+}
+