Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
0f46d463
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
0f46d463
编写于
6月 28, 2019
作者:
S
starlord
提交者:
jinhai
7月 13, 2019
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
MS-129 Add more unitest
Former-commit-id: 6d0b616c8efca6f281aca0a9db82055bcde19abf
上级
007cde0e
变更
8
隐藏空白更改
内联
并排
Showing
8 changed file
with
380 addition
and
337 deletion
+380
-337
cpp/src/sdk/src/client/ClientProxy.cpp
cpp/src/sdk/src/client/ClientProxy.cpp
+0
-1
cpp/src/sdk/src/util/ConvertUtil.cpp
cpp/src/sdk/src/util/ConvertUtil.cpp
+0
-44
cpp/src/sdk/src/util/ConvertUtil.h
cpp/src/sdk/src/util/ConvertUtil.h
+0
-18
cpp/src/storage/s3/S3ClientWrapper.cpp
cpp/src/storage/s3/S3ClientWrapper.cpp
+159
-159
cpp/src/storage/s3/S3ClientWrapper.h
cpp/src/storage/s3/S3ClientWrapper.h
+45
-45
cpp/unittest/server/cache_test.cpp
cpp/unittest/server/cache_test.cpp
+33
-4
cpp/unittest/server/config_test.cpp
cpp/unittest/server/config_test.cpp
+77
-0
cpp/unittest/storage/S3ClientTest.cpp
cpp/unittest/storage/S3ClientTest.cpp
+66
-66
未找到文件。
cpp/src/sdk/src/client/ClientProxy.cpp
浏览文件 @
0f46d463
...
...
@@ -4,7 +4,6 @@
* Proprietary and confidential.
******************************************************************************/
#include "ClientProxy.h"
#include "util/ConvertUtil.h"
namespace
milvus
{
...
...
cpp/src/sdk/src/util/ConvertUtil.cpp
已删除
100644 → 0
浏览文件 @
007cde0e
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#include "ConvertUtil.h"
#include "Exception.h"
#include <map>
namespace
milvus
{
static
const
std
::
string
INDEX_RAW
=
"raw"
;
static
const
std
::
string
INDEX_IVFFLAT
=
"ivfflat"
;
std
::
string
ConvertUtil
::
IndexType2Str
(
IndexType
index
)
{
static
const
std
::
map
<
IndexType
,
std
::
string
>
s_index2str
=
{
{
IndexType
::
cpu_idmap
,
INDEX_RAW
},
{
IndexType
::
gpu_ivfflat
,
INDEX_IVFFLAT
}
};
const
auto
&
iter
=
s_index2str
.
find
(
index
);
if
(
iter
==
s_index2str
.
end
())
{
throw
Exception
(
StatusCode
::
InvalidAgument
,
"Invalid index type"
);
}
return
iter
->
second
;
}
IndexType
ConvertUtil
::
Str2IndexType
(
const
std
::
string
&
type
)
{
static
const
std
::
map
<
std
::
string
,
IndexType
>
s_str2index
=
{
{
INDEX_RAW
,
IndexType
::
cpu_idmap
},
{
INDEX_IVFFLAT
,
IndexType
::
gpu_ivfflat
}
};
const
auto
&
iter
=
s_str2index
.
find
(
type
);
if
(
iter
==
s_str2index
.
end
())
{
throw
Exception
(
StatusCode
::
InvalidAgument
,
"Invalid index type"
);
}
return
iter
->
second
;
}
}
\ No newline at end of file
cpp/src/sdk/src/util/ConvertUtil.h
已删除
100644 → 0
浏览文件 @
007cde0e
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include "MilvusApi.h"
namespace
milvus
{
class
ConvertUtil
{
public:
static
std
::
string
IndexType2Str
(
IndexType
index
);
static
IndexType
Str2IndexType
(
const
std
::
string
&
type
);
};
}
cpp/src/storage/s3/S3ClientWrapper.cpp
浏览文件 @
0f46d463
#include "S3ClientWrapper.h"
#include <aws/s3/model/CreateBucketRequest.h>
#include <aws/s3/model/DeleteBucketRequest.h>
#include <aws/s3/model/PutObjectRequest.h>
#include <aws/s3/model/GetObjectRequest.h>
#include <aws/s3/model/DeleteObjectRequest.h>
#include <iostream>
#include <fstream>
namespace
zilliz
{
namespace
milvus
{
namespace
engine
{
namespace
storage
{
Status
S3ClientWrapper
::
Create
(
const
std
::
string
&
ip_address
,
const
std
::
string
&
port
,
const
std
::
string
&
access_key
,
const
std
::
string
&
secret_key
)
{
Aws
::
InitAPI
(
options_
);
Aws
::
Client
::
ClientConfiguration
cfg
;
// TODO: ip_address need to be validated.
cfg
.
endpointOverride
=
ip_address
+
":"
+
port
;
// S3 server ip address and port
cfg
.
scheme
=
Aws
::
Http
::
Scheme
::
HTTP
;
cfg
.
verifySSL
=
false
;
//Aws::Auth::AWSCredentials cred("RPW421T9GSIO4A45Y9ZR", "2owKYy9emSS90Q0pXuyqpX1OxBCyEDYodsiBemcq"); // 认证的Key
client_
=
new
S3Client
(
Aws
::
Auth
::
AWSCredentials
(
access_key
,
secret_key
),
cfg
,
Aws
::
Client
::
AWSAuthV4Signer
::
PayloadSigningPolicy
::
Always
,
false
);
if
(
client_
==
nullptr
)
{
std
::
string
error
=
"Can't connect server."
;
return
Status
::
Error
(
error
);
}
else
{
return
Status
::
OK
();
}
}
Status
S3ClientWrapper
::
Close
()
{
if
(
client_
!=
nullptr
)
{
delete
client_
;
client_
=
nullptr
;
}
Aws
::
ShutdownAPI
(
options_
);
return
Status
::
OK
();
}
Status
S3ClientWrapper
::
CreateBucket
(
std
::
string
&
bucket_name
)
{
Aws
::
S3
::
Model
::
CreateBucketRequest
request
;
request
.
SetBucket
(
bucket_name
);
auto
outcome
=
client_
->
CreateBucket
(
request
);
if
(
outcome
.
IsSuccess
())
{
return
Status
::
OK
();
}
else
{
std
::
cout
<<
"CreateBucket error: "
<<
outcome
.
GetError
().
GetExceptionName
()
<<
std
::
endl
<<
outcome
.
GetError
().
GetMessage
()
<<
std
::
endl
;
switch
(
outcome
.
GetError
().
GetErrorType
())
{
case
Aws
::
S3
::
S3Errors
::
BUCKET_ALREADY_EXISTS
:
case
Aws
::
S3
::
S3Errors
::
BUCKET_ALREADY_OWNED_BY_YOU
:
return
Status
::
AlreadyExist
(
outcome
.
GetError
().
GetMessage
());
default:
return
Status
::
Error
(
outcome
.
GetError
().
GetMessage
());
}
}
}
Status
S3ClientWrapper
::
DeleteBucket
(
std
::
string
&
bucket_name
)
{
Aws
::
S3
::
Model
::
DeleteBucketRequest
bucket_request
;
bucket_request
.
SetBucket
(
bucket_name
);
auto
outcome
=
client_
->
DeleteBucket
(
bucket_request
);
if
(
outcome
.
IsSuccess
())
{
return
Status
::
OK
();
}
else
{
std
::
cout
<<
"DeleteBucket error: "
<<
outcome
.
GetError
().
GetExceptionName
()
<<
" - "
<<
outcome
.
GetError
().
GetMessage
()
<<
std
::
endl
;
return
Status
::
Error
(
outcome
.
GetError
().
GetMessage
());
}
}
Status
S3ClientWrapper
::
UploadFile
(
std
::
string
&
BucketName
,
std
::
string
&
objectKey
,
std
::
string
&
pathkey
)
{
PutObjectRequest
putObjectRequest
;
putObjectRequest
.
WithBucket
(
BucketName
.
c_str
()).
WithKey
(
objectKey
.
c_str
());
auto
input_data
=
Aws
::
MakeShared
<
Aws
::
FStream
>
(
"PutObjectInputStream"
,
pathkey
.
c_str
(),
std
::
ios_base
::
in
|
std
::
ios_base
::
binary
);
putObjectRequest
.
SetBody
(
input_data
);
auto
put_object_result
=
client_
->
PutObject
(
putObjectRequest
);
if
(
put_object_result
.
IsSuccess
())
{
return
Status
::
OK
();
}
else
{
std
::
cout
<<
"PutObject error: "
<<
put_object_result
.
GetError
().
GetExceptionName
()
<<
" "
<<
put_object_result
.
GetError
().
GetMessage
()
<<
std
::
endl
;
return
Status
::
Error
(
put_object_result
.
GetError
().
GetMessage
());
}
}
Status
S3ClientWrapper
::
DownloadFile
(
std
::
string
&
BucketName
,
std
::
string
&
objectKey
,
std
::
string
&
pathkey
)
{
GetObjectRequest
object_request
;
object_request
.
WithBucket
(
BucketName
.
c_str
()).
WithKey
(
objectKey
.
c_str
());
auto
get_object_outcome
=
client_
->
GetObject
(
object_request
);
if
(
get_object_outcome
.
IsSuccess
())
{
Aws
::
OFStream
local_file
(
pathkey
.
c_str
(),
std
::
ios
::
out
|
std
::
ios
::
binary
);
local_file
<<
get_object_outcome
.
GetResult
().
GetBody
().
rdbuf
();
return
Status
::
OK
();
}
else
{
std
::
cout
<<
"GetObject error: "
<<
get_object_outcome
.
GetError
().
GetExceptionName
()
<<
" "
<<
get_object_outcome
.
GetError
().
GetMessage
()
<<
std
::
endl
;
return
Status
::
Error
(
get_object_outcome
.
GetError
().
GetMessage
());
}
}
Status
S3ClientWrapper
::
DeleteFile
(
std
::
string
&
bucket_name
,
std
::
string
&
object_key
)
{
Aws
::
S3
::
Model
::
DeleteObjectRequest
object_request
;
object_request
.
WithBucket
(
bucket_name
).
WithKey
(
object_key
);
auto
delete_object_outcome
=
client_
->
DeleteObject
(
object_request
);
if
(
delete_object_outcome
.
IsSuccess
())
{
return
Status
::
OK
();
}
else
{
std
::
cout
<<
"DeleteObject error: "
<<
delete_object_outcome
.
GetError
().
GetExceptionName
()
<<
" "
<<
delete_object_outcome
.
GetError
().
GetMessage
()
<<
std
::
endl
;
return
Status
::
Error
(
delete_object_outcome
.
GetError
().
GetMessage
());
}
}
}
}
}
}
\ No newline at end of file
//#include "S3ClientWrapper.h"
//
//#include <aws/s3/model/CreateBucketRequest.h>
//#include <aws/s3/model/DeleteBucketRequest.h>
//#include <aws/s3/model/PutObjectRequest.h>
//#include <aws/s3/model/GetObjectRequest.h>
//#include <aws/s3/model/DeleteObjectRequest.h>
//
//#include <iostream>
//#include <fstream>
//
//
//namespace zilliz {
//namespace milvus {
//namespace engine {
//namespace storage {
//
//Status
//S3ClientWrapper::Create(const std::string &ip_address,
// const std::string &port,
// const std::string &access_key,
// const std::string &secret_key) {
// Aws::InitAPI(options_);
// Aws::Client::ClientConfiguration cfg;
//
// // TODO: ip_address need to be validated.
//
// cfg.endpointOverride = ip_address + ":" + port; // S3 server ip address and port
// cfg.scheme = Aws::Http::Scheme::HTTP;
// cfg.verifySSL =
// false; //Aws::Auth::AWSCredentials cred("RPW421T9GSIO4A45Y9ZR", "2owKYy9emSS90Q0pXuyqpX1OxBCyEDYodsiBemcq"); // 认证的Key
// client_ =
// new S3Client(Aws::Auth::AWSCredentials(access_key, secret_key),
// cfg,
// Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always,
// false);
// if (client_ == nullptr) {
// std::string error = "Can't connect server.";
// return Status::Error(error);
// } else {
// return Status::OK();
// }
//}
//
//
//Status
//S3ClientWrapper::Close() {
// if (client_ != nullptr) {
// delete client_;
// client_ = nullptr;
// }
// Aws::ShutdownAPI(options_);
// return Status::OK();
//}
//
//Status
//S3ClientWrapper::CreateBucket(std::string& bucket_name) {
// Aws::S3::Model::CreateBucketRequest request;
// request.SetBucket(bucket_name);
//
// auto outcome = client_->CreateBucket(request);
//
// if (outcome.IsSuccess())
// {
// return Status::OK();
// }
// else
// {
// std::cout << "CreateBucket error: "
// << outcome.GetError().GetExceptionName() << std::endl
// << outcome.GetError().GetMessage() << std::endl;
// switch(outcome.GetError().GetErrorType()) {
// case Aws::S3::S3Errors::BUCKET_ALREADY_EXISTS:
// case Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU:
// return Status::AlreadyExist(outcome.GetError().GetMessage());
// default:
// return Status::Error(outcome.GetError().GetMessage());
// }
// }
//}
//
//Status
//S3ClientWrapper::DeleteBucket(std::string& bucket_name) {
// Aws::S3::Model::DeleteBucketRequest bucket_request;
// bucket_request.SetBucket(bucket_name);
//
// auto outcome = client_->DeleteBucket(bucket_request);
//
// if (outcome.IsSuccess())
// {
// return Status::OK();
// }
// else
// {
// std::cout << "DeleteBucket error: "
// << outcome.GetError().GetExceptionName() << " - "
// << outcome.GetError().GetMessage() << std::endl;
// return Status::Error(outcome.GetError().GetMessage());
// }
//}
//
//Status
//S3ClientWrapper::UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) {
//
// PutObjectRequest putObjectRequest;
// putObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str());
//
// auto input_data = Aws::MakeShared<Aws::FStream>("PutObjectInputStream",
// pathkey.c_str(),
// std::ios_base::in | std::ios_base::binary);
// putObjectRequest.SetBody(input_data);
// auto put_object_result = client_->PutObject(putObjectRequest);
// if (put_object_result.IsSuccess()) {
// return Status::OK();
// } else {
// std::cout << "PutObject error: " << put_object_result.GetError().GetExceptionName() << " "
// << put_object_result.GetError().GetMessage() << std::endl;
// return Status::Error(put_object_result.GetError().GetMessage());
// }
//}
//
//Status
//S3ClientWrapper::DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) {
// GetObjectRequest object_request;
// object_request.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str());
// auto get_object_outcome = client_->GetObject(object_request);
// if (get_object_outcome.IsSuccess()) {
// Aws::OFStream local_file(pathkey.c_str(), std::ios::out | std::ios::binary);
// local_file << get_object_outcome.GetResult().GetBody().rdbuf();
// return Status::OK();
// } else {
// std::cout << "GetObject error: " << get_object_outcome.GetError().GetExceptionName() << " "
// << get_object_outcome.GetError().GetMessage() << std::endl;
// return Status::Error(get_object_outcome.GetError().GetMessage());
// }
//}
//
//Status
//S3ClientWrapper::DeleteFile(std::string &bucket_name, std::string &object_key) {
// Aws::S3::Model::DeleteObjectRequest object_request;
// object_request.WithBucket(bucket_name).WithKey(object_key);
//
// auto delete_object_outcome = client_->DeleteObject(object_request);
//
// if (delete_object_outcome.IsSuccess()) {
// return Status::OK();
// } else {
// std::cout << "DeleteObject error: " <<
// delete_object_outcome.GetError().GetExceptionName() << " " <<
// delete_object_outcome.GetError().GetMessage() << std::endl;
//
// return Status::Error(delete_object_outcome.GetError().GetMessage());
// }
//}
//
//}
//}
//}
//}
\ No newline at end of file
cpp/src/storage/s3/S3ClientWrapper.h
浏览文件 @
0f46d463
#pragma once
#include "storage/IStorage.h"
#include <aws/s3/S3Client.h>
#include <aws/core/Aws.h>
#include <aws/core/auth/AWSCredentialsProvider.h>
using
namespace
Aws
::
S3
;
using
namespace
Aws
::
S3
::
Model
;
namespace
zilliz
{
namespace
milvus
{
namespace
engine
{
namespace
storage
{
class
S3ClientWrapper
:
public
IStorage
{
public:
S3ClientWrapper
()
=
default
;
~
S3ClientWrapper
()
=
default
;
Status
Create
(
const
std
::
string
&
ip_address
,
const
std
::
string
&
port
,
const
std
::
string
&
access_key
,
const
std
::
string
&
secret_key
)
override
;
Status
Close
()
override
;
Status
CreateBucket
(
std
::
string
&
bucket_name
)
override
;
Status
DeleteBucket
(
std
::
string
&
bucket_name
)
override
;
Status
UploadFile
(
std
::
string
&
BucketName
,
std
::
string
&
objectKey
,
std
::
string
&
pathkey
)
override
;
Status
DownloadFile
(
std
::
string
&
BucketName
,
std
::
string
&
objectKey
,
std
::
string
&
pathkey
)
override
;
Status
DeleteFile
(
std
::
string
&
bucket_name
,
std
::
string
&
object_key
)
override
;
private:
S3Client
*
client_
=
nullptr
;
Aws
::
SDKOptions
options_
;
};
}
}
}
}
\ No newline at end of file
//#pragma once
//
//#include "storage/IStorage.h"
//
//
//#include <aws/s3/S3Client.h>
//#include <aws/core/Aws.h>
//#include <aws/core/auth/AWSCredentialsProvider.h>
//
//
//using namespace Aws::S3;
//using namespace Aws::S3::Model;
//
//namespace zilliz {
//namespace milvus {
//namespace engine {
//namespace storage {
//
//class S3ClientWrapper : public IStorage {
// public:
//
// S3ClientWrapper() = default;
// ~S3ClientWrapper() = default;
//
// Status Create(const std::string &ip_address,
// const std::string &port,
// const std::string &access_key,
// const std::string &secret_key) override;
// Status Close() override;
//
// Status CreateBucket(std::string& bucket_name) override;
// Status DeleteBucket(std::string& bucket_name) override;
// Status UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override;
// Status DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override;
// Status DeleteFile(std::string &bucket_name, std::string &object_key) override;
//
// private:
// S3Client *client_ = nullptr;
// Aws::SDKOptions options_;
//};
//
//}
//}
//}
//}
\ No newline at end of file
cpp/unittest/server/cache_test.cpp
浏览文件 @
0f46d463
...
...
@@ -11,18 +11,18 @@
using
namespace
zilliz
::
milvus
;
TEST
(
CacheTest
,
CACHE_TEST
)
{
cache
::
CacheMgr
*
cpu_mgr
=
cache
::
CpuCacheMgr
::
GetInstance
();
TEST
(
CacheTest
,
C
PU_C
ACHE_TEST
)
{
cache
::
CacheMgr
*
cpu_mgr
=
cache
::
CpuCacheMgr
::
GetInstance
();
const
int64_t
gbyte
=
1
<<
30
;
int64_t
g_num
=
16
;
int64_t
cap
=
g_num
*
gbyte
;
int64_t
cap
=
g_num
*
gbyte
;
cpu_mgr
->
SetCapacity
(
cap
);
ASSERT_EQ
(
cpu_mgr
->
CacheCapacity
(),
cap
);
const
int
dim
=
256
;
for
(
int
i
=
0
;
i
<
20
;
i
++
)
{
for
(
int
i
=
0
;
i
<
20
;
i
++
)
{
std
::
shared_ptr
<
faiss
::
Index
>
raw_index
(
faiss
::
index_factory
(
dim
,
"IDMap,Flat"
));
engine
::
Index_ptr
index
=
std
::
make_shared
<
engine
::
Index
>
(
raw_index
);
index
->
ntotal
=
1000000
;
//less 1G per index
...
...
@@ -31,6 +31,12 @@ TEST(CacheTest, CACHE_TEST) {
}
ASSERT_LT
(
cpu_mgr
->
ItemCount
(),
g_num
);
auto
obj
=
cpu_mgr
->
GetIndex
(
"index_0"
);
ASSERT_TRUE
(
obj
==
nullptr
);
obj
=
cpu_mgr
->
GetIndex
(
"index_19"
);
ASSERT_TRUE
(
obj
!=
nullptr
);
{
std
::
string
item
=
"index_15"
;
ASSERT_TRUE
(
cpu_mgr
->
ItemExists
(
item
));
...
...
@@ -49,4 +55,27 @@ TEST(CacheTest, CACHE_TEST) {
cpu_mgr
->
InsertItem
(
"index_6g"
,
index
);
ASSERT_EQ
(
cpu_mgr
->
ItemCount
(),
0
);
//data greater than capacity can not be inserted sucessfully
}
cpu_mgr
->
PrintInfo
();
}
TEST
(
CacheTest
,
GPU_CACHE_TEST
)
{
cache
::
CacheMgr
*
gpu_mgr
=
cache
::
GpuCacheMgr
::
GetInstance
();
const
int
dim
=
256
;
for
(
int
i
=
0
;
i
<
20
;
i
++
)
{
std
::
shared_ptr
<
faiss
::
Index
>
raw_index
(
faiss
::
index_factory
(
dim
,
"IDMap,Flat"
));
engine
::
Index_ptr
index
=
std
::
make_shared
<
engine
::
Index
>
(
raw_index
);
index
->
ntotal
=
1000
;
cache
::
DataObjPtr
obj
=
std
::
make_shared
<
cache
::
DataObj
>
(
index
);
gpu_mgr
->
InsertItem
(
"index_"
+
std
::
to_string
(
i
),
obj
);
}
auto
obj
=
gpu_mgr
->
GetItem
(
"index_0"
);
gpu_mgr
->
ClearCache
();
ASSERT_EQ
(
gpu_mgr
->
ItemCount
(),
0
);
}
\ No newline at end of file
cpp/unittest/server/config_test.cpp
0 → 100644
浏览文件 @
0f46d463
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include <gtest/gtest.h>
#include "config/IConfigMgr.h"
#include "server/ServerConfig.h"
using
namespace
zilliz
::
milvus
;
static
const
std
::
string
CONFIG_FILE_PATH
=
"../../../conf/server_config.yaml"
;
TEST
(
ConfigTest
,
CONFIG_TEST
)
{
server
::
IConfigMgr
*
config_mgr
=
server
::
IConfigMgr
::
GetInstance
();
server
::
ServerError
err
=
config_mgr
->
LoadConfigFile
(
CONFIG_FILE_PATH
);
ASSERT_EQ
(
err
,
server
::
SERVER_SUCCESS
);
server
::
ConfigNode
&
root_config
=
config_mgr
->
GetRootNode
();
server
::
ConfigNode
&
server_config
=
root_config
.
GetChild
(
"server_config"
);
server
::
ConfigNode
&
db_config
=
root_config
.
GetChild
(
"db_config"
);
server
::
ConfigNode
&
metric_config
=
root_config
.
GetChild
(
"metric_config"
);
server
::
ConfigNode
&
cache_config
=
root_config
.
GetChild
(
"cache_config"
);
std
::
string
address
=
server_config
.
GetValue
(
"address"
);
ASSERT_TRUE
(
!
address
.
empty
());
int64_t
port
=
server_config
.
GetInt64Value
(
"port"
);
ASSERT_TRUE
(
port
!=
0
);
server_config
.
SetValue
(
"test"
,
"2.5"
);
double
test
=
server_config
.
GetDoubleValue
(
"test"
);
ASSERT_EQ
(
test
,
2.5
);
server
::
ConfigNode
fake
;
server_config
.
AddChild
(
"fake"
,
fake
);
fake
=
server_config
.
GetChild
(
"fake"
);
server
::
ConfigNodeArr
arr
;
server_config
.
GetChildren
(
arr
);
ASSERT_EQ
(
arr
.
size
(),
1UL
);
server_config
.
ClearChildren
();
auto
children
=
server_config
.
GetChildren
();
ASSERT_TRUE
(
children
.
empty
());
root_config
.
PrintAll
();
std
::
string
all
=
root_config
.
DumpString
();
ASSERT_TRUE
(
!
all
.
empty
());
server_config
.
ClearConfig
();
auto
configs
=
server_config
.
GetConfig
();
ASSERT_TRUE
(
configs
.
empty
());
server_config
.
AddSequenceItem
(
"seq"
,
"aaa"
);
server_config
.
AddSequenceItem
(
"seq"
,
"bbb"
);
auto
seq
=
server_config
.
GetSequence
(
"seq"
);
ASSERT_EQ
(
seq
.
size
(),
2UL
);
server_config
.
ClearSequences
();
auto
seqs
=
server_config
.
GetSequences
();
ASSERT_TRUE
(
seqs
.
empty
());
}
TEST
(
ConfigTest
,
SERVER_CONFIG_TEST
)
{
server
::
ServerConfig
&
config
=
server
::
ServerConfig
::
GetInstance
();
server
::
ServerError
err
=
config
.
LoadConfigFile
(
CONFIG_FILE_PATH
);
ASSERT_EQ
(
err
,
server
::
SERVER_SUCCESS
);
server
::
ConfigNode
node1
=
config
.
GetConfig
(
"server_config"
);
server
::
ConfigNode
&
node2
=
config
.
GetConfig
(
"cache_config"
);
node1
.
Combine
(
node2
);
int32_t
cap
=
node1
.
GetInt32Value
(
"cpu_cache_capacity"
);
ASSERT_GT
(
cap
,
0
);
config
.
PrintAll
();
}
\ No newline at end of file
cpp/unittest/storage/S3ClientTest.cpp
浏览文件 @
0f46d463
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include "storage/IStorage.h"
#include "storage/s3/S3ClientWrapper.h"
#include <gtest/gtest.h>
#include <memory.h>
#include <fstream>
using
namespace
zilliz
::
milvus
::
engine
;
TEST
(
s3_client_wrapper
,
CLIENT_WRAPPER_TEST
)
{
std
::
shared_ptr
<
storage
::
IStorage
>
storage_ptr
=
std
::
make_shared
<
storage
::
S3ClientWrapper
>
();
std
::
string
ip_address
=
"127.0.0.1"
;
std
::
string
port
=
"9000"
;
std
::
string
access_key
=
"AKIAIOSFODNN7EXAMPLE"
;
std
::
string
secret_key
=
"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
;
Status
status
=
storage_ptr
->
Create
(
ip_address
,
port
,
access_key
,
secret_key
);
ASSERT_TRUE
(
status
.
ok
());
std
::
string
filename
=
"/tmp/s3_test_file"
;
std
::
string
bucket_name
=
"bucktname"
;
std
::
string
object_name
=
"test_file"
;
status
=
storage_ptr
->
CreateBucket
(
bucket_name
);
std
::
cout
<<
status
.
IsAlreadyExist
()
<<
std
::
endl
;
if
(
status
.
IsAlreadyExist
())
{
status
=
storage_ptr
->
DeleteBucket
(
bucket_name
);
status
=
storage_ptr
->
CreateBucket
(
bucket_name
);
}
ASSERT_TRUE
(
status
.
ok
());
std
::
ofstream
ofile
(
filename
);
std
::
stringstream
ss
;
for
(
int
i
=
0
;
i
<
1024
;
++
i
)
{
ss
<<
i
;
}
ofile
<<
ss
.
str
()
<<
std
::
endl
;
ofile
.
close
();
status
=
storage_ptr
->
UploadFile
(
bucket_name
,
object_name
,
filename
);
ASSERT_TRUE
(
status
.
ok
());
status
=
storage_ptr
->
DownloadFile
(
bucket_name
,
object_name
,
filename
);
std
::
ifstream
infile
(
filename
);
std
::
string
in_buffer
;
infile
>>
in_buffer
;
ASSERT_STREQ
(
in_buffer
.
c_str
(),
ss
.
str
().
c_str
());
status
=
storage_ptr
->
DeleteFile
(
bucket_name
,
object_name
);
ASSERT_TRUE
(
status
.
ok
());
status
=
storage_ptr
->
DeleteBucket
(
bucket_name
);
ASSERT_TRUE
(
status
.
ok
());
status
=
storage_ptr
->
Close
();
ASSERT_TRUE
(
status
.
ok
());
}
////////////////////////////////////////////////////////////////////////////////
//
//
//
Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
//
//
Unauthorized copying of this file, via any medium is strictly prohibited.
//
//
Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
//
//
//
#include "storage/IStorage.h"
//
#include "storage/s3/S3ClientWrapper.h"
//
#include <gtest/gtest.h>
//
#include <memory.h>
//
#include <fstream>
//
//
//
using namespace zilliz::milvus::engine;
//
//
TEST(s3_client_wrapper, CLIENT_WRAPPER_TEST) {
//
//
std::shared_ptr<storage::IStorage> storage_ptr = std::make_shared<storage::S3ClientWrapper>();
//
//
std::string ip_address = "127.0.0.1";
//
std::string port = "9000";
//
std::string access_key = "AKIAIOSFODNN7EXAMPLE";
//
std::string secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY";
//
//
Status status = storage_ptr->Create(ip_address, port, access_key, secret_key);
//
ASSERT_TRUE(status.ok());
//
//
std::string filename = "/tmp/s3_test_file";
//
std::string bucket_name = "bucktname";
//
std::string object_name = "test_file";
//
//
status = storage_ptr->CreateBucket(bucket_name);
//
std::cout << status.IsAlreadyExist() << std::endl;
//
if (status.IsAlreadyExist()) {
//
status = storage_ptr->DeleteBucket(bucket_name);
//
status = storage_ptr->CreateBucket(bucket_name);
//
}
//
//
ASSERT_TRUE(status.ok());
//
//
std::ofstream ofile(filename);
//
std::stringstream ss;
//
for (int i = 0; i < 1024; ++i) {
//
ss << i;
//
}
//
ofile << ss.str() << std::endl;
//
ofile.close();
//
status = storage_ptr->UploadFile(bucket_name, object_name, filename);
//
ASSERT_TRUE(status.ok());
//
//
status = storage_ptr->DownloadFile(bucket_name, object_name, filename);
//
std::ifstream infile(filename);
//
std::string in_buffer;
//
infile >> in_buffer;
//
ASSERT_STREQ(in_buffer.c_str(), ss.str().c_str());
//
//
status = storage_ptr->DeleteFile(bucket_name, object_name);
//
ASSERT_TRUE(status.ok());
//
//
status = storage_ptr->DeleteBucket(bucket_name);
//
ASSERT_TRUE(status.ok());
//
//
status = storage_ptr->Close();
//
ASSERT_TRUE(status.ok());
//
}
//
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录