Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
12c4fd59
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
12c4fd59
编写于
10月 31, 2019
作者:
J
JinHai-CN
浏览文件
操作
浏览文件
下载
差异文件
Merge remote-tracking branch 'main/0.5.1' into 0.5.1
Former-commit-id: b33002967bc42d8290955e2aaa7771badc87b1a8
上级
732d4642
296b2b0d
变更
15
展开全部
显示空白变更内容
内联
并排
Showing
15 changed file
with
519 addition
and
545 deletion
+519
-545
CHANGELOG.md
CHANGELOG.md
+2
-0
ci/jenkins/Jenkinsfile
ci/jenkins/Jenkinsfile
+11
-10
ci/jenkins/step/build.groovy
ci/jenkins/step/build.groovy
+0
-0
ci/jenkins/step/cleanupSingleDev.groovy
ci/jenkins/step/cleanupSingleDev.groovy
+0
-0
ci/jenkins/step/coverage.groovy
ci/jenkins/step/coverage.groovy
+0
-0
ci/jenkins/step/deploySingle2Dev.groovy
ci/jenkins/step/deploySingle2Dev.groovy
+1
-1
ci/jenkins/step/package.groovy
ci/jenkins/step/package.groovy
+0
-0
ci/jenkins/step/publishImages.groovy
ci/jenkins/step/publishImages.groovy
+0
-0
ci/jenkins/step/singleDevNightlyTest.groovy
ci/jenkins/step/singleDevNightlyTest.groovy
+1
-1
ci/jenkins/step/singleDevTest.groovy
ci/jenkins/step/singleDevTest.groovy
+1
-1
core/cmake/ThirdPartyPackages.cmake
core/cmake/ThirdPartyPackages.cmake
+193
-182
core/src/db/engine/ExecutionEngineImpl.cpp
core/src/db/engine/ExecutionEngineImpl.cpp
+1
-1
core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp
...c/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp
+296
-339
core/src/scheduler/optimizer/LargeSQ8HPass.cpp
core/src/scheduler/optimizer/LargeSQ8HPass.cpp
+12
-10
core/src/scheduler/optimizer/LargeSQ8HPass.h
core/src/scheduler/optimizer/LargeSQ8HPass.h
+1
-0
未找到文件。
CHANGELOG.md
浏览文件 @
12c4fd59
...
...
@@ -5,6 +5,7 @@ Please mark all change in change log and use the ticket from JIRA.
# Milvus 0.5.1 (TODO)
## Bug
-
\#
134 - JFrog cache error
## Feature
-
\#
90 - The server start error messages could be improved to enhance user experience
...
...
@@ -21,6 +22,7 @@ Please mark all change in change log and use the ticket from JIRA.
-
\#
118 - Using shared_ptr instead of weak_ptr to avoid performance loss
-
\#
122 - Add unique id for Job
-
\#
130 - Set task state MOVED after resource copy it completed
-
\#
149 - Improve large query optimizer pass
## Task
...
...
ci/jenkins/Jenkinsfile
浏览文件 @
12c4fd59
#
!
/usr/
bin
/
env
groovy
String
cron_timezone
=
"TZ=Asia/Shanghai"
String
cron_string
=
BRANCH_NAME
==
"master"
?
"H 0 * * * "
:
""
cron_string
=
BRANCH_NAME
==
"0.5.1"
?
"H 1 * * * "
:
cron_string
...
...
@@ -16,7 +18,6 @@ pipeline {
parameters
{
choice
choices:
[
'Release'
,
'Debug'
],
description:
''
,
name:
'BUILD_TYPE'
string
defaultValue:
'cf1434e7-5a4b-4d25-82e8-88d667aef9e5'
,
description:
'GIT CREDENTIALS ID'
,
name:
'GIT_CREDENTIALS_ID'
,
trim:
true
string
defaultValue:
'registry.zilliz.com'
,
description:
'DOCKER REGISTRY URL'
,
name:
'DOKCER_REGISTRY_URL'
,
trim:
true
string
defaultValue:
'ba070c98-c8cc-4f7c-b657-897715f359fc'
,
description:
'DOCKER CREDENTIALS ID'
,
name:
'DOCKER_CREDENTIALS_ID'
,
trim:
true
string
defaultValue:
'http://192.168.1.202/artifactory/milvus'
,
description:
'JFROG ARTFACTORY URL'
,
name:
'JFROG_ARTFACTORY_URL'
,
trim:
true
...
...
@@ -56,7 +57,7 @@ pipeline {
steps
{
container
(
'milvus-build-env'
)
{
script
{
load
"${env.WORKSPACE}/ci/jenkins/
jenkinsfile
/build.groovy"
load
"${env.WORKSPACE}/ci/jenkins/
step
/build.groovy"
}
}
}
...
...
@@ -65,7 +66,7 @@ pipeline {
steps
{
container
(
'milvus-build-env'
)
{
script
{
load
"${env.WORKSPACE}/ci/jenkins/
jenkinsfile
/coverage.groovy"
load
"${env.WORKSPACE}/ci/jenkins/
step
/coverage.groovy"
}
}
}
...
...
@@ -74,7 +75,7 @@ pipeline {
steps
{
container
(
'milvus-build-env'
)
{
script
{
load
"${env.WORKSPACE}/ci/jenkins/
jenkinsfile
/package.groovy"
load
"${env.WORKSPACE}/ci/jenkins/
step
/package.groovy"
}
}
}
...
...
@@ -96,7 +97,7 @@ pipeline {
steps
{
container
(
'publish-images'
){
script
{
load
"${env.WORKSPACE}/ci/jenkins/
jenkinsfile
/publishImages.groovy"
load
"${env.WORKSPACE}/ci/jenkins/
step
/publishImages.groovy"
}
}
}
...
...
@@ -118,7 +119,7 @@ pipeline {
steps
{
container
(
'milvus-test-env'
)
{
script
{
load
"${env.WORKSPACE}/ci/jenkins/
jenkinsfile
/deploySingle2Dev.groovy"
load
"${env.WORKSPACE}/ci/jenkins/
step
/deploySingle2Dev.groovy"
}
}
}
...
...
@@ -130,9 +131,9 @@ pipeline {
script
{
boolean
isNightlyTest
=
isTimeTriggeredBuild
()
if
(
isNightlyTest
)
{
load
"${env.WORKSPACE}/ci/jenkins/
jenkinsfile
/singleDevNightlyTest.groovy"
load
"${env.WORKSPACE}/ci/jenkins/
step
/singleDevNightlyTest.groovy"
}
else
{
load
"${env.WORKSPACE}/ci/jenkins/
jenkinsfile
/singleDevTest.groovy"
load
"${env.WORKSPACE}/ci/jenkins/
step
/singleDevTest.groovy"
}
}
}
...
...
@@ -143,7 +144,7 @@ pipeline {
steps
{
container
(
'milvus-test-env'
)
{
script
{
load
"${env.WORKSPACE}/ci/jenkins/
jenkinsfile
/cleanupSingleDev.groovy"
load
"${env.WORKSPACE}/ci/jenkins/
step
/cleanupSingleDev.groovy"
}
}
}
...
...
@@ -153,7 +154,7 @@ pipeline {
unsuccessful
{
container
(
'milvus-test-env'
)
{
script
{
load
"${env.WORKSPACE}/ci/jenkins/
jenkinsfile
/cleanupSingleDev.groovy"
load
"${env.WORKSPACE}/ci/jenkins/
step
/cleanupSingleDev.groovy"
}
}
}
...
...
ci/jenkins/
jenkinsfile
/build.groovy
→
ci/jenkins/
step
/build.groovy
浏览文件 @
12c4fd59
文件已移动
ci/jenkins/
jenkinsfile
/cleanupSingleDev.groovy
→
ci/jenkins/
step
/cleanupSingleDev.groovy
浏览文件 @
12c4fd59
文件已移动
ci/jenkins/
jenkinsfile
/coverage.groovy
→
ci/jenkins/
step
/coverage.groovy
浏览文件 @
12c4fd59
文件已移动
ci/jenkins/
jenkinsfile
/deploySingle2Dev.groovy
→
ci/jenkins/
step
/deploySingle2Dev.groovy
浏览文件 @
12c4fd59
sh
'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
sh
'helm repo update'
dir
(
'milvus-helm'
)
{
checkout
([
$class
:
'GitSCM'
,
branches:
[[
name:
"0.5.0"
]],
doGenerateSubmoduleConfigurations:
false
,
extensions:
[],
submoduleCfg:
[],
userRemoteConfigs:
[[
credentialsId:
"${params.GIT_CREDENTIALS_ID}"
,
url:
"https://github.com/milvus-io/milvus-helm.git"
,
name:
'origin'
,
refspec:
"+refs/heads/0.5.0:refs/remotes/origin/0.5.0"
]]])
checkout
([
$class
:
'GitSCM'
,
branches:
[[
name:
"0.5.0"
]],
userRemoteConfigs:
[[
url:
"https://github.com/milvus-io/milvus-helm.git"
,
name:
'origin'
,
refspec:
"+refs/heads/0.5.0:refs/remotes/origin/0.5.0"
]]])
dir
(
"milvus-gpu"
)
{
sh
"helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/db_backend/sqlite_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
}
...
...
ci/jenkins/
jenkinsfile
/package.groovy
→
ci/jenkins/
step
/package.groovy
浏览文件 @
12c4fd59
文件已移动
ci/jenkins/
jenkinsfile
/publishImages.groovy
→
ci/jenkins/
step
/publishImages.groovy
浏览文件 @
12c4fd59
文件已移动
ci/jenkins/
jenkinsfile
/singleDevNightlyTest.groovy
→
ci/jenkins/
step
/singleDevNightlyTest.groovy
浏览文件 @
12c4fd59
...
...
@@ -8,7 +8,7 @@ timeout(time: 90, unit: 'MINUTES') {
if
(!
fileExists
(
'milvus-helm'
))
{
dir
(
"milvus-helm"
)
{
checkout
([
$class
:
'GitSCM'
,
branches:
[[
name:
"0.5.0"
]],
doGenerateSubmoduleConfigurations:
false
,
extensions:
[],
submoduleCfg:
[],
userRemoteConfigs:
[[
credentialsId:
"${params.GIT_CREDENTIALS_ID}"
,
url:
"https://github.com/milvus-io/milvus-helm.git"
,
name:
'origin'
,
refspec:
"+refs/heads/0.5.0:refs/remotes/origin/0.5.0"
]]])
checkout
([
$class
:
'GitSCM'
,
branches:
[[
name:
"0.5.0"
]],
userRemoteConfigs:
[[
url:
"https://github.com/milvus-io/milvus-helm.git"
,
name:
'origin'
,
refspec:
"+refs/heads/0.5.0:refs/remotes/origin/0.5.0"
]]])
}
}
dir
(
"milvus-helm"
)
{
...
...
ci/jenkins/
jenkinsfile
/singleDevTest.groovy
→
ci/jenkins/
step
/singleDevTest.groovy
浏览文件 @
12c4fd59
...
...
@@ -10,7 +10,7 @@ timeout(time: 60, unit: 'MINUTES') {
// if (!fileExists('milvus-helm')) {
// dir ("milvus-helm") {
// checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]],
doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_CREDENTIALS_ID}",
url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]])
// checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]],
userRemoteConfigs: [[
url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]])
// }
// }
// dir ("milvus-helm") {
...
...
core/cmake/ThirdPartyPackages.cmake
浏览文件 @
12c4fd59
此差异已折叠。
点击以展开。
core/src/db/engine/ExecutionEngineImpl.cpp
浏览文件 @
12c4fd59
...
...
@@ -258,7 +258,7 @@ Status
ExecutionEngineImpl
::
CopyToGpu
(
uint64_t
device_id
,
bool
hybrid
)
{
if
(
hybrid
)
{
const
std
::
string
key
=
location_
+
".quantizer"
;
std
::
vector
<
uint64_t
>
gpus
=
scheduler
::
get_gpu_pool
()
;
std
::
vector
<
uint64_t
>
gpus
{
device_id
}
;
const
int64_t
NOT_FOUND
=
-
1
;
int64_t
device_id
=
NOT_FOUND
;
...
...
core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp
浏览文件 @
12c4fd59
此差异已折叠。
点击以展开。
core/src/scheduler/optimizer/LargeSQ8HPass.cpp
浏览文件 @
12c4fd59
...
...
@@ -55,16 +55,18 @@ LargeSQ8HPass::Run(const TaskPtr& task) {
}
std
::
vector
<
uint64_t
>
gpus
=
scheduler
::
get_gpu_pool
();
std
::
vector
<
int64_t
>
all_free_mem
;
for
(
auto
&
gpu
:
gpus
)
{
auto
cache
=
cache
::
GpuCacheMgr
::
GetInstance
(
gpu
);
auto
free_mem
=
cache
->
CacheCapacity
()
-
cache
->
CacheUsage
();
all_free_mem
.
push_back
(
free_mem
);
}
auto
max_e
=
std
::
max_element
(
all_free_mem
.
begin
(),
all_free_mem
.
end
());
auto
best_index
=
std
::
distance
(
all_free_mem
.
begin
(),
max_e
);
auto
best_device_id
=
gpus
[
best_index
];
// std::vector<int64_t> all_free_mem;
// for (auto& gpu : gpus) {
// auto cache = cache::GpuCacheMgr::GetInstance(gpu);
// auto free_mem = cache->CacheCapacity() - cache->CacheUsage();
// all_free_mem.push_back(free_mem);
// }
//
// auto max_e = std::max_element(all_free_mem.begin(), all_free_mem.end());
// auto best_index = std::distance(all_free_mem.begin(), max_e);
// auto best_device_id = gpus[best_index];
auto
best_device_id
=
count_
%
gpus
.
size
();
count_
++
;
ResourcePtr
res_ptr
=
ResMgrInst
::
GetInstance
()
->
GetResource
(
ResourceType
::
GPU
,
best_device_id
);
if
(
not
res_ptr
)
{
...
...
core/src/scheduler/optimizer/LargeSQ8HPass.h
浏览文件 @
12c4fd59
...
...
@@ -43,6 +43,7 @@ class LargeSQ8HPass : public Pass {
private:
int32_t
threshold_
=
std
::
numeric_limits
<
int32_t
>::
max
();
int64_t
count_
=
0
;
};
using
LargeSQ8HPassPtr
=
std
::
shared_ptr
<
LargeSQ8HPass
>
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录