Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
6e6c2153
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
6e6c2153
编写于
10月 31, 2019
作者:
Y
yudong.cai
浏览文件
操作
浏览文件
下载
差异文件
Merge remote-tracking branch 'main/0.5.1' into 0.5.1
Former-commit-id: 2ab08f424150d1855333e620bbfd57ea10497e01
上级
9d95e472
7b76696a
变更
16
隐藏空白更改
内联
并排
Showing
16 changed file
with
241 addition
and
16 deletion
+241
-16
CHANGELOG.md
CHANGELOG.md
+2
-0
ci/jenkins/Jenkinsfile
ci/jenkins/Jenkinsfile
+11
-10
ci/jenkins/step/build.groovy
ci/jenkins/step/build.groovy
+0
-0
ci/jenkins/step/cleanupSingleDev.groovy
ci/jenkins/step/cleanupSingleDev.groovy
+0
-0
ci/jenkins/step/coverage.groovy
ci/jenkins/step/coverage.groovy
+0
-0
ci/jenkins/step/deploySingle2Dev.groovy
ci/jenkins/step/deploySingle2Dev.groovy
+1
-1
ci/jenkins/step/package.groovy
ci/jenkins/step/package.groovy
+0
-0
ci/jenkins/step/publishImages.groovy
ci/jenkins/step/publishImages.groovy
+0
-0
ci/jenkins/step/singleDevNightlyTest.groovy
ci/jenkins/step/singleDevNightlyTest.groovy
+1
-1
ci/jenkins/step/singleDevTest.groovy
ci/jenkins/step/singleDevTest.groovy
+1
-1
core/src/scheduler/SchedInst.h
core/src/scheduler/SchedInst.h
+16
-0
core/src/scheduler/optimizer/OnlyCPUPass.cpp
core/src/scheduler/optimizer/OnlyCPUPass.cpp
+48
-0
core/src/scheduler/optimizer/OnlyCPUPass.h
core/src/scheduler/optimizer/OnlyCPUPass.h
+47
-0
core/src/scheduler/optimizer/OnlyGPUPass.cpp
core/src/scheduler/optimizer/OnlyGPUPass.cpp
+54
-0
core/src/scheduler/optimizer/OnlyGPUPass.h
core/src/scheduler/optimizer/OnlyGPUPass.h
+51
-0
core/src/server/Config.cpp
core/src/server/Config.cpp
+9
-3
未找到文件。
CHANGELOG.md
浏览文件 @
6e6c2153
...
...
@@ -12,6 +12,7 @@ Please mark all change in change log and use the ticket from JIRA.
-
\#
104 - test_scheduler core dump
-
\#
115 - Using new structure for tasktable
-
\#
139 - New config opion use_gpu_threshold
-
\#
146 - Add only GPU and only CPU version for IVF_SQ8 and IVF_FLAT
## Improvement
-
\#
64 - Improvement dump function in scheduler
...
...
@@ -23,6 +24,7 @@ Please mark all change in change log and use the ticket from JIRA.
-
\#
122 - Add unique id for Job
-
\#
130 - Set task state MOVED after resource copy it completed
-
\#
149 - Improve large query optimizer pass
-
\#
156 - Not return error when search_resources and index_build_device set cpu
## Task
...
...
ci/jenkins/Jenkinsfile
浏览文件 @
6e6c2153
#
!
/usr/
bin
/
env
groovy
String
cron_timezone
=
"TZ=Asia/Shanghai"
String
cron_string
=
BRANCH_NAME
==
"master"
?
"H 0 * * * "
:
""
cron_string
=
BRANCH_NAME
==
"0.5.1"
?
"H 1 * * * "
:
cron_string
...
...
@@ -16,7 +18,6 @@ pipeline {
parameters
{
choice
choices:
[
'Release'
,
'Debug'
],
description:
''
,
name:
'BUILD_TYPE'
string
defaultValue:
'cf1434e7-5a4b-4d25-82e8-88d667aef9e5'
,
description:
'GIT CREDENTIALS ID'
,
name:
'GIT_CREDENTIALS_ID'
,
trim:
true
string
defaultValue:
'registry.zilliz.com'
,
description:
'DOCKER REGISTRY URL'
,
name:
'DOKCER_REGISTRY_URL'
,
trim:
true
string
defaultValue:
'ba070c98-c8cc-4f7c-b657-897715f359fc'
,
description:
'DOCKER CREDENTIALS ID'
,
name:
'DOCKER_CREDENTIALS_ID'
,
trim:
true
string
defaultValue:
'http://192.168.1.202/artifactory/milvus'
,
description:
'JFROG ARTFACTORY URL'
,
name:
'JFROG_ARTFACTORY_URL'
,
trim:
true
...
...
@@ -56,7 +57,7 @@ pipeline {
steps
{
container
(
'milvus-build-env'
)
{
script
{
load
"${env.WORKSPACE}/ci/jenkins/
jenkinsfile
/build.groovy"
load
"${env.WORKSPACE}/ci/jenkins/
step
/build.groovy"
}
}
}
...
...
@@ -65,7 +66,7 @@ pipeline {
steps
{
container
(
'milvus-build-env'
)
{
script
{
load
"${env.WORKSPACE}/ci/jenkins/
jenkinsfile
/coverage.groovy"
load
"${env.WORKSPACE}/ci/jenkins/
step
/coverage.groovy"
}
}
}
...
...
@@ -74,7 +75,7 @@ pipeline {
steps
{
container
(
'milvus-build-env'
)
{
script
{
load
"${env.WORKSPACE}/ci/jenkins/
jenkinsfile
/package.groovy"
load
"${env.WORKSPACE}/ci/jenkins/
step
/package.groovy"
}
}
}
...
...
@@ -96,7 +97,7 @@ pipeline {
steps
{
container
(
'publish-images'
){
script
{
load
"${env.WORKSPACE}/ci/jenkins/
jenkinsfile
/publishImages.groovy"
load
"${env.WORKSPACE}/ci/jenkins/
step
/publishImages.groovy"
}
}
}
...
...
@@ -118,7 +119,7 @@ pipeline {
steps
{
container
(
'milvus-test-env'
)
{
script
{
load
"${env.WORKSPACE}/ci/jenkins/
jenkinsfile
/deploySingle2Dev.groovy"
load
"${env.WORKSPACE}/ci/jenkins/
step
/deploySingle2Dev.groovy"
}
}
}
...
...
@@ -130,9 +131,9 @@ pipeline {
script
{
boolean
isNightlyTest
=
isTimeTriggeredBuild
()
if
(
isNightlyTest
)
{
load
"${env.WORKSPACE}/ci/jenkins/
jenkinsfile
/singleDevNightlyTest.groovy"
load
"${env.WORKSPACE}/ci/jenkins/
step
/singleDevNightlyTest.groovy"
}
else
{
load
"${env.WORKSPACE}/ci/jenkins/
jenkinsfile
/singleDevTest.groovy"
load
"${env.WORKSPACE}/ci/jenkins/
step
/singleDevTest.groovy"
}
}
}
...
...
@@ -143,7 +144,7 @@ pipeline {
steps
{
container
(
'milvus-test-env'
)
{
script
{
load
"${env.WORKSPACE}/ci/jenkins/
jenkinsfile
/cleanupSingleDev.groovy"
load
"${env.WORKSPACE}/ci/jenkins/
step
/cleanupSingleDev.groovy"
}
}
}
...
...
@@ -153,7 +154,7 @@ pipeline {
unsuccessful
{
container
(
'milvus-test-env'
)
{
script
{
load
"${env.WORKSPACE}/ci/jenkins/
jenkinsfile
/cleanupSingleDev.groovy"
load
"${env.WORKSPACE}/ci/jenkins/
step
/cleanupSingleDev.groovy"
}
}
}
...
...
ci/jenkins/
jenkinsfile
/build.groovy
→
ci/jenkins/
step
/build.groovy
浏览文件 @
6e6c2153
文件已移动
ci/jenkins/
jenkinsfile
/cleanupSingleDev.groovy
→
ci/jenkins/
step
/cleanupSingleDev.groovy
浏览文件 @
6e6c2153
文件已移动
ci/jenkins/
jenkinsfile
/coverage.groovy
→
ci/jenkins/
step
/coverage.groovy
浏览文件 @
6e6c2153
文件已移动
ci/jenkins/
jenkinsfile
/deploySingle2Dev.groovy
→
ci/jenkins/
step
/deploySingle2Dev.groovy
浏览文件 @
6e6c2153
sh
'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
sh
'helm repo update'
dir
(
'milvus-helm'
)
{
checkout
([
$class
:
'GitSCM'
,
branches:
[[
name:
"0.5.0"
]],
doGenerateSubmoduleConfigurations:
false
,
extensions:
[],
submoduleCfg:
[],
userRemoteConfigs:
[[
credentialsId:
"${params.GIT_CREDENTIALS_ID}"
,
url:
"https://github.com/milvus-io/milvus-helm.git"
,
name:
'origin'
,
refspec:
"+refs/heads/0.5.0:refs/remotes/origin/0.5.0"
]]])
checkout
([
$class
:
'GitSCM'
,
branches:
[[
name:
"0.5.0"
]],
userRemoteConfigs:
[[
url:
"https://github.com/milvus-io/milvus-helm.git"
,
name:
'origin'
,
refspec:
"+refs/heads/0.5.0:refs/remotes/origin/0.5.0"
]]])
dir
(
"milvus-gpu"
)
{
sh
"helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/db_backend/sqlite_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
}
...
...
ci/jenkins/
jenkinsfile
/package.groovy
→
ci/jenkins/
step
/package.groovy
浏览文件 @
6e6c2153
文件已移动
ci/jenkins/
jenkinsfile
/publishImages.groovy
→
ci/jenkins/
step
/publishImages.groovy
浏览文件 @
6e6c2153
文件已移动
ci/jenkins/
jenkinsfile
/singleDevNightlyTest.groovy
→
ci/jenkins/
step
/singleDevNightlyTest.groovy
浏览文件 @
6e6c2153
...
...
@@ -8,7 +8,7 @@ timeout(time: 90, unit: 'MINUTES') {
if
(!
fileExists
(
'milvus-helm'
))
{
dir
(
"milvus-helm"
)
{
checkout
([
$class
:
'GitSCM'
,
branches:
[[
name:
"0.5.0"
]],
doGenerateSubmoduleConfigurations:
false
,
extensions:
[],
submoduleCfg:
[],
userRemoteConfigs:
[[
credentialsId:
"${params.GIT_CREDENTIALS_ID}"
,
url:
"https://github.com/milvus-io/milvus-helm.git"
,
name:
'origin'
,
refspec:
"+refs/heads/0.5.0:refs/remotes/origin/0.5.0"
]]])
checkout
([
$class
:
'GitSCM'
,
branches:
[[
name:
"0.5.0"
]],
userRemoteConfigs:
[[
url:
"https://github.com/milvus-io/milvus-helm.git"
,
name:
'origin'
,
refspec:
"+refs/heads/0.5.0:refs/remotes/origin/0.5.0"
]]])
}
}
dir
(
"milvus-helm"
)
{
...
...
ci/jenkins/
jenkinsfile
/singleDevTest.groovy
→
ci/jenkins/
step
/singleDevTest.groovy
浏览文件 @
6e6c2153
...
...
@@ -10,7 +10,7 @@ timeout(time: 60, unit: 'MINUTES') {
// if (!fileExists('milvus-helm')) {
// dir ("milvus-helm") {
// checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]],
doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_CREDENTIALS_ID}",
url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]])
// checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]],
userRemoteConfigs: [[
url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]])
// }
// }
// dir ("milvus-helm") {
...
...
core/src/scheduler/SchedInst.h
浏览文件 @
6e6c2153
...
...
@@ -23,10 +23,14 @@
#include "Scheduler.h"
#include "optimizer/HybridPass.h"
#include "optimizer/LargeSQ8HPass.h"
#include "optimizer/OnlyCPUPass.h"
#include "optimizer/OnlyGPUPass.h"
#include "optimizer/Optimizer.h"
#include "server/Config.h"
#include <memory>
#include <mutex>
#include <string>
#include <vector>
namespace
milvus
{
...
...
@@ -93,9 +97,21 @@ class OptimizerInst {
if
(
instance
==
nullptr
)
{
std
::
lock_guard
<
std
::
mutex
>
lock
(
mutex_
);
if
(
instance
==
nullptr
)
{
server
::
Config
&
config
=
server
::
Config
::
GetInstance
();
std
::
vector
<
std
::
string
>
search_resources
;
bool
has_cpu
=
false
;
config
.
GetResourceConfigSearchResources
(
search_resources
);
for
(
auto
&
resource
:
search_resources
)
{
if
(
resource
==
"cpu"
)
{
has_cpu
=
true
;
}
}
std
::
vector
<
PassPtr
>
pass_list
;
pass_list
.
push_back
(
std
::
make_shared
<
LargeSQ8HPass
>
());
pass_list
.
push_back
(
std
::
make_shared
<
HybridPass
>
());
pass_list
.
push_back
(
std
::
make_shared
<
OnlyCPUPass
>
());
pass_list
.
push_back
(
std
::
make_shared
<
OnlyGPUPass
>
(
has_cpu
));
instance
=
std
::
make_shared
<
Optimizer
>
(
pass_list
);
}
}
...
...
core/src/scheduler/optimizer/OnlyCPUPass.cpp
0 → 100644
浏览文件 @
6e6c2153
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "scheduler/optimizer/OnlyCPUPass.h"
#include "scheduler/SchedInst.h"
#include "scheduler/Utils.h"
#include "scheduler/task/SearchTask.h"
#include "scheduler/tasklabel/SpecResLabel.h"
namespace
milvus
{
namespace
scheduler
{
bool
OnlyCPUPass
::
Run
(
const
TaskPtr
&
task
)
{
if
(
task
->
Type
()
!=
TaskType
::
SearchTask
)
return
false
;
auto
search_task
=
std
::
static_pointer_cast
<
XSearchTask
>
(
task
);
if
(
search_task
->
file_
->
engine_type_
!=
(
int
)
engine
::
EngineType
::
FAISS_IVFSQ8
&&
search_task
->
file_
->
engine_type_
!=
(
int
)
engine
::
EngineType
::
FAISS_IVFFLAT
)
{
return
false
;
}
auto
gpu_id
=
get_gpu_pool
();
if
(
not
gpu_id
.
empty
())
return
false
;
ResourcePtr
res_ptr
=
ResMgrInst
::
GetInstance
()
->
GetResource
(
"cpu"
);
auto
label
=
std
::
make_shared
<
SpecResLabel
>
(
std
::
weak_ptr
<
Resource
>
(
res_ptr
));
task
->
label
()
=
label
;
return
true
;
}
}
// namespace scheduler
}
// namespace milvus
core/src/scheduler/optimizer/OnlyCPUPass.h
0 → 100644
浏览文件 @
6e6c2153
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <condition_variable>
#include <deque>
#include <list>
#include <memory>
#include <mutex>
#include <queue>
#include <string>
#include <thread>
#include <unordered_map>
#include <vector>
#include "Pass.h"
namespace
milvus
{
namespace
scheduler
{
class
OnlyCPUPass
:
public
Pass
{
public:
OnlyCPUPass
()
=
default
;
public:
bool
Run
(
const
TaskPtr
&
task
)
override
;
};
using
OnlyCPUPassPtr
=
std
::
shared_ptr
<
OnlyCPUPass
>
;
}
// namespace scheduler
}
// namespace milvus
core/src/scheduler/optimizer/OnlyGPUPass.cpp
0 → 100644
浏览文件 @
6e6c2153
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "scheduler/optimizer/OnlyGPUPass.h"
#include "scheduler/SchedInst.h"
#include "scheduler/Utils.h"
#include "scheduler/task/SearchTask.h"
#include "scheduler/tasklabel/SpecResLabel.h"
namespace
milvus
{
namespace
scheduler
{
OnlyGPUPass
::
OnlyGPUPass
(
bool
has_cpu
)
:
has_cpu_
(
has_cpu
)
{
}
bool
OnlyGPUPass
::
Run
(
const
TaskPtr
&
task
)
{
if
(
task
->
Type
()
!=
TaskType
::
SearchTask
||
has_cpu_
)
return
false
;
auto
search_task
=
std
::
static_pointer_cast
<
XSearchTask
>
(
task
);
if
(
search_task
->
file_
->
engine_type_
!=
(
int
)
engine
::
EngineType
::
FAISS_IVFSQ8
&&
search_task
->
file_
->
engine_type_
!=
(
int
)
engine
::
EngineType
::
FAISS_IVFFLAT
)
{
return
false
;
}
auto
gpu_id
=
get_gpu_pool
();
if
(
gpu_id
.
empty
())
return
false
;
ResourcePtr
res_ptr
=
ResMgrInst
::
GetInstance
()
->
GetResource
(
ResourceType
::
GPU
,
gpu_id
[
specified_gpu_id_
]);
auto
label
=
std
::
make_shared
<
SpecResLabel
>
(
std
::
weak_ptr
<
Resource
>
(
res_ptr
));
task
->
label
()
=
label
;
specified_gpu_id_
=
specified_gpu_id_
++
%
gpu_id
.
size
();
return
true
;
}
}
// namespace scheduler
}
// namespace milvus
core/src/scheduler/optimizer/OnlyGPUPass.h
0 → 100644
浏览文件 @
6e6c2153
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <condition_variable>
#include <deque>
#include <list>
#include <memory>
#include <mutex>
#include <queue>
#include <string>
#include <thread>
#include <unordered_map>
#include <vector>
#include "Pass.h"
namespace
milvus
{
namespace
scheduler
{
class
OnlyGPUPass
:
public
Pass
{
public:
explicit
OnlyGPUPass
(
bool
has_cpu
);
public:
bool
Run
(
const
TaskPtr
&
task
)
override
;
private:
uint64_t
specified_gpu_id_
=
0
;
bool
has_cpu_
=
false
;
};
using
OnlyGPUPassPtr
=
std
::
shared_ptr
<
OnlyGPUPass
>
;
}
// namespace scheduler
}
// namespace milvus
core/src/server/Config.cpp
浏览文件 @
6e6c2153
...
...
@@ -714,9 +714,12 @@ Config::CheckResourceConfigSearchResources(const std::vector<std::string>& value
return
Status
(
SERVER_INVALID_ARGUMENT
,
msg
);
}
for
(
auto
&
gpu_device
:
value
)
{
if
(
!
CheckGpuDevice
(
gpu_device
).
ok
())
{
std
::
string
msg
=
"Invalid search resource: "
+
gpu_device
+
for
(
auto
&
device
:
value
)
{
if
(
device
==
"cpu"
)
{
continue
;
}
if
(
!
CheckGpuDevice
(
device
).
ok
())
{
std
::
string
msg
=
"Invalid search resource: "
+
device
+
". Possible reason: resource_config.search_resources does not match your hardware."
;
return
Status
(
SERVER_INVALID_ARGUMENT
,
msg
);
}
...
...
@@ -726,6 +729,9 @@ Config::CheckResourceConfigSearchResources(const std::vector<std::string>& value
Status
Config
::
CheckResourceConfigIndexBuildDevice
(
const
std
::
string
&
value
)
{
if
(
value
==
"cpu"
)
{
return
Status
::
OK
();
}
if
(
!
CheckGpuDevice
(
value
).
ok
())
{
std
::
string
msg
=
"Invalid index build device: "
+
value
+
". Possible reason: resource_config.index_build_device does not match your hardware."
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录