Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
doujutun3207
flink
提交
dac02696
F
flink
项目概览
doujutun3207
/
flink
与 Fork 源项目一致
从无法访问的项目Fork
通知
24
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
F
flink
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
dac02696
编写于
7月 13, 2019
作者:
G
Gao Yun
提交者:
Stephan Ewen
7月 25, 2019
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[FLINK-12765][coordinator] Add the remaining resources to the slot selection strategy
上级
dd6508f9
变更
9
隐藏空白更改
内联
并排
Showing
9 changed file
with
228 addition
and
49 deletion
+228
-49
flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/LocationPreferenceSlotSelectionStrategy.java
...ter/slotpool/LocationPreferenceSlotSelectionStrategy.java
+12
-13
flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/PreviousAllocationSlotSelectionStrategy.java
...ter/slotpool/PreviousAllocationSlotSelectionStrategy.java
+10
-11
flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SchedulerImpl.java
...pache/flink/runtime/jobmaster/slotpool/SchedulerImpl.java
+8
-3
flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotSelectionStrategy.java
...ink/runtime/jobmaster/slotpool/SlotSelectionStrategy.java
+33
-2
flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotSharingManager.java
.../flink/runtime/jobmaster/slotpool/SlotSharingManager.java
+9
-5
flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/types/LocationPreferenceSlotSelectionStrategyTest.java
...rk/types/LocationPreferenceSlotSelectionStrategyTest.java
+11
-2
flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/types/SlotSelectionStrategyTestBase.java
...clusterframework/types/SlotSelectionStrategyTestBase.java
+7
-8
flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolSlotSharingTest.java
...k/runtime/jobmaster/slotpool/SlotPoolSlotSharingTest.java
+89
-0
flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotSharingManagerTest.java
...nk/runtime/jobmaster/slotpool/SlotSharingManagerTest.java
+49
-5
未找到文件。
flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/LocationPreferenceSlotSelectionStrategy.java
浏览文件 @
dac02696
...
...
@@ -22,7 +22,6 @@ import org.apache.flink.runtime.clusterframework.types.ResourceID;
import
org.apache.flink.runtime.clusterframework.types.ResourceProfile
;
import
org.apache.flink.runtime.clusterframework.types.SlotProfile
;
import
org.apache.flink.runtime.jobmanager.scheduler.Locality
;
import
org.apache.flink.runtime.jobmaster.SlotInfo
;
import
org.apache.flink.runtime.taskmanager.TaskManagerLocation
;
import
javax.annotation.Nonnull
;
...
...
@@ -48,7 +47,7 @@ public enum LocationPreferenceSlotSelectionStrategy implements SlotSelectionStra
@Override
public
Optional
<
SlotInfoAndLocality
>
selectBestSlotForProfile
(
@Nonnull
Collection
<
?
extends
SlotInfo
>
availableSlots
,
@Nonnull
Collection
<
SlotInfoAndResources
>
availableSlots
,
@Nonnull
SlotProfile
slotProfile
)
{
Collection
<
TaskManagerLocation
>
locationPreferences
=
slotProfile
.
getPreferredLocations
();
...
...
@@ -67,12 +66,12 @@ public enum LocationPreferenceSlotSelectionStrategy implements SlotSelectionStra
@Nonnull
private
Optional
<
SlotInfoAndLocality
>
selectWithoutLocationPreference
(
@Nonnull
Collection
<
?
extends
SlotInfo
>
availableSlots
,
@Nonnull
Collection
<
SlotInfoAndResources
>
availableSlots
,
@Nonnull
ResourceProfile
resourceProfile
)
{
for
(
SlotInfo
candidate
:
availableSlots
)
{
if
(
candidate
.
getRe
sourceProfile
().
isMatching
(
resourceProfile
))
{
return
Optional
.
of
(
SlotInfoAndLocality
.
of
(
candidate
,
Locality
.
UNCONSTRAINED
));
for
(
SlotInfo
AndResources
candidate
:
availableSlots
)
{
if
(
candidate
.
getRe
mainingResources
().
isMatching
(
resourceProfile
))
{
return
Optional
.
of
(
SlotInfoAndLocality
.
of
(
candidate
.
getSlotInfo
()
,
Locality
.
UNCONSTRAINED
));
}
}
return
Optional
.
empty
();
...
...
@@ -80,7 +79,7 @@ public enum LocationPreferenceSlotSelectionStrategy implements SlotSelectionStra
@Nonnull
private
Optional
<
SlotInfoAndLocality
>
selectWitLocationPreference
(
@Nonnull
Collection
<
?
extends
SlotInfo
>
availableSlots
,
@Nonnull
Collection
<
SlotInfoAndResources
>
availableSlots
,
@Nonnull
Collection
<
TaskManagerLocation
>
locationPreferences
,
@Nonnull
ResourceProfile
resourceProfile
)
{
...
...
@@ -93,21 +92,21 @@ public enum LocationPreferenceSlotSelectionStrategy implements SlotSelectionStra
preferredFQHostNames
.
merge
(
locationPreference
.
getFQDNHostname
(),
1
,
Integer:
:
sum
);
}
SlotInfo
bestCandidate
=
null
;
SlotInfo
AndResources
bestCandidate
=
null
;
Locality
bestCandidateLocality
=
Locality
.
UNKNOWN
;
int
bestCandidateScore
=
Integer
.
MIN_VALUE
;
for
(
SlotInfo
candidate
:
availableSlots
)
{
for
(
SlotInfo
AndResources
candidate
:
availableSlots
)
{
if
(
candidate
.
getRe
sourceProfile
().
isMatching
(
resourceProfile
))
{
if
(
candidate
.
getRe
mainingResources
().
isMatching
(
resourceProfile
))
{
// this gets candidate is local-weigh
Integer
localWeigh
=
preferredResourceIDs
.
getOrDefault
(
candidate
.
getTaskManagerLocation
().
getResourceID
(),
0
);
candidate
.
get
SlotInfo
().
get
TaskManagerLocation
().
getResourceID
(),
0
);
// this gets candidate is host-local-weigh
Integer
hostLocalWeigh
=
preferredFQHostNames
.
getOrDefault
(
candidate
.
getTaskManagerLocation
().
getFQDNHostname
(),
0
);
candidate
.
get
SlotInfo
().
get
TaskManagerLocation
().
getFQDNHostname
(),
0
);
int
candidateScore
=
LOCALITY_EVALUATION_FUNCTION
.
apply
(
localWeigh
,
hostLocalWeigh
);
if
(
candidateScore
>
bestCandidateScore
)
{
...
...
@@ -122,7 +121,7 @@ public enum LocationPreferenceSlotSelectionStrategy implements SlotSelectionStra
// at the end of the iteration, we return the candidate with best possible locality or null.
return
bestCandidate
!=
null
?
Optional
.
of
(
SlotInfoAndLocality
.
of
(
bestCandidate
,
bestCandidateLocality
))
:
Optional
.
of
(
SlotInfoAndLocality
.
of
(
bestCandidate
.
getSlotInfo
()
,
bestCandidateLocality
))
:
Optional
.
empty
();
}
}
flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/PreviousAllocationSlotSelectionStrategy.java
浏览文件 @
dac02696
...
...
@@ -21,7 +21,6 @@ package org.apache.flink.runtime.jobmaster.slotpool;
import
org.apache.flink.runtime.clusterframework.types.AllocationID
;
import
org.apache.flink.runtime.clusterframework.types.SlotProfile
;
import
org.apache.flink.runtime.jobmanager.scheduler.Locality
;
import
org.apache.flink.runtime.jobmaster.SlotInfo
;
import
javax.annotation.Nonnull
;
...
...
@@ -41,39 +40,39 @@ public enum PreviousAllocationSlotSelectionStrategy implements SlotSelectionStra
@Override
public
Optional
<
SlotInfoAndLocality
>
selectBestSlotForProfile
(
@Nonnull
Collection
<
?
extends
SlotInfo
>
availableSlots
,
@Nonnull
Collection
<
SlotInfoAndResources
>
availableSlots
,
@Nonnull
SlotProfile
slotProfile
)
{
Collection
<
AllocationID
>
priorAllocations
=
slotProfile
.
getPreferredAllocations
();
// First, if there was a prior allocation try to schedule to the same/old slot
if
(!
priorAllocations
.
isEmpty
())
{
for
(
SlotInfo
availableSlot
:
availableSlots
)
{
if
(
priorAllocations
.
contains
(
availableSlot
.
getAllocationId
()))
{
for
(
SlotInfo
AndResources
availableSlot
:
availableSlots
)
{
if
(
priorAllocations
.
contains
(
availableSlot
.
get
SlotInfo
().
get
AllocationId
()))
{
return
Optional
.
of
(
SlotInfoAndLocality
.
of
(
availableSlot
,
Locality
.
LOCAL
));
SlotInfoAndLocality
.
of
(
availableSlot
.
getSlotInfo
()
,
Locality
.
LOCAL
));
}
}
}
// Second, select based on location preference, excluding blacklisted allocations
Set
<
AllocationID
>
blackListedAllocations
=
slotProfile
.
getPreviousExecutionGraphAllocations
();
Collection
<
?
extends
SlotInfo
>
availableAndAllowedSlots
=
computeWithoutBlacklistedSlots
(
availableSlots
,
blackListedAllocations
);
Collection
<
SlotInfoAndResources
>
availableAndAllowedSlots
=
computeWithoutBlacklistedSlots
(
availableSlots
,
blackListedAllocations
);
return
LocationPreferenceSlotSelectionStrategy
.
INSTANCE
.
selectBestSlotForProfile
(
availableAndAllowedSlots
,
slotProfile
);
}
@Nonnull
private
Collection
<
SlotInfo
>
computeWithoutBlacklistedSlots
(
@Nonnull
Collection
<
?
extends
SlotInfo
>
availableSlots
,
private
Collection
<
SlotInfo
AndResources
>
computeWithoutBlacklistedSlots
(
@Nonnull
Collection
<
SlotInfoAndResources
>
availableSlots
,
@Nonnull
Set
<
AllocationID
>
blacklistedAllocations
)
{
if
(
blacklistedAllocations
.
isEmpty
())
{
return
Collections
.
unmodifiableCollection
(
availableSlots
);
}
ArrayList
<
SlotInfo
>
availableAndAllowedSlots
=
new
ArrayList
<>(
availableSlots
.
size
());
for
(
SlotInfo
availableSlot
:
availableSlots
)
{
if
(!
blacklistedAllocations
.
contains
(
availableSlot
.
getAllocationId
()))
{
ArrayList
<
SlotInfo
AndResources
>
availableAndAllowedSlots
=
new
ArrayList
<>(
availableSlots
.
size
());
for
(
SlotInfo
AndResources
availableSlot
:
availableSlots
)
{
if
(!
blacklistedAllocations
.
contains
(
availableSlot
.
get
SlotInfo
().
get
AllocationId
()))
{
availableAndAllowedSlots
.
add
(
availableSlot
);
}
}
...
...
flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SchedulerImpl.java
浏览文件 @
dac02696
...
...
@@ -30,7 +30,6 @@ import org.apache.flink.runtime.jobmanager.scheduler.NoResourceAvailableExceptio
import
org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit
;
import
org.apache.flink.runtime.jobmaster.LogicalSlot
;
import
org.apache.flink.runtime.jobmaster.SlotContext
;
import
org.apache.flink.runtime.jobmaster.SlotInfo
;
import
org.apache.flink.runtime.jobmaster.SlotRequestId
;
import
org.apache.flink.util.AbstractID
;
import
org.apache.flink.util.ExceptionUtils
;
...
...
@@ -51,6 +50,7 @@ import java.util.Objects;
import
java.util.Optional
;
import
java.util.concurrent.CompletableFuture
;
import
java.util.concurrent.CompletionException
;
import
java.util.stream.Collectors
;
/**
* Scheduler that assigns tasks to slots. This class is currently work in progress, comments will be updated as we
...
...
@@ -293,7 +293,11 @@ public class SchedulerImpl implements Scheduler {
@Nonnull
SlotRequestId
slotRequestId
,
@Nonnull
SlotProfile
slotProfile
)
{
Collection
<
SlotInfo
>
slotInfoList
=
slotPool
.
getAvailableSlotsInformation
();
Collection
<
SlotSelectionStrategy
.
SlotInfoAndResources
>
slotInfoList
=
slotPool
.
getAvailableSlotsInformation
()
.
stream
()
.
map
(
SlotSelectionStrategy
.
SlotInfoAndResources
::
new
)
.
collect
(
Collectors
.
toList
());
Optional
<
SlotSelectionStrategy
.
SlotInfoAndLocality
>
selectedAvailableSlot
=
slotSelectionStrategy
.
selectBestSlotForProfile
(
slotInfoList
,
slotProfile
);
...
...
@@ -479,7 +483,8 @@ public class SchedulerImpl implements Scheduler {
boolean
allowQueuedScheduling
,
@Nullable
Time
allocationTimeout
)
throws
NoResourceAvailableException
{
Collection
<
SlotInfo
>
resolvedRootSlotsInfo
=
slotSharingManager
.
listResolvedRootSlotInfo
(
groupId
);
Collection
<
SlotSelectionStrategy
.
SlotInfoAndResources
>
resolvedRootSlotsInfo
=
slotSharingManager
.
listResolvedRootSlotInfo
(
groupId
);
SlotSelectionStrategy
.
SlotInfoAndLocality
bestResolvedRootSlotWithLocality
=
slotSelectionStrategy
.
selectBestSlotForProfile
(
resolvedRootSlotsInfo
,
slotProfile
).
orElse
(
null
);
...
...
flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotSelectionStrategy.java
浏览文件 @
dac02696
...
...
@@ -18,6 +18,7 @@
package
org.apache.flink.runtime.jobmaster.slotpool
;
import
org.apache.flink.runtime.clusterframework.types.ResourceProfile
;
import
org.apache.flink.runtime.clusterframework.types.SlotProfile
;
import
org.apache.flink.runtime.jobmanager.scheduler.Locality
;
import
org.apache.flink.runtime.jobmaster.SlotInfo
;
...
...
@@ -36,14 +37,44 @@ public interface SlotSelectionStrategy {
* Selects the best {@link SlotInfo} w.r.t. a certain selection criterion from the provided list of available slots
* and considering the given {@link SlotProfile} that describes the requirements.
*
* @param availableSlots a list of the available slots to select from.
* @param availableSlots a list of the available slots to
gether with their remaining resources to
select from.
* @param slotProfile a slot profile, describing requirements for the slot selection.
* @return the selected slot info with the corresponding locality hint.
*/
Optional
<
SlotInfoAndLocality
>
selectBestSlotForProfile
(
@Nonnull
Collection
<
?
extends
SlotInfo
>
availableSlots
,
@Nonnull
Collection
<
SlotInfoAndResources
>
availableSlots
,
@Nonnull
SlotProfile
slotProfile
);
/**
* This class is a value type that combines a {@link SlotInfo} with its remaining {@link ResourceProfile}.
*/
final
class
SlotInfoAndResources
{
@Nonnull
private
final
SlotInfo
slotInfo
;
@Nonnull
private
final
ResourceProfile
remainingResources
;
public
SlotInfoAndResources
(
@Nonnull
SlotInfo
slotInfo
)
{
this
(
slotInfo
,
slotInfo
.
getResourceProfile
());
}
public
SlotInfoAndResources
(
@Nonnull
SlotInfo
slotInfo
,
@Nonnull
ResourceProfile
remainingResources
)
{
this
.
slotInfo
=
slotInfo
;
this
.
remainingResources
=
remainingResources
;
}
@Nonnull
public
SlotInfo
getSlotInfo
()
{
return
slotInfo
;
}
@Nonnull
public
ResourceProfile
getRemainingResources
()
{
return
remainingResources
;
}
}
/**
* This class is a value type that combines a {@link SlotInfo} with a {@link Locality} hint.
...
...
flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotSharingManager.java
浏览文件 @
dac02696
...
...
@@ -180,14 +180,18 @@ public class SlotSharingManager {
}
@Nonnull
public
Collection
<
Slot
Info
>
listResolvedRootSlotInfo
(
@Nullable
AbstractID
groupId
)
{
public
Collection
<
Slot
SelectionStrategy
.
SlotInfoAndResources
>
listResolvedRootSlotInfo
(
@Nullable
AbstractID
groupId
)
{
return
resolvedRootSlots
.
values
()
.
stream
()
.
flatMap
((
Map
<
AllocationID
,
MultiTaskSlot
>
map
)
->
map
.
values
().
stream
())
.
filter
((
MultiTaskSlot
multiTaskSlot
)
->
!
multiTaskSlot
.
contains
(
groupId
))
.
map
((
MultiTaskSlot
multiTaskSlot
)
->
(
SlotInfo
)
multiTaskSlot
.
getSlotContextFuture
().
join
())
.
collect
(
Collectors
.
toList
());
.
flatMap
((
Map
<
AllocationID
,
MultiTaskSlot
>
map
)
->
map
.
values
().
stream
())
.
filter
((
MultiTaskSlot
multiTaskSlot
)
->
!
multiTaskSlot
.
contains
(
groupId
))
.
map
((
MultiTaskSlot
multiTaskSlot
)
->
{
SlotInfo
slotInfo
=
multiTaskSlot
.
getSlotContextFuture
().
join
();
return
new
SlotSelectionStrategy
.
SlotInfoAndResources
(
slotInfo
,
slotInfo
.
getResourceProfile
().
subtract
(
multiTaskSlot
.
getReservedResources
()));
}).
collect
(
Collectors
.
toList
());
}
@Nullable
...
...
flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/types/LocationPreferenceSlotSelectionStrategyTest.java
浏览文件 @
dac02696
...
...
@@ -28,6 +28,7 @@ import java.util.Arrays;
import
java.util.Collections
;
import
java.util.HashSet
;
import
java.util.Optional
;
import
java.util.stream.Collectors
;
public
class
LocationPreferenceSlotSelectionStrategyTest
extends
SlotSelectionStrategyTestBase
{
...
...
@@ -61,7 +62,11 @@ public class LocationPreferenceSlotSelectionStrategyTest extends SlotSelectionSt
SlotProfile
slotProfile
=
new
SlotProfile
(
ResourceProfile
.
UNKNOWN
,
Collections
.
emptyList
(),
Collections
.
emptySet
());
Optional
<
SlotSelectionStrategy
.
SlotInfoAndLocality
>
match
=
runMatching
(
slotProfile
);
Assert
.
assertTrue
(
candidates
.
contains
(
match
.
get
().
getSlotInfo
()));
Assert
.
assertTrue
(
candidates
.
stream
()
.
map
(
SlotSelectionStrategy
.
SlotInfoAndResources
::
getSlotInfo
)
.
collect
(
Collectors
.
toList
())
.
contains
(
match
.
get
().
getSlotInfo
()));
}
@Test
...
...
@@ -70,7 +75,11 @@ public class LocationPreferenceSlotSelectionStrategyTest extends SlotSelectionSt
SlotProfile
slotProfile
=
new
SlotProfile
(
resourceProfile
,
Collections
.
singletonList
(
tmlX
),
Collections
.
emptySet
());
Optional
<
SlotSelectionStrategy
.
SlotInfoAndLocality
>
match
=
runMatching
(
slotProfile
);
Assert
.
assertTrue
(
candidates
.
contains
(
match
.
get
().
getSlotInfo
()));
Assert
.
assertTrue
(
candidates
.
stream
()
.
map
(
SlotSelectionStrategy
.
SlotInfoAndResources
::
getSlotInfo
)
.
collect
(
Collectors
.
toList
())
.
contains
(
match
.
get
().
getSlotInfo
()));
}
@Test
...
...
flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/types/SlotSelectionStrategyTestBase.java
浏览文件 @
dac02696
...
...
@@ -21,7 +21,6 @@ package org.apache.flink.runtime.clusterframework.types;
import
org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway
;
import
org.apache.flink.runtime.instance.SimpleSlotContext
;
import
org.apache.flink.runtime.jobmanager.slots.TaskManagerGateway
;
import
org.apache.flink.runtime.jobmaster.SlotContext
;
import
org.apache.flink.runtime.jobmaster.slotpool.SlotSelectionStrategy
;
import
org.apache.flink.runtime.taskmanager.TaskManagerLocation
;
import
org.apache.flink.util.TestLogger
;
...
...
@@ -56,7 +55,7 @@ public abstract class SlotSelectionStrategyTestBase extends TestLogger {
protected
final
SimpleSlotContext
ssc3
=
new
SimpleSlotContext
(
aid3
,
tml3
,
3
,
taskManagerGateway
,
resourceProfile
);
protected
final
SimpleSlotContext
ssc4
=
new
SimpleSlotContext
(
aid4
,
tml4
,
4
,
taskManagerGateway
,
resourceProfile
);
protected
final
Set
<
Slot
Context
>
candidates
=
Collections
.
unmodifiableSet
(
createCandidates
());
protected
final
Set
<
Slot
SelectionStrategy
.
SlotInfoAndResources
>
candidates
=
Collections
.
unmodifiableSet
(
createCandidates
());
protected
final
SlotSelectionStrategy
selectionStrategy
;
...
...
@@ -64,12 +63,12 @@ public abstract class SlotSelectionStrategyTestBase extends TestLogger {
this
.
selectionStrategy
=
slotSelectionStrategy
;
}
private
Set
<
Slot
Context
>
createCandidates
()
{
Set
<
Slot
Context
>
candidates
=
new
HashSet
<>(
4
);
candidates
.
add
(
ssc1
);
candidates
.
add
(
ssc2
);
candidates
.
add
(
ssc3
);
candidates
.
add
(
ssc4
);
private
Set
<
Slot
SelectionStrategy
.
SlotInfoAndResources
>
createCandidates
()
{
Set
<
Slot
SelectionStrategy
.
SlotInfoAndResources
>
candidates
=
new
HashSet
<>(
4
);
candidates
.
add
(
new
SlotSelectionStrategy
.
SlotInfoAndResources
(
ssc1
)
);
candidates
.
add
(
new
SlotSelectionStrategy
.
SlotInfoAndResources
(
ssc2
)
);
candidates
.
add
(
new
SlotSelectionStrategy
.
SlotInfoAndResources
(
ssc3
)
);
candidates
.
add
(
new
SlotSelectionStrategy
.
SlotInfoAndResources
(
ssc4
)
);
return
candidates
;
}
...
...
flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolSlotSharingTest.java
浏览文件 @
dac02696
...
...
@@ -331,6 +331,95 @@ public class SlotPoolSlotSharingTest extends TestLogger {
assertEquals
(
allocationId2
,
logicalSlot3
.
getAllocationId
());
}
/**
* Tests that when matching from the allocated slot, the remaining resources of the slot
* will be used instead of the total resource.
*/
@Test
public
void
testSlotSharingRespectsRemainingResource
()
throws
Exception
{
final
ResourceProfile
allocatedSlotRp
=
new
ResourceProfile
(
3.0
,
300
);
final
ResourceProfile
largeRequestResource
=
new
ResourceProfile
(
2.0
,
200
);
final
ResourceProfile
smallRequestResource
=
new
ResourceProfile
(
1.0
,
100
);
final
BlockingQueue
<
AllocationID
>
allocationIds
=
new
ArrayBlockingQueue
<>(
2
);
final
TestingResourceManagerGateway
testingResourceManagerGateway
=
slotPoolResource
.
getTestingResourceManagerGateway
();
testingResourceManagerGateway
.
setRequestSlotConsumer
(
(
SlotRequest
slotRequest
)
->
allocationIds
.
offer
(
slotRequest
.
getAllocationId
()));
final
TaskManagerLocation
taskManagerLocation
=
new
LocalTaskManagerLocation
();
final
SlotPoolImpl
slotPool
=
slotPoolResource
.
getSlotPool
();
slotPool
.
registerTaskManager
(
taskManagerLocation
.
getResourceID
());
final
SlotSharingGroupId
slotSharingGroupId
=
new
SlotSharingGroupId
();
final
JobVertexID
jobVertexId1
=
new
JobVertexID
();
final
JobVertexID
jobVertexId2
=
new
JobVertexID
();
final
JobVertexID
jobVertexId3
=
new
JobVertexID
();
final
SlotProvider
slotProvider
=
slotPoolResource
.
getSlotProvider
();
CompletableFuture
<
LogicalSlot
>
logicalSlotFuture1
=
slotProvider
.
allocateSlot
(
new
ScheduledUnit
(
jobVertexId1
,
slotSharingGroupId
,
null
),
true
,
SlotProfile
.
noLocality
(
largeRequestResource
),
TestingUtils
.
infiniteTime
());
final
AllocationID
allocationId1
=
allocationIds
.
take
();
// This should fulfill the first request.
boolean
offerFuture
=
slotPool
.
offerSlot
(
taskManagerLocation
,
new
SimpleAckingTaskManagerGateway
(),
new
SlotOffer
(
allocationId1
,
0
,
allocatedSlotRp
));
assertTrue
(
offerFuture
);
assertTrue
(
logicalSlotFuture1
.
isDone
());
assertEquals
(
allocationId1
,
logicalSlotFuture1
.
get
().
getAllocationId
());
// The second request should not share the same slot with the first request since it is large.
CompletableFuture
<
LogicalSlot
>
logicalSlotFuture2
=
slotProvider
.
allocateSlot
(
new
ScheduledUnit
(
jobVertexId2
,
slotSharingGroupId
,
null
),
true
,
SlotProfile
.
noLocality
(
largeRequestResource
),
TestingUtils
.
infiniteTime
());
assertFalse
(
logicalSlotFuture2
.
isDone
());
// The third request should be able to share the same slot with the first request since it is small.
CompletableFuture
<
LogicalSlot
>
logicalSlotFuture3
=
slotProvider
.
allocateSlot
(
new
ScheduledUnit
(
jobVertexId3
,
slotSharingGroupId
,
null
),
true
,
SlotProfile
.
noLocality
(
smallRequestResource
),
TestingUtils
.
infiniteTime
());
assertTrue
(
logicalSlotFuture3
.
isDone
());
assertEquals
(
allocationId1
,
logicalSlotFuture1
.
get
().
getAllocationId
());
// The second request should be finally fulfilled by a new slot.
final
AllocationID
allocationId2
=
allocationIds
.
take
();
// This should fulfill the first two requests.
offerFuture
=
slotPool
.
offerSlot
(
taskManagerLocation
,
new
SimpleAckingTaskManagerGateway
(),
new
SlotOffer
(
allocationId2
,
0
,
allocatedSlotRp
));
assertTrue
(
offerFuture
);
assertTrue
(
logicalSlotFuture2
.
isDone
());
assertEquals
(
allocationId2
,
logicalSlotFuture2
.
get
().
getAllocationId
());
}
@Test
public
void
testRetryOnSharedSlotOverAllocated
()
throws
InterruptedException
,
ExecutionException
{
final
ResourceProfile
rp1
=
new
ResourceProfile
(
1.0
,
100
);
...
...
flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotSharingManagerTest.java
浏览文件 @
dac02696
...
...
@@ -31,7 +31,6 @@ import org.apache.flink.runtime.jobmanager.slots.DummySlotOwner;
import
org.apache.flink.runtime.jobmanager.slots.TaskManagerGateway
;
import
org.apache.flink.runtime.jobmaster.LogicalSlot
;
import
org.apache.flink.runtime.jobmaster.SlotContext
;
import
org.apache.flink.runtime.jobmaster.SlotInfo
;
import
org.apache.flink.runtime.jobmaster.SlotRequestId
;
import
org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation
;
import
org.apache.flink.runtime.taskmanager.TaskManagerLocation
;
...
...
@@ -439,12 +438,12 @@ public class SlotSharingManagerTest extends TestLogger {
AbstractID
groupId
=
new
AbstractID
();
Collection
<
Slot
Info
>
slotInfos
=
slotSharingManager
.
listResolvedRootSlotInfo
(
groupId
);
Collection
<
Slot
SelectionStrategy
.
SlotInfoAndResources
>
slotInfos
=
slotSharingManager
.
listResolvedRootSlotInfo
(
groupId
);
Assert
.
assertEquals
(
1
,
slotInfos
.
size
());
Slot
Info
slotInfo
=
slotInfos
.
iterator
().
next
();
Slot
SelectionStrategy
.
SlotInfoAndResources
slotInfoAndRemainingResource
=
slotInfos
.
iterator
().
next
();
SlotSharingManager
.
MultiTaskSlot
resolvedMultiTaskSlot
=
slotSharingManager
.
getResolvedRootSlot
(
slotInfo
);
slotSharingManager
.
getResolvedRootSlot
(
slotInfo
AndRemainingResource
.
getSlotInfo
()
);
SlotSelectionStrategy
.
SlotInfoAndLocality
slotInfoAndLocality
=
LocationPreferenceSlotSelectionStrategy
.
INSTANCE
.
selectBestSlotForProfile
(
slotInfos
,
SlotProfile
.
noRequirements
()).
get
();
...
...
@@ -501,7 +500,7 @@ public class SlotSharingManagerTest extends TestLogger {
SlotProfile
slotProfile
=
SlotProfile
.
preferredLocality
(
ResourceProfile
.
UNKNOWN
,
Collections
.
singleton
(
taskManagerLocation
));
Collection
<
Slot
Info
>
slotInfos
=
slotSharingManager
.
listResolvedRootSlotInfo
(
groupId
);
Collection
<
Slot
SelectionStrategy
.
SlotInfoAndResources
>
slotInfos
=
slotSharingManager
.
listResolvedRootSlotInfo
(
groupId
);
SlotSelectionStrategy
.
SlotInfoAndLocality
slotInfoAndLocality
=
LocationPreferenceSlotSelectionStrategy
.
INSTANCE
.
selectBestSlotForProfile
(
slotInfos
,
slotProfile
).
get
();
SlotSharingManager
.
MultiTaskSlot
resolvedRootSlot
=
slotSharingManager
.
getResolvedRootSlot
(
slotInfoAndLocality
.
getSlotInfo
());
...
...
@@ -619,6 +618,51 @@ public class SlotSharingManagerTest extends TestLogger {
assertEquals
(
ResourceProfile
.
ZERO
,
unresolvedRootSlot
.
getReservedResources
());
}
@Test
public
void
testGetResolvedSlotWithResourceConfigured
()
{
ResourceProfile
rp1
=
new
ResourceProfile
(
1.0
,
100
);
ResourceProfile
rp2
=
new
ResourceProfile
(
2.0
,
200
);
ResourceProfile
allocatedSlotRp
=
new
ResourceProfile
(
5.0
,
500
);
final
TestingAllocatedSlotActions
allocatedSlotActions
=
new
TestingAllocatedSlotActions
();
SlotSharingManager
slotSharingManager
=
new
SlotSharingManager
(
SLOT_SHARING_GROUP_ID
,
allocatedSlotActions
,
SLOT_OWNER
);
SlotSharingManager
.
MultiTaskSlot
rootSlot
=
slotSharingManager
.
createRootSlot
(
new
SlotRequestId
(),
CompletableFuture
.
completedFuture
(
new
SimpleSlotContext
(
new
AllocationID
(),
new
LocalTaskManagerLocation
(),
0
,
new
SimpleAckingTaskManagerGateway
(),
allocatedSlotRp
)),
new
SlotRequestId
());
rootSlot
.
allocateSingleTaskSlot
(
new
SlotRequestId
(),
rp1
,
new
SlotSharingGroupId
(),
Locality
.
LOCAL
);
Collection
<
SlotSelectionStrategy
.
SlotInfoAndResources
>
resolvedRoots
=
slotSharingManager
.
listResolvedRootSlotInfo
(
new
AbstractID
());
assertEquals
(
1
,
resolvedRoots
.
size
());
assertEquals
(
allocatedSlotRp
.
subtract
(
rp1
),
resolvedRoots
.
iterator
().
next
().
getRemainingResources
());
rootSlot
.
allocateSingleTaskSlot
(
new
SlotRequestId
(),
rp2
,
new
SlotSharingGroupId
(),
Locality
.
LOCAL
);
resolvedRoots
=
slotSharingManager
.
listResolvedRootSlotInfo
(
new
AbstractID
());
assertEquals
(
1
,
resolvedRoots
.
size
());
assertEquals
(
allocatedSlotRp
.
subtract
(
rp1
).
subtract
(
rp2
),
resolvedRoots
.
iterator
().
next
().
getRemainingResources
());
}
@Test
public
void
testHashEnoughResourceOfMultiTaskSlot
()
{
ResourceProfile
rp1
=
new
ResourceProfile
(
1.0
,
100
);
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录