Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
Forever310
druid
提交
23edba9e
D
druid
项目概览
Forever310
/
druid
与 Fork 源项目一致
从无法访问的项目Fork
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
D
druid
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
提交
23edba9e
编写于
12月 19, 2013
作者:
F
fjy
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #338 from metamx/indexing-service-bugs
Indexing service bugs
上级
fba6caf7
0ff7f0e8
变更
19
隐藏空白更改
内联
并排
Showing
19 changed file
with
160 addition
and
75 deletion
+160
-75
indexing-service/src/main/java/io/druid/indexing/common/TaskToolbox.java
...e/src/main/java/io/druid/indexing/common/TaskToolbox.java
+29
-1
indexing-service/src/main/java/io/druid/indexing/common/actions/LocalTaskActionClient.java
.../druid/indexing/common/actions/LocalTaskActionClient.java
+5
-4
indexing-service/src/main/java/io/druid/indexing/common/actions/SegmentInsertAction.java
...io/druid/indexing/common/actions/SegmentInsertAction.java
+1
-4
indexing-service/src/main/java/io/druid/indexing/common/actions/SegmentMetadataUpdateAction.java
.../indexing/common/actions/SegmentMetadataUpdateAction.java
+1
-5
indexing-service/src/main/java/io/druid/indexing/common/actions/SegmentNukeAction.java
...a/io/druid/indexing/common/actions/SegmentNukeAction.java
+1
-5
indexing-service/src/main/java/io/druid/indexing/common/actions/TaskActionToolbox.java
...a/io/druid/indexing/common/actions/TaskActionToolbox.java
+34
-0
indexing-service/src/main/java/io/druid/indexing/common/task/AbstractFixedIntervalTask.java
...druid/indexing/common/task/AbstractFixedIntervalTask.java
+3
-1
indexing-service/src/main/java/io/druid/indexing/common/task/DeleteTask.java
...c/main/java/io/druid/indexing/common/task/DeleteTask.java
+2
-1
indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java
...n/java/io/druid/indexing/common/task/HadoopIndexTask.java
+10
-7
indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java
...rc/main/java/io/druid/indexing/common/task/IndexTask.java
+1
-1
indexing-service/src/main/java/io/druid/indexing/common/task/KillTask.java
...src/main/java/io/druid/indexing/common/task/KillTask.java
+1
-3
indexing-service/src/main/java/io/druid/indexing/common/task/MergeTaskBase.java
...ain/java/io/druid/indexing/common/task/MergeTaskBase.java
+3
-2
indexing-service/src/main/java/io/druid/indexing/common/task/MoveTask.java
...src/main/java/io/druid/indexing/common/task/MoveTask.java
+2
-8
indexing-service/src/main/java/io/druid/indexing/common/task/RealtimeIndexTask.java
...java/io/druid/indexing/common/task/RealtimeIndexTask.java
+1
-1
indexing-service/src/main/java/io/druid/indexing/common/task/VersionConverterTask.java
...a/io/druid/indexing/common/task/VersionConverterTask.java
+2
-1
indexing-service/src/main/java/io/druid/indexing/overlord/DbTaskStorage.java
...c/main/java/io/druid/indexing/overlord/DbTaskStorage.java
+12
-1
indexing-service/src/main/java/io/druid/indexing/overlord/IndexerDBCoordinator.java
...java/io/druid/indexing/overlord/IndexerDBCoordinator.java
+44
-29
indexing-service/src/main/java/io/druid/indexing/overlord/TaskLockbox.java
...src/main/java/io/druid/indexing/overlord/TaskLockbox.java
+7
-0
pom.xml
pom.xml
+1
-1
未找到文件。
indexing-service/src/main/java/io/druid/indexing/common/TaskToolbox.java
浏览文件 @
23edba9e
...
...
@@ -20,10 +20,15 @@
package
io.druid.indexing.common
;
import
com.fasterxml.jackson.databind.ObjectMapper
;
import
com.google.common.base.Function
;
import
com.google.common.collect.ImmutableSet
;
import
com.google.common.collect.Maps
;
import
com.google.common.collect.Multimap
;
import
com.google.common.collect.Multimaps
;
import
com.metamx.emitter.service.ServiceEmitter
;
import
com.metamx.metrics.MonitorScheduler
;
import
io.druid.client.ServerView
;
import
io.druid.indexing.common.actions.SegmentInsertAction
;
import
io.druid.indexing.common.actions.TaskActionClient
;
import
io.druid.indexing.common.actions.TaskActionClientFactory
;
import
io.druid.indexing.common.config.TaskConfig
;
...
...
@@ -37,10 +42,14 @@ import io.druid.segment.loading.SegmentLoader;
import
io.druid.segment.loading.SegmentLoadingException
;
import
io.druid.server.coordination.DataSegmentAnnouncer
;
import
io.druid.timeline.DataSegment
;
import
org.joda.time.Interval
;
import
java.io.File
;
import
java.io.IOException
;
import
java.util.Collection
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Set
;
import
java.util.concurrent.ExecutorService
;
/**
...
...
@@ -167,7 +176,7 @@ public class TaskToolbox
return
objectMapper
;
}
public
Map
<
DataSegment
,
File
>
get
Segments
(
List
<
DataSegment
>
segments
)
public
Map
<
DataSegment
,
File
>
fetch
Segments
(
List
<
DataSegment
>
segments
)
throws
SegmentLoadingException
{
Map
<
DataSegment
,
File
>
retVal
=
Maps
.
newLinkedHashMap
();
...
...
@@ -178,6 +187,25 @@ public class TaskToolbox
return
retVal
;
}
public
void
pushSegments
(
Iterable
<
DataSegment
>
segments
)
throws
IOException
{
// Request segment pushes for each set
final
Multimap
<
Interval
,
DataSegment
>
segmentMultimap
=
Multimaps
.
index
(
segments
,
new
Function
<
DataSegment
,
Interval
>()
{
@Override
public
Interval
apply
(
DataSegment
segment
)
{
return
segment
.
getInterval
();
}
}
);
for
(
final
Collection
<
DataSegment
>
segmentCollection
:
segmentMultimap
.
asMap
().
values
())
{
getTaskActionClient
().
submit
(
new
SegmentInsertAction
(
ImmutableSet
.
copyOf
(
segmentCollection
)));
}
}
public
File
getTaskWorkDir
()
{
return
taskWorkDir
;
...
...
indexing-service/src/main/java/io/druid/indexing/common/actions/LocalTaskActionClient.java
浏览文件 @
23edba9e
...
...
@@ -19,6 +19,7 @@
package
io.druid.indexing.common.actions
;
import
com.metamx.common.ISE
;
import
com.metamx.emitter.EmittingLogger
;
import
io.druid.indexing.common.task.Task
;
import
io.druid.indexing.overlord.TaskStorage
;
...
...
@@ -45,21 +46,21 @@ public class LocalTaskActionClient implements TaskActionClient
{
log
.
info
(
"Performing action for task[%s]: %s"
,
task
.
getId
(),
taskAction
);
final
RetType
ret
=
taskAction
.
perform
(
task
,
toolbox
);
if
(
taskAction
.
isAudited
())
{
// Add audit log
try
{
storage
.
addAuditLog
(
task
,
taskAction
);
}
catch
(
Exception
e
)
{
final
String
actionClass
=
taskAction
.
getClass
().
getName
();
log
.
makeAlert
(
e
,
"Failed to record action in audit log"
)
.
addData
(
"task"
,
task
.
getId
())
.
addData
(
"actionClass"
,
taskAction
.
getClass
().
getName
()
)
.
addData
(
"actionClass"
,
actionClass
)
.
emit
();
throw
new
ISE
(
e
,
"Failed to record action [%s] in audit log"
,
actionClass
);
}
}
return
ret
;
return
taskAction
.
perform
(
task
,
toolbox
)
;
}
}
indexing-service/src/main/java/io/druid/indexing/common/actions/SegmentInsertAction.java
浏览文件 @
23edba9e
...
...
@@ -24,7 +24,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
import
com.fasterxml.jackson.annotation.JsonProperty
;
import
com.fasterxml.jackson.core.type.TypeReference
;
import
com.google.common.collect.ImmutableSet
;
import
com.metamx.common.ISE
;
import
com.metamx.emitter.service.ServiceMetricEvent
;
import
io.druid.indexing.common.task.Task
;
import
io.druid.timeline.DataSegment
;
...
...
@@ -80,9 +79,7 @@ public class SegmentInsertAction implements TaskAction<Set<DataSegment>>
@Override
public
Set
<
DataSegment
>
perform
(
Task
task
,
TaskActionToolbox
toolbox
)
throws
IOException
{
if
(!
toolbox
.
taskLockCoversSegments
(
task
,
segments
,
allowOlderVersions
))
{
throw
new
ISE
(
"Segments not covered by locks for task[%s]: %s"
,
task
.
getId
(),
segments
);
}
toolbox
.
verifyTaskLocksAndSinglePartitionSettitude
(
task
,
segments
,
true
);
final
Set
<
DataSegment
>
retVal
=
toolbox
.
getIndexerDBCoordinator
().
announceHistoricalSegments
(
segments
);
...
...
indexing-service/src/main/java/io/druid/indexing/common/actions/SegmentMetadataUpdateAction.java
浏览文件 @
23edba9e
...
...
@@ -5,7 +5,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
import
com.fasterxml.jackson.annotation.JsonProperty
;
import
com.fasterxml.jackson.core.type.TypeReference
;
import
com.google.common.collect.ImmutableSet
;
import
com.metamx.common.ISE
;
import
com.metamx.emitter.service.ServiceMetricEvent
;
import
io.druid.indexing.common.task.Task
;
import
io.druid.timeline.DataSegment
;
...
...
@@ -42,10 +41,7 @@ public class SegmentMetadataUpdateAction implements TaskAction<Void>
Task
task
,
TaskActionToolbox
toolbox
)
throws
IOException
{
if
(!
toolbox
.
taskLockCoversSegments
(
task
,
segments
,
true
))
{
throw
new
ISE
(
"Segments not covered by locks for task: %s"
,
task
.
getId
());
}
toolbox
.
verifyTaskLocksAndSinglePartitionSettitude
(
task
,
segments
,
true
);
toolbox
.
getIndexerDBCoordinator
().
updateSegmentMetadata
(
segments
);
// Emit metrics
...
...
indexing-service/src/main/java/io/druid/indexing/common/actions/SegmentNukeAction.java
浏览文件 @
23edba9e
...
...
@@ -24,7 +24,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
import
com.fasterxml.jackson.annotation.JsonProperty
;
import
com.fasterxml.jackson.core.type.TypeReference
;
import
com.google.common.collect.ImmutableSet
;
import
com.metamx.common.ISE
;
import
com.metamx.emitter.service.ServiceMetricEvent
;
import
io.druid.indexing.common.task.Task
;
import
io.druid.timeline.DataSegment
;
...
...
@@ -59,10 +58,7 @@ public class SegmentNukeAction implements TaskAction<Void>
@Override
public
Void
perform
(
Task
task
,
TaskActionToolbox
toolbox
)
throws
IOException
{
if
(!
toolbox
.
taskLockCoversSegments
(
task
,
segments
,
true
))
{
throw
new
ISE
(
"Segments not covered by locks for task: %s"
,
task
.
getId
());
}
toolbox
.
verifyTaskLocksAndSinglePartitionSettitude
(
task
,
segments
,
true
);
toolbox
.
getIndexerDBCoordinator
().
deleteSegments
(
segments
);
// Emit metrics
...
...
indexing-service/src/main/java/io/druid/indexing/common/actions/TaskActionToolbox.java
浏览文件 @
23edba9e
...
...
@@ -19,9 +19,11 @@
package
io.druid.indexing.common.actions
;
import
com.google.api.client.repackaged.com.google.common.base.Preconditions
;
import
com.google.common.base.Predicate
;
import
com.google.common.collect.Iterables
;
import
com.google.inject.Inject
;
import
com.metamx.common.ISE
;
import
com.metamx.emitter.service.ServiceEmitter
;
import
io.druid.indexing.common.TaskLock
;
import
io.druid.indexing.common.task.Task
;
...
...
@@ -65,6 +67,38 @@ public class TaskActionToolbox
return
emitter
;
}
public
boolean
segmentsAreFromSamePartitionSet
(
final
Set
<
DataSegment
>
segments
)
{
// Verify that these segments are all in the same partition set
Preconditions
.
checkArgument
(!
segments
.
isEmpty
(),
"segments nonempty"
);
final
DataSegment
firstSegment
=
segments
.
iterator
().
next
();
for
(
final
DataSegment
segment
:
segments
)
{
if
(!
segment
.
getDataSource
().
equals
(
firstSegment
.
getDataSource
())
||
!
segment
.
getInterval
().
equals
(
firstSegment
.
getInterval
())
||
!
segment
.
getVersion
().
equals
(
firstSegment
.
getVersion
()))
{
return
false
;
}
}
return
true
;
}
public
void
verifyTaskLocksAndSinglePartitionSettitude
(
final
Task
task
,
final
Set
<
DataSegment
>
segments
,
final
boolean
allowOlderVersions
)
{
if
(!
taskLockCoversSegments
(
task
,
segments
,
allowOlderVersions
))
{
throw
new
ISE
(
"Segments not covered by locks for task: %s"
,
task
.
getId
());
}
if
(!
segmentsAreFromSamePartitionSet
(
segments
))
{
throw
new
ISE
(
"Segments are not in the same partition set: %s"
,
segments
);
}
}
public
boolean
taskLockCoversSegments
(
final
Task
task
,
final
Set
<
DataSegment
>
segments
,
...
...
indexing-service/src/main/java/io/druid/indexing/common/task/AbstractFixedIntervalTask.java
浏览文件 @
23edba9e
...
...
@@ -21,6 +21,7 @@ package io.druid.indexing.common.task;
import
com.fasterxml.jackson.annotation.JsonIgnore
;
import
com.fasterxml.jackson.annotation.JsonProperty
;
import
com.google.common.base.Preconditions
;
import
io.druid.indexing.common.actions.LockTryAcquireAction
;
import
io.druid.indexing.common.actions.TaskActionClient
;
import
org.joda.time.Interval
;
...
...
@@ -58,7 +59,8 @@ public abstract class AbstractFixedIntervalTask extends AbstractTask
)
{
super
(
id
,
groupId
,
taskResource
,
dataSource
);
this
.
interval
=
interval
;
this
.
interval
=
Preconditions
.
checkNotNull
(
interval
,
"interval"
);
Preconditions
.
checkArgument
(
interval
.
toDurationMillis
()
>
0
,
"interval empty"
);
}
@Override
...
...
indexing-service/src/main/java/io/druid/indexing/common/task/DeleteTask.java
浏览文件 @
23edba9e
...
...
@@ -22,6 +22,7 @@ package io.druid.indexing.common.task;
import
com.fasterxml.jackson.annotation.JsonCreator
;
import
com.fasterxml.jackson.annotation.JsonProperty
;
import
com.google.common.base.Preconditions
;
import
com.google.common.collect.ImmutableList
;
import
com.google.common.collect.ImmutableSet
;
import
com.google.common.collect.Iterables
;
import
com.google.common.collect.Lists
;
...
...
@@ -103,7 +104,7 @@ public class DeleteTask extends AbstractFixedIntervalTask
segment
.
getVersion
()
);
toolbox
.
getTaskActionClient
().
submit
(
new
SegmentInsertAction
(
ImmutableSet
.
of
(
uploadedSegment
)
));
toolbox
.
pushSegments
(
ImmutableList
.
of
(
uploadedSegment
));
return
TaskStatus
.
success
(
getId
());
}
...
...
indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java
浏览文件 @
23edba9e
...
...
@@ -24,10 +24,14 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
import
com.fasterxml.jackson.annotation.JsonProperty
;
import
com.fasterxml.jackson.core.type.TypeReference
;
import
com.google.api.client.util.Lists
;
import
com.google.common.base.Function
;
import
com.google.common.base.Joiner
;
import
com.google.common.base.Preconditions
;
import
com.google.common.collect.ImmutableSet
;
import
com.google.common.collect.Iterables
;
import
com.google.common.collect.Maps
;
import
com.google.common.collect.Multimap
;
import
com.google.common.collect.Multimaps
;
import
com.metamx.common.logger.Logger
;
import
io.druid.common.utils.JodaUtils
;
import
io.druid.indexer.HadoopDruidIndexerConfig
;
...
...
@@ -47,12 +51,15 @@ import io.tesla.aether.internal.DefaultTeslaAether;
import
org.joda.time.DateTime
;
import
org.joda.time.Interval
;
import
javax.annotation.Nullable
;
import
java.io.File
;
import
java.lang.reflect.Method
;
import
java.net.URL
;
import
java.net.URLClassLoader
;
import
java.util.Arrays
;
import
java.util.Collection
;
import
java.util.List
;
import
java.util.Map
;
public
class
HadoopIndexTask
extends
AbstractFixedIntervalTask
{
...
...
@@ -180,14 +187,10 @@ public class HadoopIndexTask extends AbstractFixedIntervalTask
if
(
segments
!=
null
)
{
List
<
DataSegment
>
publishedSegments
=
toolbox
.
getObjectMapper
().
readValue
(
segments
,
new
TypeReference
<
List
<
DataSegment
>>()
{
}
segments
,
new
TypeReference
<
List
<
DataSegment
>>()
{}
);
// Request segment pushes
toolbox
.
getTaskActionClient
().
submit
(
new
SegmentInsertAction
(
ImmutableSet
.
copyOf
(
publishedSegments
)));
// Done
toolbox
.
pushSegments
(
publishedSegments
);
return
TaskStatus
.
success
(
getId
());
}
else
{
return
TaskStatus
.
failure
(
getId
());
...
...
indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java
浏览文件 @
23edba9e
...
...
@@ -156,7 +156,7 @@ public class IndexTask extends AbstractFixedIntervalTask
segments
.
add
(
segment
);
}
}
toolbox
.
getTaskActionClient
().
submit
(
new
SegmentInsertAction
(
segments
)
);
toolbox
.
pushSegments
(
segments
);
return
TaskStatus
.
success
(
getId
());
}
...
...
indexing-service/src/main/java/io/druid/indexing/common/task/KillTask.java
浏览文件 @
23edba9e
...
...
@@ -97,11 +97,9 @@ public class KillTask extends AbstractFixedIntervalTask
// Kill segments
for
(
DataSegment
segment
:
unusedSegments
)
{
toolbox
.
getDataSegmentKiller
().
kill
(
segment
);
toolbox
.
getTaskActionClient
().
submit
(
new
SegmentNukeAction
(
ImmutableSet
.
of
(
segment
)));
}
// Remove metadata for these segments
toolbox
.
getTaskActionClient
().
submit
(
new
SegmentNukeAction
(
ImmutableSet
.
copyOf
(
unusedSegments
)));
return
TaskStatus
.
success
(
getId
());
}
}
indexing-service/src/main/java/io/druid/indexing/common/task/MergeTaskBase.java
浏览文件 @
23edba9e
...
...
@@ -27,6 +27,7 @@ import com.google.common.base.Joiner;
import
com.google.common.base.Objects
;
import
com.google.common.base.Preconditions
;
import
com.google.common.base.Predicate
;
import
com.google.common.collect.ImmutableList
;
import
com.google.common.collect.ImmutableSet
;
import
com.google.common.collect.Iterables
;
import
com.google.common.collect.Lists
;
...
...
@@ -142,7 +143,7 @@ public abstract class MergeTaskBase extends AbstractFixedIntervalTask
);
// download segments to merge
final
Map
<
DataSegment
,
File
>
gettedSegments
=
toolbox
.
get
Segments
(
segments
);
final
Map
<
DataSegment
,
File
>
gettedSegments
=
toolbox
.
fetch
Segments
(
segments
);
// merge files together
final
File
fileToUpload
=
merge
(
gettedSegments
,
new
File
(
taskDir
,
"merged"
));
...
...
@@ -165,7 +166,7 @@ public abstract class MergeTaskBase extends AbstractFixedIntervalTask
emitter
.
emit
(
builder
.
build
(
"merger/uploadTime"
,
System
.
currentTimeMillis
()
-
uploadStart
));
emitter
.
emit
(
builder
.
build
(
"merger/mergeSize"
,
uploadedSegment
.
getSize
()));
toolbox
.
getTaskActionClient
().
submit
(
new
SegmentInsertAction
(
ImmutableSet
.
of
(
uploadedSegment
)
));
toolbox
.
pushSegments
(
ImmutableList
.
of
(
uploadedSegment
));
return
TaskStatus
.
success
(
getId
());
}
...
...
indexing-service/src/main/java/io/druid/indexing/common/task/MoveTask.java
浏览文件 @
23edba9e
...
...
@@ -98,18 +98,12 @@ public class MoveTask extends AbstractFixedIntervalTask
log
.
info
(
"OK to move segment: %s"
,
unusedSegment
.
getIdentifier
());
}
List
<
DataSegment
>
movedSegments
=
Lists
.
newLinkedList
();
// Move segments
for
(
DataSegment
segment
:
unusedSegments
)
{
movedSegments
.
add
(
toolbox
.
getDataSegmentMover
().
move
(
segment
,
targetLoadSpec
));
final
DataSegment
movedSegment
=
toolbox
.
getDataSegmentMover
().
move
(
segment
,
targetLoadSpec
);
toolbox
.
getTaskActionClient
().
submit
(
new
SegmentMetadataUpdateAction
(
ImmutableSet
.
of
(
movedSegment
)));
}
// Update metadata for moved segments
toolbox
.
getTaskActionClient
().
submit
(
new
SegmentMetadataUpdateAction
(
ImmutableSet
.
copyOf
(
movedSegments
)
));
return
TaskStatus
.
success
(
getId
());
}
...
...
indexing-service/src/main/java/io/druid/indexing/common/task/RealtimeIndexTask.java
浏览文件 @
23edba9e
...
...
@@ -418,7 +418,7 @@ public class RealtimeIndexTask extends AbstractTask
@Override
public
void
publishSegment
(
DataSegment
segment
)
throws
IOException
{
taskToolbox
.
getTaskActionClient
().
submit
(
new
SegmentInsertAction
(
ImmutableSet
.
of
(
segment
)
));
taskToolbox
.
pushSegments
(
ImmutableList
.
of
(
segment
));
}
}
}
indexing-service/src/main/java/io/druid/indexing/common/task/VersionConverterTask.java
浏览文件 @
23edba9e
...
...
@@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
import
com.fasterxml.jackson.annotation.JsonProperty
;
import
com.google.common.base.Function
;
import
com.google.common.base.Preconditions
;
import
com.google.common.collect.ImmutableList
;
import
com.google.common.collect.Sets
;
import
com.metamx.common.guava.FunctionalIterable
;
import
com.metamx.common.logger.Logger
;
...
...
@@ -248,7 +249,7 @@ public class VersionConverterTask extends AbstractFixedIntervalTask
}
}
final
Map
<
DataSegment
,
File
>
localSegments
=
toolbox
.
get
Segments
(
Arrays
.
asList
(
segment
));
final
Map
<
DataSegment
,
File
>
localSegments
=
toolbox
.
fetch
Segments
(
Arrays
.
asList
(
segment
));
final
File
location
=
localSegments
.
get
(
segment
);
final
File
outLocation
=
new
File
(
location
,
"v9_out"
);
...
...
indexing-service/src/main/java/io/druid/indexing/overlord/DbTaskStorage.java
浏览文件 @
23edba9e
...
...
@@ -489,7 +489,18 @@ public class DbTaskStorage implements TaskStorage
final
Map
<
Long
,
TaskLock
>
retMap
=
Maps
.
newHashMap
();
for
(
final
Map
<
String
,
Object
>
row
:
dbTaskLocks
)
{
retMap
.
put
((
Long
)
row
.
get
(
"id"
),
jsonMapper
.
readValue
((
byte
[])
row
.
get
(
"lock_payload"
),
TaskLock
.
class
));
try
{
retMap
.
put
(
(
Long
)
row
.
get
(
"id"
),
jsonMapper
.
readValue
((
byte
[])
row
.
get
(
"lock_payload"
),
TaskLock
.
class
)
);
}
catch
(
Exception
e
)
{
log
.
makeAlert
(
e
,
"Failed to deserialize TaskLock"
)
.
addData
(
"task"
,
taskid
)
.
addData
(
"lockPayload"
,
row
)
.
emit
();
}
}
return
retMap
;
}
...
...
indexing-service/src/main/java/io/druid/indexing/overlord/IndexerDBCoordinator.java
浏览文件 @
23edba9e
...
...
@@ -42,6 +42,7 @@ import org.skife.jdbi.v2.ResultIterator;
import
org.skife.jdbi.v2.StatementContext
;
import
org.skife.jdbi.v2.TransactionCallback
;
import
org.skife.jdbi.v2.TransactionStatus
;
import
org.skife.jdbi.v2.exceptions.CallbackFailedException
;
import
org.skife.jdbi.v2.tweak.HandleCallback
;
import
java.io.IOException
;
...
...
@@ -169,39 +170,39 @@ public class IndexerDBCoordinator
private
boolean
announceHistoricalSegment
(
final
Handle
handle
,
final
DataSegment
segment
)
throws
IOException
{
try
{
final
List
<
Map
<
String
,
Object
>>
exists
=
handle
.
createQuery
(
String
.
format
(
"SELECT id FROM %s WHERE id = :identifier"
,
dbTables
.
getSegmentsTable
()
)
).
bind
(
"identifier"
,
segment
.
getIdentifier
()
).
list
();
if
(!
exists
.
isEmpty
())
{
if
(
segmentExists
(
handle
,
segment
))
{
log
.
info
(
"Found [%s] in DB, not updating DB"
,
segment
.
getIdentifier
());
return
false
;
}
handle
.
createStatement
(
String
.
format
(
"INSERT INTO %s (id, dataSource, created_date, start, end, partitioned, version, used, payload) VALUES (:id, :dataSource, :created_date, :start, :end, :partitioned, :version, :used, :payload)"
,
dbTables
.
getSegmentsTable
()
)
)
.
bind
(
"id"
,
segment
.
getIdentifier
())
.
bind
(
"dataSource"
,
segment
.
getDataSource
())
.
bind
(
"created_date"
,
new
DateTime
().
toString
())
.
bind
(
"start"
,
segment
.
getInterval
().
getStart
().
toString
())
.
bind
(
"end"
,
segment
.
getInterval
().
getEnd
().
toString
())
.
bind
(
"partitioned"
,
segment
.
getShardSpec
().
getPartitionNum
())
.
bind
(
"version"
,
segment
.
getVersion
())
.
bind
(
"used"
,
true
)
.
bind
(
"payload"
,
jsonMapper
.
writeValueAsString
(
segment
))
.
execute
();
log
.
info
(
"Published segment [%s] to DB"
,
segment
.
getIdentifier
());
// Try/catch to work around races due to SELECT -> INSERT. Avoid ON DUPLICATE KEY since it's not portable.
try
{
handle
.
createStatement
(
String
.
format
(
"INSERT INTO %s (id, dataSource, created_date, start, end, partitioned, version, used, payload) "
+
"VALUES (:id, :dataSource, :created_date, :start, :end, :partitioned, :version, :used, :payload)"
,
dbTables
.
getSegmentsTable
()
)
)
.
bind
(
"id"
,
segment
.
getIdentifier
())
.
bind
(
"dataSource"
,
segment
.
getDataSource
())
.
bind
(
"created_date"
,
new
DateTime
().
toString
())
.
bind
(
"start"
,
segment
.
getInterval
().
getStart
().
toString
())
.
bind
(
"end"
,
segment
.
getInterval
().
getEnd
().
toString
())
.
bind
(
"partitioned"
,
segment
.
getShardSpec
().
getPartitionNum
())
.
bind
(
"version"
,
segment
.
getVersion
())
.
bind
(
"used"
,
true
)
.
bind
(
"payload"
,
jsonMapper
.
writeValueAsString
(
segment
))
.
execute
();
log
.
info
(
"Published segment [%s] to DB"
,
segment
.
getIdentifier
());
}
catch
(
Exception
e
)
{
if
(
e
.
getCause
()
instanceof
SQLException
&&
segmentExists
(
handle
,
segment
))
{
log
.
info
(
"Found [%s] in DB, not updating DB"
,
segment
.
getIdentifier
());
}
else
{
throw
e
;
}
}
}
catch
(
IOException
e
)
{
log
.
error
(
e
,
"Exception inserting into DB"
);
...
...
@@ -211,6 +212,20 @@ public class IndexerDBCoordinator
return
true
;
}
private
boolean
segmentExists
(
final
Handle
handle
,
final
DataSegment
segment
)
{
final
List
<
Map
<
String
,
Object
>>
exists
=
handle
.
createQuery
(
String
.
format
(
"SELECT id FROM %s WHERE id = :identifier"
,
dbTables
.
getSegmentsTable
()
)
).
bind
(
"identifier"
,
segment
.
getIdentifier
()
).
list
();
return
!
exists
.
isEmpty
();
}
public
void
updateSegmentMetadata
(
final
Set
<
DataSegment
>
segments
)
throws
IOException
{
dbi
.
inTransaction
(
...
...
indexing-service/src/main/java/io/druid/indexing/overlord/TaskLockbox.java
浏览文件 @
23edba9e
...
...
@@ -19,6 +19,7 @@
package
io.druid.indexing.overlord
;
import
com.google.api.client.repackaged.com.google.common.base.Preconditions
;
import
com.google.common.base.Function
;
import
com.google.common.base.Objects
;
import
com.google.common.base.Optional
;
...
...
@@ -109,6 +110,11 @@ public class TaskLockbox
for
(
final
Pair
<
Task
,
TaskLock
>
taskAndLock
:
byVersionOrdering
.
sortedCopy
(
storedLocks
))
{
final
Task
task
=
taskAndLock
.
lhs
;
final
TaskLock
savedTaskLock
=
taskAndLock
.
rhs
;
if
(
savedTaskLock
.
getInterval
().
toDurationMillis
()
<=
0
)
{
// "Impossible", but you never know what crazy stuff can be restored from storage.
log
.
warn
(
"WTF?! Got lock with empty interval for task: %s"
,
task
.
getId
());
continue
;
}
uniqueTaskIds
.
add
(
task
.
getId
());
final
Optional
<
TaskLock
>
acquiredTaskLock
=
tryLock
(
task
,
...
...
@@ -205,6 +211,7 @@ public class TaskLockbox
giant
.
lock
();
try
{
Preconditions
.
checkArgument
(
interval
.
toDurationMillis
()
>
0
,
"interval empty"
);
final
String
dataSource
=
task
.
getDataSource
();
final
List
<
TaskLockPosse
>
foundPosses
=
findLockPossesForInterval
(
dataSource
,
interval
);
final
TaskLockPosse
posseToUse
;
...
...
pom.xml
浏览文件 @
23edba9e
...
...
@@ -73,7 +73,7 @@
<dependency>
<groupId>
com.metamx
</groupId>
<artifactId>
emitter
</artifactId>
<version>
0.2.
6
</version>
<version>
0.2.
7
</version>
</dependency>
<dependency>
<groupId>
com.metamx
</groupId>
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录