Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
Forever310
druid
提交
5d53b14b
D
druid
项目概览
Forever310
/
druid
与 Fork 源项目一致
从无法访问的项目Fork
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
D
druid
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
5d53b14b
编写于
7月 18, 2014
作者:
F
fjy
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #642 from metamx/numShards-IndexTask
Add way to skip determine partitions for index task
上级
2be4c04b
4ce12470
变更
7
隐藏空白更改
内联
并排
Showing
7 changed file
with
70 addition
and
21 deletion
+70
-21
indexing-hadoop/src/main/java/io/druid/indexer/DetermineHashedPartitionsJob.java
...n/java/io/druid/indexer/DetermineHashedPartitionsJob.java
+10
-1
indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidDetermineConfigurationJob.java
...o/druid/indexer/HadoopDruidDetermineConfigurationJob.java
+1
-1
indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java
...rc/main/java/io/druid/indexing/common/task/IndexTask.java
+37
-6
indexing-service/src/test/java/io/druid/indexing/common/task/TaskSerdeTest.java
...est/java/io/druid/indexing/common/task/TaskSerdeTest.java
+7
-2
indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java
...st/java/io/druid/indexing/overlord/TaskLifecycleTest.java
+5
-2
server/src/main/java/io/druid/timeline/partition/HashBasedNumberedShardSpec.java
.../druid/timeline/partition/HashBasedNumberedShardSpec.java
+4
-4
server/src/test/java/io/druid/server/shard/HashBasedNumberedShardSpecTest.java
...io/druid/server/shard/HashBasedNumberedShardSpecTest.java
+6
-5
未找到文件。
indexing-hadoop/src/main/java/io/druid/indexer/DetermineHashedPartitionsJob.java
浏览文件 @
5d53b14b
...
...
@@ -179,7 +179,16 @@ public class DetermineHashedPartitionsJob implements Jobby
actualSpecs
.
add
(
new
HadoopyShardSpec
(
new
NoneShardSpec
(),
shardCount
++));
}
else
{
for
(
int
i
=
0
;
i
<
numberOfShards
;
++
i
)
{
actualSpecs
.
add
(
new
HadoopyShardSpec
(
new
HashBasedNumberedShardSpec
(
i
,
numberOfShards
),
shardCount
++));
actualSpecs
.
add
(
new
HadoopyShardSpec
(
new
HashBasedNumberedShardSpec
(
i
,
numberOfShards
,
HadoopDruidIndexerConfig
.
jsonMapper
),
shardCount
++
)
);
log
.
info
(
"DateTime[%s], partition[%d], spec[%s]"
,
bucket
,
i
,
actualSpecs
.
get
(
i
));
}
}
...
...
indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidDetermineConfigurationJob.java
浏览文件 @
5d53b14b
...
...
@@ -67,7 +67,7 @@ public class HadoopDruidDetermineConfigurationJob implements Jobby
for
(
int
i
=
0
;
i
<
shardsPerInterval
;
i
++)
{
specs
.
add
(
new
HadoopyShardSpec
(
new
HashBasedNumberedShardSpec
(
i
,
shardsPerInterval
),
new
HashBasedNumberedShardSpec
(
i
,
shardsPerInterval
,
HadoopDruidIndexerConfig
.
jsonMapper
),
shardCount
++
)
);
...
...
indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java
浏览文件 @
5d53b14b
...
...
@@ -19,10 +19,13 @@
package
io.druid.indexing.common.task
;
import
com.fasterxml.jackson.annotation.JacksonInject
;
import
com.fasterxml.jackson.annotation.JsonCreator
;
import
com.fasterxml.jackson.annotation.JsonIgnore
;
import
com.fasterxml.jackson.annotation.JsonProperty
;
import
com.fasterxml.jackson.annotation.JsonTypeName
;
import
com.fasterxml.jackson.databind.ObjectMapper
;
import
com.google.common.base.Preconditions
;
import
com.google.common.collect.ImmutableList
;
import
com.google.common.collect.Iterables
;
import
com.google.common.collect.Lists
;
...
...
@@ -53,12 +56,14 @@ import io.druid.segment.loading.DataSegmentPusher;
import
io.druid.segment.realtime.FireDepartmentMetrics
;
import
io.druid.segment.realtime.plumber.Plumber
;
import
io.druid.timeline.DataSegment
;
import
io.druid.timeline.partition.HashBasedNumberedShardSpec
;
import
io.druid.timeline.partition.NoneShardSpec
;
import
io.druid.timeline.partition.ShardSpec
;
import
io.druid.timeline.partition.SingleDimensionShardSpec
;
import
org.joda.time.DateTime
;
import
org.joda.time.Interval
;
import
javax.annotation.Nullable
;
import
java.io.File
;
import
java.io.IOException
;
import
java.util.List
;
...
...
@@ -107,6 +112,8 @@ public class IndexTask extends AbstractFixedIntervalTask
@JsonIgnore
private
final
IndexIngestionSpec
ingestionSchema
;
private
final
ObjectMapper
jsonMapper
;
@JsonCreator
public
IndexTask
(
@JsonProperty
(
"id"
)
String
id
,
...
...
@@ -118,7 +125,8 @@ public class IndexTask extends AbstractFixedIntervalTask
@JsonProperty
(
"indexGranularity"
)
final
QueryGranularity
indexGranularity
,
@JsonProperty
(
"targetPartitionSize"
)
final
int
targetPartitionSize
,
@JsonProperty
(
"firehose"
)
final
FirehoseFactory
firehoseFactory
,
@JsonProperty
(
"rowFlushBoundary"
)
final
int
rowFlushBoundary
@JsonProperty
(
"rowFlushBoundary"
)
final
int
rowFlushBoundary
,
@JacksonInject
ObjectMapper
jsonMapper
)
{
super
(
...
...
@@ -139,9 +147,10 @@ public class IndexTask extends AbstractFixedIntervalTask
granularitySpec
.
withQueryGranularity
(
indexGranularity
==
null
?
QueryGranularity
.
NONE
:
indexGranularity
)
),
new
IndexIOConfig
(
firehoseFactory
),
new
IndexTuningConfig
(
targetPartitionSize
,
rowFlushBoundary
)
new
IndexTuningConfig
(
targetPartitionSize
,
rowFlushBoundary
,
null
)
);
}
this
.
jsonMapper
=
jsonMapper
;
}
@Override
...
...
@@ -175,7 +184,15 @@ public class IndexTask extends AbstractFixedIntervalTask
if
(
targetPartitionSize
>
0
)
{
shardSpecs
=
determinePartitions
(
bucket
,
targetPartitionSize
);
}
else
{
shardSpecs
=
ImmutableList
.<
ShardSpec
>
of
(
new
NoneShardSpec
());
int
numShards
=
ingestionSchema
.
getTuningConfig
().
getNumShards
();
if
(
numShards
>
0
)
{
shardSpecs
=
Lists
.
newArrayList
();
for
(
int
i
=
0
;
i
<
numShards
;
i
++)
{
shardSpecs
.
add
(
new
HashBasedNumberedShardSpec
(
i
,
numShards
,
jsonMapper
));
}
}
else
{
shardSpecs
=
ImmutableList
.<
ShardSpec
>
of
(
new
NoneShardSpec
());
}
}
for
(
final
ShardSpec
shardSpec
:
shardSpecs
)
{
final
DataSegment
segment
=
generateSegment
(
...
...
@@ -206,6 +223,7 @@ public class IndexTask extends AbstractFixedIntervalTask
retVal
.
add
(
interval
);
}
}
return
retVal
;
}
...
...
@@ -477,7 +495,7 @@ public class IndexTask extends AbstractFixedIntervalTask
this
.
dataSchema
=
dataSchema
;
this
.
ioConfig
=
ioConfig
;
this
.
tuningConfig
=
tuningConfig
==
null
?
new
IndexTuningConfig
(
0
,
0
)
:
tuningConfig
;
this
.
tuningConfig
=
tuningConfig
==
null
?
new
IndexTuningConfig
(
0
,
0
,
null
)
:
tuningConfig
;
}
@Override
...
...
@@ -530,15 +548,22 @@ public class IndexTask extends AbstractFixedIntervalTask
private
final
int
targetPartitionSize
;
private
final
int
rowFlushBoundary
;
private
final
int
numShards
;
@JsonCreator
public
IndexTuningConfig
(
@JsonProperty
(
"targetPartitionSize"
)
int
targetPartitionSize
,
@JsonProperty
(
"rowFlushBoundary"
)
int
rowFlushBoundary
)
@JsonProperty
(
"rowFlushBoundary"
)
int
rowFlushBoundary
,
@JsonProperty
(
"numShards"
)
@Nullable
Integer
numShards
)
{
this
.
targetPartitionSize
=
targetPartitionSize
==
0
?
DEFAULT_TARGET_PARTITION_SIZE
:
targetPartitionSize
;
this
.
rowFlushBoundary
=
rowFlushBoundary
==
0
?
DEFAULT_ROW_FLUSH_BOUNDARY
:
rowFlushBoundary
;
this
.
numShards
=
numShards
==
null
?
-
1
:
numShards
;
Preconditions
.
checkArgument
(
this
.
targetPartitionSize
==
-
1
||
this
.
numShards
==
-
1
,
"targetPartitionsSize and shardCount both cannot be set"
);
}
@JsonProperty
...
...
@@ -552,5 +577,11 @@ public class IndexTask extends AbstractFixedIntervalTask
{
return
rowFlushBoundary
;
}
@JsonProperty
public
int
getNumShards
()
{
return
numShards
;
}
}
}
indexing-service/src/test/java/io/druid/indexing/common/task/TaskSerdeTest.java
浏览文件 @
5d53b14b
...
...
@@ -19,6 +19,8 @@
package
io.druid.indexing.common.task
;
import
com.fasterxml.jackson.annotation.JsonTypeInfo
;
import
com.fasterxml.jackson.databind.InjectableValues
;
import
com.fasterxml.jackson.databind.Module
;
import
com.fasterxml.jackson.databind.ObjectMapper
;
import
com.google.common.collect.ImmutableList
;
...
...
@@ -67,16 +69,19 @@ public class TaskSerdeTest
QueryGranularity
.
NONE
,
10000
,
new
LocalFirehoseFactory
(
new
File
(
"lol"
),
"rofl"
,
null
),
-
1
-
1
,
jsonMapper
);
for
(
final
Module
jacksonModule
:
new
FirehoseModule
().
getJacksonModules
())
{
jsonMapper
.
registerModule
(
jacksonModule
);
}
InjectableValues
inject
=
new
InjectableValues
.
Std
()
.
addValue
(
ObjectMapper
.
class
,
jsonMapper
);
final
String
json
=
jsonMapper
.
writeValueAsString
(
task
);
Thread
.
sleep
(
100
);
// Just want to run the clock a bit to make sure the task id doesn't change
final
IndexTask
task2
=
(
IndexTask
)
jsonMapper
.
readValue
(
json
,
Task
.
class
);
final
IndexTask
task2
=
jsonMapper
.
reader
(
Task
.
class
).
with
(
inject
).
readValue
(
json
);
Assert
.
assertEquals
(
"foo"
,
task
.
getDataSource
());
Assert
.
assertEquals
(
new
Interval
(
"2010-01-01/P2D"
),
task
.
getInterval
());
...
...
indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java
浏览文件 @
5d53b14b
...
...
@@ -43,6 +43,7 @@ import io.druid.data.input.InputRow;
import
io.druid.data.input.MapBasedInputRow
;
import
io.druid.data.input.impl.InputRowParser
;
import
io.druid.granularity.QueryGranularity
;
import
io.druid.indexing.common.TestUtils
;
import
io.druid.segment.column.ColumnConfig
;
import
io.druid.segment.indexing.granularity.UniformGranularitySpec
;
import
io.druid.indexing.common.SegmentLoaderFactory
;
...
...
@@ -249,7 +250,8 @@ public class TaskLifecycleTest
IR
(
"2010-01-02T01"
,
"a"
,
"c"
,
1
)
)
),
-
1
-
1
,
TestUtils
.
MAPPER
);
final
Optional
<
TaskStatus
>
preRunTaskStatus
=
tsqa
.
getStatus
(
indexTask
.
getId
());
...
...
@@ -297,7 +299,8 @@ public class TaskLifecycleTest
QueryGranularity
.
NONE
,
10000
,
newMockExceptionalFirehoseFactory
(),
-
1
-
1
,
TestUtils
.
MAPPER
);
final
TaskStatus
status
=
runTask
(
indexTask
);
...
...
server/src/main/java/io/druid/timeline/partition/HashBasedNumberedShardSpec.java
浏览文件 @
5d53b14b
...
...
@@ -34,18 +34,18 @@ import java.util.List;
public
class
HashBasedNumberedShardSpec
extends
NumberedShardSpec
{
private
static
final
HashFunction
hashFunction
=
Hashing
.
murmur3_32
();
@JacksonInject
private
ObjectMapper
jsonMapper
;
private
final
ObjectMapper
jsonMapper
;
@JsonCreator
public
HashBasedNumberedShardSpec
(
@JsonProperty
(
"partitionNum"
)
int
partitionNum
,
@JsonProperty
(
"partitions"
)
int
partitions
@JsonProperty
(
"partitions"
)
int
partitions
,
@JacksonInject
ObjectMapper
jsonMapper
)
{
super
(
partitionNum
,
partitions
);
this
.
jsonMapper
=
jsonMapper
;
}
@Override
...
...
server/src/test/java/io/druid/server/shard/HashBasedNumberedShardSpecTest.java
浏览文件 @
5d53b14b
...
...
@@ -26,6 +26,7 @@ import com.metamx.common.ISE;
import
io.druid.TestUtil
;
import
io.druid.data.input.InputRow
;
import
io.druid.data.input.Row
;
import
io.druid.jackson.DefaultObjectMapper
;
import
io.druid.timeline.partition.HashBasedNumberedShardSpec
;
import
io.druid.timeline.partition.PartitionChunk
;
import
io.druid.timeline.partition.ShardSpec
;
...
...
@@ -43,7 +44,7 @@ public class HashBasedNumberedShardSpecTest
{
final
ShardSpec
spec
=
TestUtil
.
MAPPER
.
readValue
(
TestUtil
.
MAPPER
.
writeValueAsBytes
(
new
HashBasedNumberedShardSpec
(
1
,
2
)),
TestUtil
.
MAPPER
.
writeValueAsBytes
(
new
HashBasedNumberedShardSpec
(
1
,
2
,
TestUtil
.
MAPPER
)),
ShardSpec
.
class
);
Assert
.
assertEquals
(
1
,
spec
.
getPartitionNum
());
...
...
@@ -65,9 +66,9 @@ public class HashBasedNumberedShardSpecTest
public
void
testPartitionChunks
()
{
final
List
<
ShardSpec
>
specs
=
ImmutableList
.<
ShardSpec
>
of
(
new
HashBasedNumberedShardSpec
(
0
,
3
),
new
HashBasedNumberedShardSpec
(
1
,
3
),
new
HashBasedNumberedShardSpec
(
2
,
3
)
new
HashBasedNumberedShardSpec
(
0
,
3
,
TestUtil
.
MAPPER
),
new
HashBasedNumberedShardSpec
(
1
,
3
,
TestUtil
.
MAPPER
),
new
HashBasedNumberedShardSpec
(
2
,
3
,
TestUtil
.
MAPPER
)
);
final
List
<
PartitionChunk
<
String
>>
chunks
=
Lists
.
transform
(
...
...
@@ -141,7 +142,7 @@ public class HashBasedNumberedShardSpecTest
int
partitions
)
{
super
(
partitionNum
,
partitions
);
super
(
partitionNum
,
partitions
,
TestUtil
.
MAPPER
);
}
@Override
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录