Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
Forever310
druid
提交
d17abeda
D
druid
项目概览
Forever310
/
druid
与 Fork 源项目一致
从无法访问的项目Fork
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
D
druid
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
d17abeda
编写于
4月 22, 2014
作者:
X
xvrl
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #499 from metamx/fix-pull-from-cache
Fix pull from cache
上级
2df2cbf2
9d762b87
变更
5
显示空白变更内容
内联
并排
Showing
5 changed file
with
143 addition
and
54 deletion
+143
-54
processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryQueryToolChest.java
...druid/query/timeseries/TimeseriesQueryQueryToolChest.java
+5
-1
processing/src/main/java/io/druid/query/topn/TopNBinaryFn.java
...ssing/src/main/java/io/druid/query/topn/TopNBinaryFn.java
+2
-25
processing/src/main/java/io/druid/query/topn/TopNQueryQueryToolChest.java
...ain/java/io/druid/query/topn/TopNQueryQueryToolChest.java
+15
-2
processing/src/test/java/io/druid/query/topn/TopNBinaryFnTest.java
...g/src/test/java/io/druid/query/topn/TopNBinaryFnTest.java
+14
-7
server/src/test/java/io/druid/client/CachingClusteredClientTest.java
...test/java/io/druid/client/CachingClusteredClientTest.java
+107
-19
未找到文件。
processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryQueryToolChest.java
浏览文件 @
d17abeda
...
...
@@ -265,8 +265,12 @@ public class TimeseriesQueryQueryToolChest extends QueryToolChest<Result<Timeser
final
Map
<
String
,
Object
>
values
=
Maps
.
newHashMap
();
final
TimeseriesResultValue
holder
=
result
.
getValue
();
if
(
calculatePostAggs
)
{
// put non finalized aggregators for calculating dependent post Aggregators
for
(
AggregatorFactory
agg
:
query
.
getAggregatorSpecs
())
{
values
.
put
(
agg
.
getName
(),
holder
.
getMetric
(
agg
.
getName
()));
}
for
(
PostAggregator
postAgg
:
query
.
getPostAggregatorSpecs
())
{
values
.
put
(
postAgg
.
getName
(),
postAgg
.
compute
(
holder
.
getBaseObject
()
));
values
.
put
(
postAgg
.
getName
(),
postAgg
.
compute
(
values
));
}
}
for
(
AggregatorFactory
agg
:
query
.
getAggregatorSpecs
())
{
...
...
processing/src/main/java/io/druid/query/topn/TopNBinaryFn.java
浏览文件 @
d17abeda
...
...
@@ -90,19 +90,7 @@ public class TopNBinaryFn implements BinaryFn<Result<TopNResultValue>, Result<To
TopNResultValue
arg2Vals
=
arg2
.
getValue
();
for
(
DimensionAndMetricValueExtractor
arg1Val
:
arg1Vals
)
{
final
String
dimensionValue
=
arg1Val
.
getStringDimensionValue
(
dimension
);
Map
<
String
,
Object
>
retVal
=
new
LinkedHashMap
<
String
,
Object
>(
aggregations
.
size
()
+
2
);
retVal
.
put
(
dimension
,
dimensionValue
);
for
(
AggregatorFactory
factory
:
aggregations
)
{
final
String
metricName
=
factory
.
getName
();
retVal
.
put
(
metricName
,
arg1Val
.
getMetric
(
metricName
));
}
for
(
PostAggregator
postAgg
:
postAggregations
)
{
retVal
.
put
(
postAgg
.
getName
(),
postAgg
.
compute
(
retVal
));
}
retVals
.
put
(
dimensionValue
,
new
DimensionAndMetricValueExtractor
(
retVal
));
retVals
.
put
(
arg1Val
.
getStringDimensionValue
(
dimension
),
arg1Val
);
}
for
(
DimensionAndMetricValueExtractor
arg2Val
:
arg2Vals
)
{
final
String
dimensionValue
=
arg2Val
.
getStringDimensionValue
(
dimension
);
...
...
@@ -124,18 +112,7 @@ public class TopNBinaryFn implements BinaryFn<Result<TopNResultValue>, Result<To
retVals
.
put
(
dimensionValue
,
new
DimensionAndMetricValueExtractor
(
retVal
));
}
else
{
Map
<
String
,
Object
>
retVal
=
new
LinkedHashMap
<
String
,
Object
>(
aggregations
.
size
()
+
2
);
retVal
.
put
(
dimension
,
dimensionValue
);
for
(
AggregatorFactory
factory
:
aggregations
)
{
final
String
metricName
=
factory
.
getName
();
retVal
.
put
(
metricName
,
arg2Val
.
getMetric
(
metricName
));
}
for
(
PostAggregator
postAgg
:
postAggregations
)
{
retVal
.
put
(
postAgg
.
getName
(),
postAgg
.
compute
(
retVal
));
}
retVals
.
put
(
dimensionValue
,
new
DimensionAndMetricValueExtractor
(
retVal
));
retVals
.
put
(
dimensionValue
,
arg2Val
);
}
}
...
...
processing/src/main/java/io/druid/query/topn/TopNQueryQueryToolChest.java
浏览文件 @
d17abeda
...
...
@@ -208,13 +208,17 @@ public class TopNQueryQueryToolChest extends QueryToolChest<Result<TopNResultVal
public
Map
<
String
,
Object
>
apply
(
DimensionAndMetricValueExtractor
input
)
{
final
Map
<
String
,
Object
>
values
=
Maps
.
newHashMap
();
// compute all post aggs
// put non finalized aggregators for calculating dependent post Aggregators
for
(
AggregatorFactory
agg
:
query
.
getAggregatorSpecs
())
{
values
.
put
(
agg
.
getName
(),
input
.
getMetric
(
agg
.
getName
()));
}
for
(
PostAggregator
postAgg
:
query
.
getPostAggregatorSpecs
())
{
Object
calculatedPostAgg
=
input
.
getMetric
(
postAgg
.
getName
());
if
(
calculatedPostAgg
!=
null
)
{
values
.
put
(
postAgg
.
getName
(),
calculatedPostAgg
);
}
else
{
values
.
put
(
postAgg
.
getName
(),
postAgg
.
compute
(
input
.
getBaseObject
()
));
values
.
put
(
postAgg
.
getName
(),
postAgg
.
compute
(
values
));
}
}
for
(
AggregatorFactory
agg
:
query
.
getAggregatorSpecs
())
{
...
...
@@ -249,6 +253,11 @@ public class TopNQueryQueryToolChest extends QueryToolChest<Result<TopNResultVal
return
new
CacheStrategy
<
Result
<
TopNResultValue
>,
Object
,
TopNQuery
>()
{
private
final
List
<
AggregatorFactory
>
aggs
=
query
.
getAggregatorSpecs
();
private
final
List
<
PostAggregator
>
postAggs
=
AggregatorUtil
.
pruneDependentPostAgg
(
query
.
getPostAggregatorSpecs
(),
query
.
getTopNMetricSpec
()
.
getMetricName
(
query
.
getDimensionSpec
())
);
@Override
public
byte
[]
computeCacheKey
(
TopNQuery
query
)
...
...
@@ -338,6 +347,10 @@ public class TopNQueryQueryToolChest extends QueryToolChest<Result<TopNResultVal
vals
.
put
(
factory
.
getName
(),
factory
.
deserialize
(
resultIter
.
next
()));
}
for
(
PostAggregator
postAgg
:
postAggs
)
{
vals
.
put
(
postAgg
.
getName
(),
postAgg
.
compute
(
vals
));
}
retVal
.
add
(
vals
);
}
...
...
processing/src/test/java/io/druid/query/topn/TopNBinaryFnTest.java
浏览文件 @
d17abeda
...
...
@@ -294,17 +294,20 @@ public class TopNBinaryFnTest
ImmutableMap
.<
String
,
Object
>
of
(
"rows"
,
1L
,
"index"
,
2L
,
"testdim"
,
"1"
"testdim"
,
"1"
,
"addrowsindexconstant"
,
3.0
),
ImmutableMap
.<
String
,
Object
>
of
(
"rows"
,
2L
,
"index"
,
4L
,
"testdim"
,
"2"
"testdim"
,
"2"
,
"addrowsindexconstant"
,
7.0
),
ImmutableMap
.<
String
,
Object
>
of
(
"rows"
,
0L
,
"index"
,
2L
,
"testdim"
,
"3"
"testdim"
,
"3"
,
"addrowsindexconstant"
,
3.0
)
)
)
...
...
@@ -316,17 +319,20 @@ public class TopNBinaryFnTest
ImmutableMap
.<
String
,
Object
>
of
(
"rows"
,
2L
,
"index"
,
3L
,
"testdim"
,
"1"
"testdim"
,
"1"
,
"addrowsindexconstant"
,
6.0
),
ImmutableMap
.<
String
,
Object
>
of
(
"rows"
,
2L
,
"index"
,
0L
,
"testdim"
,
"2"
"testdim"
,
"2"
,
"addrowsindexconstant"
,
3.0
),
ImmutableMap
.<
String
,
Object
>
of
(
"rows"
,
4L
,
"index"
,
5L
,
"testdim"
,
"other"
"testdim"
,
"other"
,
"addrowsindexconstant"
,
10.0
)
)
)
...
...
@@ -434,7 +440,8 @@ public class TopNBinaryFnTest
ImmutableMap
.<
String
,
Object
>
of
(
"testdim"
,
"2"
,
"rows"
,
4L
,
"index"
,
4L
)
"index"
,
4L
)
)
)
);
...
...
server/src/test/java/io/druid/client/CachingClusteredClientTest.java
浏览文件 @
d17abeda
...
...
@@ -61,6 +61,7 @@ import io.druid.query.aggregation.CountAggregatorFactory;
import
io.druid.query.aggregation.LongSumAggregatorFactory
;
import
io.druid.query.aggregation.PostAggregator
;
import
io.druid.query.aggregation.post.ArithmeticPostAggregator
;
import
io.druid.query.aggregation.post.ConstantPostAggregator
;
import
io.druid.query.aggregation.post.FieldAccessPostAggregator
;
import
io.druid.query.filter.DimFilter
;
import
io.druid.query.search.SearchQueryQueryToolChest
;
...
...
@@ -145,6 +146,22 @@ public class CachingClusteredClientTest
new
FieldAccessPostAggregator
(
"imps"
,
"imps"
),
new
FieldAccessPostAggregator
(
"rows"
,
"rows"
)
)
),
new
ArithmeticPostAggregator
(
"avg_imps_per_row_double"
,
"*"
,
Arrays
.<
PostAggregator
>
asList
(
new
FieldAccessPostAggregator
(
"avg_imps_per_row"
,
"avg_imps_per_row"
),
new
ConstantPostAggregator
(
"constant"
,
2
,
2
)
)
),
new
ArithmeticPostAggregator
(
"avg_imps_per_row_half"
,
"/"
,
Arrays
.<
PostAggregator
>
asList
(
new
FieldAccessPostAggregator
(
"avg_imps_per_row"
,
"avg_imps_per_row"
),
new
ConstantPostAggregator
(
"constant"
,
2
,
2
)
)
)
);
private
static
final
List
<
AggregatorFactory
>
RENAMED_AGGS
=
Arrays
.
asList
(
...
...
@@ -412,7 +429,7 @@ public class CachingClusteredClientTest
new
DateTime
(
"2011-01-06"
),
"a"
,
50
,
4991
,
"b"
,
50
,
4990
,
"c"
,
50
,
4989
,
new
DateTime
(
"2011-01-07"
),
"a"
,
50
,
4991
,
"b"
,
50
,
4990
,
"c"
,
50
,
4989
,
new
DateTime
(
"2011-01-08"
),
"a"
,
50
,
4988
,
"b"
,
50
,
4987
,
"c"
,
50
,
4986
,
new
DateTime
(
"2011-01-09"
),
"
a
"
,
50
,
4985
,
"b"
,
50
,
4984
,
"c"
,
50
,
4983
new
DateTime
(
"2011-01-09"
),
"
c1
"
,
50
,
4985
,
"b"
,
50
,
4984
,
"c"
,
50
,
4983
),
new
Interval
(
"2011-01-05/2011-01-10"
),
...
...
@@ -421,7 +438,7 @@ public class CachingClusteredClientTest
new
DateTime
(
"2011-01-06T01"
),
"a"
,
50
,
4991
,
"b"
,
50
,
4990
,
"c"
,
50
,
4989
,
new
DateTime
(
"2011-01-07T01"
),
"a"
,
50
,
4991
,
"b"
,
50
,
4990
,
"c"
,
50
,
4989
,
new
DateTime
(
"2011-01-08T01"
),
"a"
,
50
,
4988
,
"b"
,
50
,
4987
,
"c"
,
50
,
4986
,
new
DateTime
(
"2011-01-09T01"
),
"
a
"
,
50
,
4985
,
"b"
,
50
,
4984
,
"c"
,
50
,
4983
new
DateTime
(
"2011-01-09T01"
),
"
c2
"
,
50
,
4985
,
"b"
,
50
,
4984
,
"c"
,
50
,
4983
)
);
...
...
@@ -437,8 +454,8 @@ public class CachingClusteredClientTest
new
DateTime
(
"2011-01-07T01"
),
"a"
,
50
,
4991
,
"b"
,
50
,
4990
,
"c"
,
50
,
4989
,
new
DateTime
(
"2011-01-08"
),
"a"
,
50
,
4988
,
"b"
,
50
,
4987
,
"c"
,
50
,
4986
,
new
DateTime
(
"2011-01-08T01"
),
"a"
,
50
,
4988
,
"b"
,
50
,
4987
,
"c"
,
50
,
4986
,
new
DateTime
(
"2011-01-09"
),
"
a
"
,
50
,
4985
,
"b"
,
50
,
4984
,
"c"
,
50
,
4983
,
new
DateTime
(
"2011-01-09T01"
),
"
a
"
,
50
,
4985
,
"b"
,
50
,
4984
,
"c"
,
50
,
4983
new
DateTime
(
"2011-01-09"
),
"
c1
"
,
50
,
4985
,
"b"
,
50
,
4984
,
"c"
,
50
,
4983
,
new
DateTime
(
"2011-01-09T01"
),
"
c2
"
,
50
,
4985
,
"b"
,
50
,
4984
,
"c"
,
50
,
4983
),
runner
.
run
(
builder
.
intervals
(
"2011-01-01/2011-01-10"
)
...
...
@@ -567,6 +584,73 @@ public class CachingClusteredClientTest
);
}
@Test
public
void
testTopNOnPostAggMetricCaching
()
{
final
TopNQueryBuilder
builder
=
new
TopNQueryBuilder
()
.
dataSource
(
DATA_SOURCE
)
.
dimension
(
TOP_DIM
)
.
metric
(
"avg_imps_per_row_double"
)
.
threshold
(
3
)
.
intervals
(
SEG_SPEC
)
.
filters
(
DIM_FILTER
)
.
granularity
(
GRANULARITY
)
.
aggregators
(
AGGS
)
.
postAggregators
(
POST_AGGS
)
.
context
(
CONTEXT
);
QueryRunner
runner
=
new
FinalizeResultsQueryRunner
(
client
,
new
TopNQueryQueryToolChest
(
new
TopNQueryConfig
()));
testQueryCaching
(
runner
,
builder
.
build
(),
new
Interval
(
"2011-01-01/2011-01-02"
),
makeTopNResults
(),
new
Interval
(
"2011-01-02/2011-01-03"
),
makeTopNResults
(),
new
Interval
(
"2011-01-05/2011-01-10"
),
makeTopNResults
(
new
DateTime
(
"2011-01-05"
),
"a"
,
50
,
4994
,
"b"
,
50
,
4993
,
"c"
,
50
,
4992
,
new
DateTime
(
"2011-01-06"
),
"a"
,
50
,
4991
,
"b"
,
50
,
4990
,
"c"
,
50
,
4989
,
new
DateTime
(
"2011-01-07"
),
"a"
,
50
,
4991
,
"b"
,
50
,
4990
,
"c"
,
50
,
4989
,
new
DateTime
(
"2011-01-08"
),
"a"
,
50
,
4988
,
"b"
,
50
,
4987
,
"c"
,
50
,
4986
,
new
DateTime
(
"2011-01-09"
),
"c1"
,
50
,
4985
,
"b"
,
50
,
4984
,
"c"
,
50
,
4983
),
new
Interval
(
"2011-01-05/2011-01-10"
),
makeTopNResults
(
new
DateTime
(
"2011-01-05T01"
),
"a"
,
50
,
4994
,
"b"
,
50
,
4993
,
"c"
,
50
,
4992
,
new
DateTime
(
"2011-01-06T01"
),
"a"
,
50
,
4991
,
"b"
,
50
,
4990
,
"c"
,
50
,
4989
,
new
DateTime
(
"2011-01-07T01"
),
"a"
,
50
,
4991
,
"b"
,
50
,
4990
,
"c"
,
50
,
4989
,
new
DateTime
(
"2011-01-08T01"
),
"a"
,
50
,
4988
,
"b"
,
50
,
4987
,
"c"
,
50
,
4986
,
new
DateTime
(
"2011-01-09T01"
),
"c2"
,
50
,
4985
,
"b"
,
50
,
4984
,
"c"
,
50
,
4983
)
);
TestHelper
.
assertExpectedResults
(
makeTopNResults
(
new
DateTime
(
"2011-01-05"
),
"a"
,
50
,
4994
,
"b"
,
50
,
4993
,
"c"
,
50
,
4992
,
new
DateTime
(
"2011-01-05T01"
),
"a"
,
50
,
4994
,
"b"
,
50
,
4993
,
"c"
,
50
,
4992
,
new
DateTime
(
"2011-01-06"
),
"a"
,
50
,
4991
,
"b"
,
50
,
4990
,
"c"
,
50
,
4989
,
new
DateTime
(
"2011-01-06T01"
),
"a"
,
50
,
4991
,
"b"
,
50
,
4990
,
"c"
,
50
,
4989
,
new
DateTime
(
"2011-01-07"
),
"a"
,
50
,
4991
,
"b"
,
50
,
4990
,
"c"
,
50
,
4989
,
new
DateTime
(
"2011-01-07T01"
),
"a"
,
50
,
4991
,
"b"
,
50
,
4990
,
"c"
,
50
,
4989
,
new
DateTime
(
"2011-01-08"
),
"a"
,
50
,
4988
,
"b"
,
50
,
4987
,
"c"
,
50
,
4986
,
new
DateTime
(
"2011-01-08T01"
),
"a"
,
50
,
4988
,
"b"
,
50
,
4987
,
"c"
,
50
,
4986
,
new
DateTime
(
"2011-01-09"
),
"c1"
,
50
,
4985
,
"b"
,
50
,
4984
,
"c"
,
50
,
4983
,
new
DateTime
(
"2011-01-09T01"
),
"c2"
,
50
,
4985
,
"b"
,
50
,
4984
,
"c"
,
50
,
4983
),
runner
.
run
(
builder
.
intervals
(
"2011-01-01/2011-01-10"
)
.
metric
(
"avg_imps_per_row_double"
)
.
aggregators
(
AGGS
)
.
postAggregators
(
POST_AGGS
)
.
build
()
)
);
}
@Test
public
void
testSearchCaching
()
throws
Exception
{
...
...
@@ -1007,17 +1091,19 @@ public class CachingClusteredClientTest
List
<
Result
<
TimeseriesResultValue
>>
retVal
=
Lists
.
newArrayListWithCapacity
(
objects
.
length
/
3
);
for
(
int
i
=
0
;
i
<
objects
.
length
;
i
+=
3
)
{
double
avg_impr
=
((
Number
)
objects
[
i
+
2
]).
doubleValue
()
/
((
Number
)
objects
[
i
+
1
]).
doubleValue
();
retVal
.
add
(
new
Result
<>(
(
DateTime
)
objects
[
i
],
new
TimeseriesResultValue
(
ImmutableMap
.
of
(
"rows"
,
objects
[
i
+
1
],
"imps"
,
objects
[
i
+
2
],
"impers"
,
objects
[
i
+
2
],
"avg_imps_per_row"
,
((
Number
)
objects
[
i
+
2
]).
doubleValue
()
/
((
Number
)
objects
[
i
+
1
]).
doubleValue
()
)
ImmutableMap
.<
String
,
Object
>
builder
()
.
put
(
"rows"
,
objects
[
i
+
1
])
.
put
(
"imps"
,
objects
[
i
+
2
])
.
put
(
"impers"
,
objects
[
i
+
2
])
.
put
(
"avg_imps_per_row"
,
avg_impr
)
.
put
(
"avg_imps_per_row_half"
,
avg_impr
/
2
)
.
put
(
"avg_imps_per_row_double"
,
avg_impr
*
2
)
.
build
()
)
)
);
...
...
@@ -1099,13 +1185,15 @@ public class CachingClusteredClientTest
final
double
imps
=
((
Number
)
objects
[
index
+
2
]).
doubleValue
();
final
double
rows
=
((
Number
)
objects
[
index
+
1
]).
doubleValue
();
values
.
add
(
ImmutableMap
.
of
(
TOP_DIM
,
objects
[
index
],
"rows"
,
rows
,
"imps"
,
imps
,
"impers"
,
imps
,
"avg_imps_per_row"
,
imps
/
rows
)
ImmutableMap
.<
String
,
Object
>
builder
()
.
put
(
TOP_DIM
,
objects
[
index
])
.
put
(
"rows"
,
rows
)
.
put
(
"imps"
,
imps
)
.
put
(
"impers"
,
imps
)
.
put
(
"avg_imps_per_row"
,
imps
/
rows
)
.
put
(
"avg_imps_per_row_double"
,
((
imps
*
2
)
/
rows
))
.
put
(
"avg_imps_per_row_half"
,
(
imps
/
(
rows
*
2
)))
.
build
()
);
index
+=
3
;
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录