Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
Forever310
druid
提交
c2137e84
D
druid
项目概览
Forever310
/
druid
与 Fork 源项目一致
从无法访问的项目Fork
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
D
druid
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
c2137e84
编写于
5月 15, 2014
作者:
G
Gian Merlino
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
ObjectColumnSelector/JS aggregator support for multi-valued columns.
上级
da285d86
变更
7
隐藏空白更改
内联
并排
Showing
7 changed file
with
235 addition
and
41 deletion
+235
-41
processing/src/main/java/io/druid/query/aggregation/JavaScriptAggregatorFactory.java
.../druid/query/aggregation/JavaScriptAggregatorFactory.java
+5
-1
processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java
...n/java/io/druid/segment/QueryableIndexStorageAdapter.java
+78
-33
processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java
...d/segment/incremental/IncrementalIndexStorageAdapter.java
+11
-7
processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java
...g/src/test/java/io/druid/query/QueryRunnerTestHelper.java
+8
-0
processing/src/test/java/io/druid/query/aggregation/JavaScriptAggregatorTest.java
.../io/druid/query/aggregation/JavaScriptAggregatorTest.java
+42
-0
processing/src/test/java/io/druid/query/aggregation/TestObjectColumnSelector.java
.../io/druid/query/aggregation/TestObjectColumnSelector.java
+53
-0
processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java
.../io/druid/query/timeseries/TimeseriesQueryRunnerTest.java
+38
-0
未找到文件。
processing/src/main/java/io/druid/query/aggregation/JavaScriptAggregatorFactory.java
浏览文件 @
c2137e84
...
...
@@ -32,12 +32,15 @@ import org.mozilla.javascript.Context;
import
org.mozilla.javascript.ContextAction
;
import
org.mozilla.javascript.ContextFactory
;
import
org.mozilla.javascript.Function
;
import
org.mozilla.javascript.NativeArray
;
import
org.mozilla.javascript.Scriptable
;
import
org.mozilla.javascript.ScriptableObject
;
import
javax.annotation.Nullable
;
import
java.nio.ByteBuffer
;
import
java.security.MessageDigest
;
import
java.security.NoSuchAlgorithmException
;
import
java.util.Arrays
;
import
java.util.Comparator
;
import
java.util.List
;
...
...
@@ -263,6 +266,7 @@ public class JavaScriptAggregatorFactory implements AggregatorFactory
Context
cx
=
Context
.
getCurrentContext
();
if
(
cx
==
null
)
{
cx
=
contextFactory
.
enterContext
();
cx
.
getWrapFactory
().
setJavaPrimitiveWrap
(
false
);
}
final
int
size
=
selectorList
.
length
;
...
...
@@ -272,7 +276,7 @@ public class JavaScriptAggregatorFactory implements AggregatorFactory
for
(
int
i
=
0
;
i
<
size
;
i
++)
{
final
ObjectColumnSelector
selector
=
selectorList
[
i
];
if
(
selector
!=
null
)
{
args
[
i
+
1
]
=
selector
.
get
(
);
args
[
i
+
1
]
=
Context
.
javaToJS
(
selector
.
get
(),
scope
);
}
}
...
...
processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java
浏览文件 @
c2137e84
...
...
@@ -386,12 +386,6 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
if
(
holder
!=
null
)
{
final
ColumnCapabilities
capabilities
=
holder
.
getCapabilities
();
if
(
capabilities
.
hasMultipleValues
())
{
throw
new
UnsupportedOperationException
(
"makeObjectColumnSelector does not support multivalued columns"
);
}
if
(
capabilities
.
isDictionaryEncoded
())
{
cachedColumnVals
=
holder
.
getDictionaryEncoding
();
}
else
if
(
capabilities
.
getType
()
==
ValueType
.
COMPLEX
)
{
...
...
@@ -414,6 +408,12 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
final
GenericColumn
columnVals
=
(
GenericColumn
)
cachedColumnVals
;
final
ValueType
type
=
columnVals
.
getType
();
if
(
columnVals
.
hasMultipleValues
())
{
throw
new
UnsupportedOperationException
(
"makeObjectColumnSelector does not support multivalued GenericColumns"
);
}
if
(
type
==
ValueType
.
FLOAT
)
{
return
new
ObjectColumnSelector
<
Float
>()
{
...
...
@@ -466,20 +466,42 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
if
(
cachedColumnVals
instanceof
DictionaryEncodedColumn
)
{
final
DictionaryEncodedColumn
columnVals
=
(
DictionaryEncodedColumn
)
cachedColumnVals
;
return
new
ObjectColumnSelector
<
String
>()
{
@Override
public
Class
classOfObject
()
if
(
columnVals
.
hasMultipleValues
())
{
return
new
ObjectColumnSelector
<
String
[]>()
{
return
String
.
class
;
}
@Override
public
Class
classOfObject
()
{
return
String
[].
class
;
}
@Override
public
String
get
()
@Override
public
String
[]
get
()
{
final
IndexedInts
multiValueRow
=
columnVals
.
getMultiValueRow
(
cursorOffset
.
getOffset
());
final
String
[]
strings
=
new
String
[
multiValueRow
.
size
()];
for
(
int
i
=
0
;
i
<
multiValueRow
.
size
()
;
i
++)
{
strings
[
i
]
=
columnVals
.
lookupName
(
multiValueRow
.
get
(
i
));
}
return
strings
;
}
};
}
else
{
return
new
ObjectColumnSelector
<
String
>()
{
return
columnVals
.
lookupName
(
columnVals
.
getSingleValueRow
(
cursorOffset
.
getOffset
()));
}
};
@Override
public
Class
classOfObject
()
{
return
String
.
class
;
}
@Override
public
String
get
()
{
return
columnVals
.
lookupName
(
columnVals
.
getSingleValueRow
(
cursorOffset
.
getOffset
()));
}
};
}
}
final
ComplexColumn
columnVals
=
(
ComplexColumn
)
cachedColumnVals
;
...
...
@@ -786,11 +808,6 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
Column
holder
=
index
.
getColumn
(
columnName
);
if
(
holder
!=
null
)
{
if
(
holder
.
getCapabilities
().
hasMultipleValues
())
{
throw
new
UnsupportedOperationException
(
"makeObjectColumnSelector does not support multivalued columns"
);
}
final
ValueType
type
=
holder
.
getCapabilities
().
getType
();
if
(
holder
.
getCapabilities
().
isDictionaryEncoded
())
{
...
...
@@ -815,6 +832,12 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
final
GenericColumn
columnVals
=
(
GenericColumn
)
cachedColumnVals
;
final
ValueType
type
=
columnVals
.
getType
();
if
(
columnVals
.
hasMultipleValues
())
{
throw
new
UnsupportedOperationException
(
"makeObjectColumnSelector does not support multivalued GenericColumns"
);
}
if
(
type
==
ValueType
.
FLOAT
)
{
return
new
ObjectColumnSelector
<
Float
>()
{
...
...
@@ -867,20 +890,42 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
if
(
cachedColumnVals
instanceof
DictionaryEncodedColumn
)
{
final
DictionaryEncodedColumn
columnVals
=
(
DictionaryEncodedColumn
)
cachedColumnVals
;
return
new
ObjectColumnSelector
<
String
>()
{
@Override
public
Class
classOfObject
()
if
(
columnVals
.
hasMultipleValues
())
{
return
new
ObjectColumnSelector
<
String
[]>()
{
return
String
.
class
;
}
@Override
public
Class
classOfObject
()
{
return
String
[].
class
;
}
@Override
public
String
get
()
@Override
public
String
[]
get
()
{
final
IndexedInts
multiValueRow
=
columnVals
.
getMultiValueRow
(
currRow
);
final
String
[]
strings
=
new
String
[
multiValueRow
.
size
()];
for
(
int
i
=
0
;
i
<
multiValueRow
.
size
()
;
i
++)
{
strings
[
i
]
=
columnVals
.
lookupName
(
multiValueRow
.
get
(
i
));
}
return
strings
;
}
};
}
else
{
return
new
ObjectColumnSelector
<
String
>()
{
return
columnVals
.
lookupName
(
columnVals
.
getSingleValueRow
(
currRow
));
}
};
@Override
public
Class
classOfObject
()
{
return
String
.
class
;
}
@Override
public
String
get
()
{
return
columnVals
.
lookupName
(
columnVals
.
getSingleValueRow
(
currRow
));
}
};
}
}
final
ComplexColumn
columnVals
=
(
ComplexColumn
)
cachedColumnVals
;
...
...
processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java
浏览文件 @
c2137e84
...
...
@@ -403,27 +403,31 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter
if
(
dimensionIndexInt
!=
null
)
{
final
int
dimensionIndex
=
dimensionIndexInt
;
return
new
ObjectColumnSelector
<
String
>()
return
new
ObjectColumnSelector
<
Object
>()
{
@Override
public
Class
classOfObject
()
{
return
String
.
class
;
if
(
currEntry
.
getKey
().
getDims
()[
dimensionIndex
].
length
>
1
)
{
return
String
[].
class
;
}
else
{
return
String
.
class
;
}
}
@Override
public
String
get
()
public
Object
get
()
{
final
String
[]
dimVals
=
currEntry
.
getKey
().
getDims
()[
dimensionIndex
];
if
(
dimVals
.
length
==
1
)
{
return
dimVals
[
0
];
}
if
(
dimVals
.
length
==
0
)
{
else
if
(
dimVals
.
length
==
0
)
{
return
null
;
}
throw
new
UnsupportedOperationException
(
"makeObjectColumnSelector does not support multivalued columns"
);
else
{
return
dimVals
;
}
}
};
}
...
...
processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java
浏览文件 @
c2137e84
...
...
@@ -25,6 +25,7 @@ import io.druid.granularity.QueryGranularity;
import
io.druid.query.aggregation.AggregatorFactory
;
import
io.druid.query.aggregation.CountAggregatorFactory
;
import
io.druid.query.aggregation.DoubleSumAggregatorFactory
;
import
io.druid.query.aggregation.JavaScriptAggregatorFactory
;
import
io.druid.query.aggregation.LongSumAggregatorFactory
;
import
io.druid.query.aggregation.hyperloglog.HyperUniqueFinalizingPostAggregator
;
import
io.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory
;
...
...
@@ -80,6 +81,13 @@ public class QueryRunnerTestHelper
public
static
final
CountAggregatorFactory
rowsCount
=
new
CountAggregatorFactory
(
"rows"
);
public
static
final
LongSumAggregatorFactory
indexLongSum
=
new
LongSumAggregatorFactory
(
"index"
,
"index"
);
public
static
final
DoubleSumAggregatorFactory
indexDoubleSum
=
new
DoubleSumAggregatorFactory
(
"index"
,
"index"
);
public
static
final
JavaScriptAggregatorFactory
indexSumJsPlacementishN
=
new
JavaScriptAggregatorFactory
(
"nindex"
,
Arrays
.
asList
(
"placementish"
,
"index"
),
"function aggregate(current, a, b) { if ((typeof a !== 'string' && a.indexOf('a') > -1) || a === 'a') { return current + b; } else { return current; } }"
,
"function reset() { return 0; }"
,
"function combine(a, b) { return a + b; }"
);
public
static
final
HyperUniquesAggregatorFactory
qualityUniques
=
new
HyperUniquesAggregatorFactory
(
"uniques"
,
"quality_uniques"
...
...
processing/src/test/java/io/druid/query/aggregation/JavaScriptAggregatorTest.java
浏览文件 @
c2137e84
...
...
@@ -68,6 +68,12 @@ public class JavaScriptAggregatorTest
selector
.
increment
();
}
private
static
void
aggregate
(
TestObjectColumnSelector
selector
,
Aggregator
agg
)
{
agg
.
aggregate
();
selector
.
increment
();
}
@Test
public
void
testAggregate
()
{
...
...
@@ -175,6 +181,42 @@ public class JavaScriptAggregatorTest
Assert
.
assertEquals
(
val
,
agg
.
get
());
}
@Test
public
void
testAggregateStrings
()
{
final
TestObjectColumnSelector
ocs
=
new
TestObjectColumnSelector
(
"what"
,
new
String
[]{
"hey"
,
"there"
});
final
JavaScriptAggregator
agg
=
new
JavaScriptAggregator
(
"billy"
,
Collections
.<
ObjectColumnSelector
>
singletonList
(
ocs
),
JavaScriptAggregatorFactory
.
compileScript
(
"function aggregate(current, a) { if (typeof a === 'string') { return current + 1; } else { return current + a.length; } }"
,
scriptDoubleSum
.
get
(
"fnReset"
),
scriptDoubleSum
.
get
(
"fnCombine"
)
)
);
agg
.
reset
();
Assert
.
assertEquals
(
"billy"
,
agg
.
getName
());
double
val
=
0
.;
Assert
.
assertEquals
(
val
,
agg
.
get
());
Assert
.
assertEquals
(
val
,
agg
.
get
());
Assert
.
assertEquals
(
val
,
agg
.
get
());
aggregate
(
ocs
,
agg
);
val
+=
1
;
Assert
.
assertEquals
(
val
,
agg
.
get
());
Assert
.
assertEquals
(
val
,
agg
.
get
());
Assert
.
assertEquals
(
val
,
agg
.
get
());
aggregate
(
ocs
,
agg
);
val
+=
2
;
Assert
.
assertEquals
(
val
,
agg
.
get
());
Assert
.
assertEquals
(
val
,
agg
.
get
());
Assert
.
assertEquals
(
val
,
agg
.
get
());
}
public
static
void
main
(
String
...
args
)
throws
Exception
{
final
LoopingFloatColumnSelector
selector
=
new
LoopingFloatColumnSelector
(
new
float
[]{
42.12f
,
9
f
});
...
...
processing/src/test/java/io/druid/query/aggregation/TestObjectColumnSelector.java
0 → 100644
浏览文件 @
c2137e84
/*
* Druid - a distributed column store.
* Copyright (C) 2012, 2013 Metamarkets Group Inc.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package
io.druid.query.aggregation
;
import
io.druid.segment.ObjectColumnSelector
;
/**
*/
public
class
TestObjectColumnSelector
implements
ObjectColumnSelector
{
private
final
Object
[]
objects
;
private
int
index
=
0
;
public
TestObjectColumnSelector
(
Object
...
objects
)
{
this
.
objects
=
objects
;
}
@Override
public
Class
classOfObject
()
{
return
objects
[
index
].
getClass
();
}
@Override
public
Object
get
()
{
return
objects
[
index
];
}
public
void
increment
()
{
++
index
;
}
}
processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java
浏览文件 @
c2137e84
...
...
@@ -19,6 +19,7 @@
package
io.druid.query.timeseries
;
import
com.google.common.collect.ImmutableList
;
import
com.google.common.collect.ImmutableMap
;
import
com.google.common.collect.Lists
;
import
com.metamx.common.guava.Sequences
;
...
...
@@ -1262,6 +1263,43 @@ public class TimeseriesQueryRunnerTest
TestHelper
.
assertExpectedResults
(
expectedResults
,
results
);
}
@Test
public
void
testTimeseriesWithMultiValueFilteringJavascriptAggregator
()
{
TimeseriesQuery
query
=
Druids
.
newTimeseriesQueryBuilder
()
.
dataSource
(
QueryRunnerTestHelper
.
dataSource
)
.
granularity
(
QueryRunnerTestHelper
.
allGran
)
.
intervals
(
QueryRunnerTestHelper
.
firstToThird
)
.
aggregators
(
ImmutableList
.
of
(
QueryRunnerTestHelper
.
indexDoubleSum
,
QueryRunnerTestHelper
.
indexSumJsPlacementishN
)
)
.
build
();
Iterable
<
Result
<
TimeseriesResultValue
>>
expectedResults
=
ImmutableList
.
of
(
new
Result
<>(
new
DateTime
(
QueryRunnerTestHelper
.
firstToThird
.
getIntervals
()
.
get
(
0
)
.
getStart
()
),
new
TimeseriesResultValue
(
ImmutableMap
.<
String
,
Object
>
of
(
"index"
,
12459.361190795898d
,
"nindex"
,
283.31103515625d
)
)
)
);
Iterable
<
Result
<
TimeseriesResultValue
>>
actualResults
=
Sequences
.
toList
(
runner
.
run
(
query
),
Lists
.<
Result
<
TimeseriesResultValue
>>
newArrayList
()
);
TestHelper
.
assertExpectedResults
(
expectedResults
,
actualResults
);
}
@Test
public
void
testTimeseriesWithMultiValueDimFilter1
()
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录