Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
jobily
Questdb
提交
cbeff753
Q
Questdb
项目概览
jobily
/
Questdb
11 个月 前同步成功
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
Q
Questdb
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
cbeff753
编写于
9月 11, 2021
作者:
E
Eugene Lysiuchenko
提交者:
GitHub
9月 11, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
feat(sql): update "within" operator to accept GeoHash literals and constant functions (#1307)
上级
38a90c85
变更
9
隐藏空白更改
内联
并排
Showing
9 changed file
with
486 addition
and
149 deletion
+486
-149
core/src/main/java/io/questdb/cairo/ColumnType.java
core/src/main/java/io/questdb/cairo/ColumnType.java
+1
-1
core/src/main/java/io/questdb/cairo/GeoHashes.java
core/src/main/java/io/questdb/cairo/GeoHashes.java
+13
-25
core/src/main/java/io/questdb/griffin/SqlCodeGenerator.java
core/src/main/java/io/questdb/griffin/SqlCodeGenerator.java
+5
-20
core/src/main/java/io/questdb/griffin/WhereClauseParser.java
core/src/main/java/io/questdb/griffin/WhereClauseParser.java
+107
-20
core/src/main/java/io/questdb/griffin/engine/table/LatestByAllIndexedFilteredRecordCursor.java
.../engine/table/LatestByAllIndexedFilteredRecordCursor.java
+1
-3
core/src/main/java/io/questdb/griffin/engine/table/LatestByAllIndexedFilteredRecordCursorFactory.java
.../table/LatestByAllIndexedFilteredRecordCursorFactory.java
+8
-11
core/src/main/java/io/questdb/griffin/engine/table/LatestByAllIndexedRecordCursor.java
.../griffin/engine/table/LatestByAllIndexedRecordCursor.java
+11
-8
core/src/test/java/io/questdb/cairo/GeoHashesTest.java
core/src/test/java/io/questdb/cairo/GeoHashesTest.java
+17
-39
core/src/test/java/io/questdb/griffin/SqlCodeGeneratorTest.java
...rc/test/java/io/questdb/griffin/SqlCodeGeneratorTest.java
+323
-22
未找到文件。
core/src/main/java/io/questdb/cairo/ColumnType.java
浏览文件 @
cbeff753
...
...
@@ -122,7 +122,7 @@ public final class ColumnType {
}
public
static
int
getGeoHashTypeWithBits
(
int
bits
)
{
assert
bits
>
0
;
assert
bits
>
0
&&
bits
<=
GEO_HASH_MAX_BITS_LENGTH
;
// this logic relies on GeoHash type value to be clustered together
return
mkGeoHashType
(
bits
,
(
short
)
(
GEOBYTE
+
pow2SizeOfBits
(
bits
)));
}
...
...
core/src/main/java/io/questdb/cairo/GeoHashes.java
浏览文件 @
cbeff753
...
...
@@ -156,34 +156,22 @@ public class GeoHashes {
return
fromString
(
geohash
,
start
,
start
+
Math
.
min
(
length
,
MAX_STRING_LENGTH
));
}
public
static
void
fromStringToBits
(
final
CharSequenceHashSet
prefixes
,
int
columnType
,
final
DirectLongList
prefixesBits
)
{
prefixesBits
.
clear
(
);
public
static
void
addNormalizedGeoPrefix
(
long
hash
,
int
prefixType
,
int
columnType
,
final
LongList
prefixes
)
throws
NumericException
{
final
int
bits
=
ColumnType
.
getGeoHashBits
(
prefixType
);
final
int
columnSize
=
ColumnType
.
sizeOf
(
columnType
);
final
int
columnBits
=
ColumnType
.
getGeoHashBits
(
columnType
);
for
(
int
i
=
0
,
sz
=
prefixes
.
size
();
i
<
sz
;
i
++)
{
try
{
final
CharSequence
prefix
=
prefixes
.
get
(
i
);
if
(
prefix
==
null
||
prefix
.
length
()
==
0
)
{
continue
;
}
final
long
hash
=
fromString
(
prefix
,
0
,
prefix
.
length
());
final
int
bits
=
5
*
prefix
.
length
();
final
int
shift
=
columnBits
-
bits
;
long
norm
=
hash
<<
shift
;
long
mask
=
bitmask
(
bits
,
shift
);
mask
|=
1L
<<
(
columnSize
*
8
-
1
);
// set the most significant bit to ignore null from prefix matching
// if the prefix is more precise than hashes,
// exclude it from matching
if
(
bits
>
columnBits
)
{
norm
=
0L
;
mask
=
-
1L
;
}
prefixesBits
.
add
(
norm
);
prefixesBits
.
add
(
mask
);
}
catch
(
NumericException
e
)
{
// Skip invalid geo hashes
}
if
(
hash
==
NULL
||
bits
>
columnBits
)
{
throw
NumericException
.
INSTANCE
;
}
final
int
shift
=
columnBits
-
bits
;
long
norm
=
hash
<<
shift
;
long
mask
=
GeoHashes
.
bitmask
(
bits
,
shift
);
mask
|=
1L
<<
(
columnSize
*
8
-
1
);
// set the most significant bit to ignore null from prefix matching
prefixes
.
add
(
norm
);
prefixes
.
add
(
mask
);
}
public
static
long
fromStringTruncatingNl
(
CharSequence
hash
,
int
start
,
int
end
,
int
toBits
)
throws
NumericException
{
...
...
core/src/main/java/io/questdb/griffin/SqlCodeGenerator.java
浏览文件 @
cbeff753
...
...
@@ -101,7 +101,7 @@ public class SqlCodeGenerator implements Mutable {
private
final
IntList
recordFunctionPositions
=
new
IntList
();
private
final
IntList
groupByFunctionPositions
=
new
IntList
();
private
boolean
fullFatJoins
=
false
;
private
final
CharSequenceHashSet
prefixes
=
new
CharSequenceHashSe
t
();
private
final
LongList
prefixes
=
new
LongLis
t
();
static
{
joinsRequiringTimestamp
[
JOIN_INNER
]
=
false
;
...
...
@@ -941,10 +941,8 @@ public class SqlCodeGenerator implements Mutable {
Function
filter
,
SqlExecutionContext
executionContext
,
int
timestampIndex
,
int
hashColumnIndex
,
int
hashColumnType
,
@NotNull
IntList
columnIndexes
,
@NotNull
CharSequenceHashSe
t
prefixes
@NotNull
LongLis
t
prefixes
)
throws
SqlException
{
final
DataFrameCursorFactory
dataFrameCursorFactory
;
if
(
intrinsicModel
.
hasIntervalFilters
())
{
...
...
@@ -1093,8 +1091,6 @@ public class SqlCodeGenerator implements Mutable {
metadata
,
dataFrameCursorFactory
,
latestByIndex
,
hashColumnIndex
,
hashColumnType
,
filter
,
columnIndexes
,
prefixes
...
...
@@ -2440,19 +2436,12 @@ public class SqlCodeGenerator implements Mutable {
final
ExpressionNode
withinExtracted
=
whereClauseParser
.
extractWithin
(
model
,
model
.
getWhereClause
(),
myMeta
,
readerMeta
,
functionParser
,
executionContext
,
prefixes
);
int
hashColumnIndex
=
-
1
;
// latest by without prefix match part
int
hashColumnType
=
ColumnType
.
UNDEFINED
;
if
(
prefixes
.
size
()
>
1
)
{
CharSequence
column
=
prefixes
.
get
(
0
);
hashColumnIndex
=
reader
.
getMetadata
().
getColumnIndex
(
column
);
hashColumnType
=
reader
.
getMetadata
().
getColumnType
(
hashColumnIndex
);
prefixes
.
remove
(
column
);
}
model
.
setWhereClause
(
withinExtracted
);
if
(
withinExtracted
!=
null
)
{
...
...
@@ -2508,8 +2497,6 @@ public class SqlCodeGenerator implements Mutable {
f
,
executionContext
,
readerTimestampIndex
,
hashColumnIndex
,
hashColumnType
,
columnIndexes
,
prefixes
);
...
...
@@ -2767,8 +2754,6 @@ public class SqlCodeGenerator implements Mutable {
myMeta
,
new
FullBwdDataFrameCursorFactory
(
engine
,
tableName
,
model
.
getTableId
(),
model
.
getTableVersion
()),
listColumnFilterA
.
getColumnIndexFactored
(
0
),
hashColumnIndex
,
hashColumnType
,
null
,
columnIndexes
,
prefixes
...
...
core/src/main/java/io/questdb/griffin/WhereClauseParser.java
浏览文件 @
cbeff753
...
...
@@ -25,8 +25,10 @@
package
io.questdb.griffin
;
import
io.questdb.cairo.ColumnType
;
import
io.questdb.cairo.GeoHashes
;
import
io.questdb.cairo.sql.Function
;
import
io.questdb.cairo.sql.RecordMetadata
;
import
io.questdb.griffin.engine.functions.AbstractGeoHashFunction
;
import
io.questdb.griffin.model.AliasTranslator
;
import
io.questdb.griffin.model.ExpressionNode
;
import
io.questdb.griffin.model.IntervalUtils
;
...
...
@@ -1077,7 +1079,9 @@ final class WhereClauseParser implements Mutable {
AliasTranslator
translator
,
ExpressionNode
node
,
RecordMetadata
metadata
,
CharSequenceHashSet
prefixes
FunctionParser
functionParser
,
SqlExecutionContext
executionContext
,
LongList
prefixes
)
throws
SqlException
{
prefixes
.
clear
();
...
...
@@ -1086,7 +1090,7 @@ final class WhereClauseParser implements Mutable {
// pre-order iterative tree traversal
// see: http://en.wikipedia.org/wiki/Tree_traversal
if
(
removeWithin
(
translator
,
node
,
metadata
,
prefixes
))
{
if
(
removeWithin
(
translator
,
node
,
metadata
,
functionParser
,
executionContext
,
prefixes
))
{
return
collapseWithinNodes
(
node
);
}
...
...
@@ -1094,10 +1098,10 @@ final class WhereClauseParser implements Mutable {
while
(!
stack
.
isEmpty
()
||
node
!=
null
)
{
if
(
node
!=
null
)
{
if
(
isAndKeyword
(
node
.
token
)
||
isOrKeyword
(
node
.
token
))
{
if
(!
removeWithin
(
translator
,
node
.
rhs
,
metadata
,
prefixes
))
{
if
(!
removeWithin
(
translator
,
node
.
rhs
,
metadata
,
functionParser
,
executionContext
,
prefixes
))
{
stack
.
push
(
node
.
rhs
);
}
node
=
removeWithin
(
translator
,
node
.
lhs
,
metadata
,
prefixes
)
?
null
:
node
.
lhs
;
node
=
removeWithin
(
translator
,
node
.
lhs
,
metadata
,
functionParser
,
executionContext
,
prefixes
)
?
null
:
node
.
lhs
;
}
else
{
node
=
stack
.
poll
();
}
...
...
@@ -1109,11 +1113,17 @@ final class WhereClauseParser implements Mutable {
return
collapseWithinNodes
(
root
);
}
private
boolean
removeWithin
(
AliasTranslator
translator
,
ExpressionNode
node
,
RecordMetadata
metadata
,
CharSequenceHashSet
prefixes
)
throws
SqlException
{
private
boolean
removeWithin
(
AliasTranslator
translator
,
ExpressionNode
node
,
RecordMetadata
metadata
,
FunctionParser
functionParser
,
SqlExecutionContext
executionContext
,
LongList
prefixes
)
throws
SqlException
{
if
(
isWithinKeyword
(
node
.
token
))
{
if
(
prefixes
.
size
()
>
0
)
{
throw
SqlException
.
$
(
node
.
position
,
"
Using more than one 'within' operator per query is not allow
ed"
);
throw
SqlException
.
$
(
node
.
position
,
"
Multiple 'within' expressions not support
ed"
);
}
if
(
node
.
paramCount
<
2
)
{
...
...
@@ -1123,7 +1133,7 @@ final class WhereClauseParser implements Mutable {
ExpressionNode
col
=
node
.
paramCount
<
3
?
node
.
lhs
:
node
.
args
.
getLast
();
if
(
col
.
type
!=
ExpressionNode
.
LITERAL
)
{
return
false
;
throw
SqlException
.
unexpectedToken
(
col
.
position
,
col
.
token
)
;
}
CharSequence
column
=
translator
.
translateAlias
(
col
.
token
);
...
...
@@ -1132,24 +1142,28 @@ final class WhereClauseParser implements Mutable {
throw
SqlException
.
invalidColumn
(
col
.
position
,
col
.
token
);
}
final
int
hashColumnIndex
=
metadata
.
getColumnIndex
(
column
);
final
int
hashColumnType
=
metadata
.
getColumnType
(
hashColumnIndex
);
if
(!
ColumnType
.
isGeoHash
(
hashColumnType
))
{
throw
SqlException
.
$
(
node
.
position
,
"GeoHash column type expected"
);
}
if
(
prefixes
.
size
()
==
0
)
{
prefixes
.
add
(
column
);
//TODO: make a proper data struct
prefixes
.
add
(
hashColumnIndex
);
prefixes
.
add
(
hashColumnType
);
}
int
i
=
node
.
paramCount
-
1
;
i
f
(
i
==
1
)
{
if
(
node
.
rhs
==
null
||
node
.
rhs
.
type
!=
ExpressionNode
.
CONSTANT
)
{
return
false
;
}
pr
efixes
.
add
(
unquote
(
node
.
rhs
.
token
)
);
i
nt
c
=
node
.
paramCount
-
1
;
if
(
c
==
1
)
{
ExpressionNode
inArg
=
node
.
rhs
;
pr
ocessArgument
(
inArg
,
metadata
,
functionParser
,
executionContext
,
hashColumnType
,
prefixes
);
}
else
{
for
(
i
--;
i
>
-
1
;
i
--)
{
ExpressionNode
c
=
node
.
args
.
getQuick
(
i
);
if
(
c
.
type
!=
ExpressionNode
.
CONSTANT
||
isNullKeyword
(
c
.
token
))
{
return
false
;
}
prefixes
.
add
(
unquote
(
c
.
token
));
for
(
c
--;
c
>
-
1
;
c
--)
{
ExpressionNode
inArg
=
node
.
args
.
getQuick
(
c
);
processArgument
(
inArg
,
metadata
,
functionParser
,
executionContext
,
hashColumnType
,
prefixes
);
}
}
return
true
;
...
...
@@ -1158,6 +1172,79 @@ final class WhereClauseParser implements Mutable {
}
}
private
void
processArgument
(
ExpressionNode
inArg
,
RecordMetadata
metadata
,
FunctionParser
functionParser
,
SqlExecutionContext
executionContext
,
int
columnType
,
LongList
prefixes
)
throws
SqlException
{
final
int
position
=
inArg
.
position
;
if
(
isNull
(
inArg
))
{
throw
SqlException
.
$
(
position
,
"GeoHash value expected"
);
}
final
int
type
;
final
long
hash
;
if
(
isFunc
(
inArg
))
{
try
(
Function
f
=
functionParser
.
parseFunction
(
inArg
,
metadata
,
executionContext
))
{
if
(
isGeoHashConstFunction
(
f
))
{
type
=
f
.
getType
();
hash
=
GeoHashes
.
getGeoLong
(
type
,
f
,
null
);
}
else
{
throw
SqlException
.
$
(
inArg
.
position
,
"GeoHash const function expected"
);
}
}
}
else
{
final
boolean
isConstant
=
inArg
.
type
==
ExpressionNode
.
CONSTANT
;
final
CharSequence
token
=
inArg
.
token
;
final
int
len
=
token
.
length
();
final
boolean
isBitsPrefix
=
len
>
2
&&
token
.
charAt
(
0
)
==
'#'
&&
token
.
charAt
(
1
)
==
'#'
;
final
boolean
isCharsPrefix
=
len
>
1
&&
token
.
charAt
(
0
)
==
'#'
;
if
(!(
isConstant
&&
(
isBitsPrefix
||
isCharsPrefix
)))
{
throw
SqlException
.
$
(
position
,
"GeoHash literal expected"
);
}
try
{
if
(!
isBitsPrefix
)
{
final
int
sdd
=
ExpressionParser
.
extractGeoHashSuffix
(
position
,
token
);
final
int
sddLen
=
Numbers
.
decodeLowShort
(
sdd
);
final
int
bits
=
Numbers
.
decodeHighShort
(
sdd
);
type
=
ColumnType
.
getGeoHashTypeWithBits
(
bits
);
hash
=
GeoHashes
.
fromStringTruncatingNl
(
token
,
1
,
len
-
sddLen
,
bits
);
}
else
{
int
bits
=
len
-
2
;
if
(
bits
<=
ColumnType
.
GEO_HASH_MAX_BITS_LENGTH
)
{
type
=
ColumnType
.
getGeoHashTypeWithBits
(
bits
);
hash
=
GeoHashes
.
fromBitStringNl
(
token
,
2
);
}
else
{
throw
SqlException
.
$
(
position
,
"GeoHash bits literal expected"
);
}
}
}
catch
(
NumericException
ignored
)
{
throw
SqlException
.
$
(
position
,
"GeoHash literal expected"
);
}
}
try
{
GeoHashes
.
addNormalizedGeoPrefix
(
hash
,
type
,
columnType
,
prefixes
);
}
catch
(
NumericException
e
)
{
throw
SqlException
.
$
(
position
,
"GeoHash prefix precision mismatch"
);
}
}
private
boolean
isNull
(
ExpressionNode
node
)
{
return
node
==
null
||
isNullKeyword
(
node
.
token
);
}
private
boolean
isGeoHashConstFunction
(
Function
fn
)
{
return
(
fn
instanceof
AbstractGeoHashFunction
)
&&
fn
.
isConstant
();
}
private
ExpressionNode
collapseWithinNodes
(
ExpressionNode
node
)
{
if
(
node
==
null
||
isWithinKeyword
(
node
.
token
))
{
return
null
;
...
...
core/src/main/java/io/questdb/griffin/engine/table/LatestByAllIndexedFilteredRecordCursor.java
浏览文件 @
cbeff753
...
...
@@ -38,14 +38,12 @@ class LatestByAllIndexedFilteredRecordCursor extends LatestByAllIndexedRecordCur
public
LatestByAllIndexedFilteredRecordCursor
(
int
columnIndex
,
int
hashColumnIndex
,
int
hashColumnType
,
@NotNull
DirectLongList
rows
,
@NotNull
Function
filter
,
@NotNull
IntList
columnIndexes
,
@NotNull
DirectLongList
prefixes
)
{
super
(
columnIndex
,
hashColumnIndex
,
hashColumnType
,
rows
,
columnIndexes
,
prefixes
);
super
(
columnIndex
,
rows
,
columnIndexes
,
prefixes
);
this
.
filter
=
filter
;
}
...
...
core/src/main/java/io/questdb/griffin/engine/table/LatestByAllIndexedFilteredRecordCursorFactory.java
浏览文件 @
cbeff753
...
...
@@ -25,14 +25,12 @@
package
io.questdb.griffin.engine.table
;
import
io.questdb.cairo.CairoConfiguration
;
import
io.questdb.cairo.ColumnType
;
import
io.questdb.cairo.GeoHashes
;
import
io.questdb.cairo.sql.DataFrameCursorFactory
;
import
io.questdb.cairo.sql.Function
;
import
io.questdb.cairo.sql.RecordMetadata
;
import
io.questdb.std.CharSequenceHashSet
;
import
io.questdb.std.DirectLongList
;
import
io.questdb.std.IntList
;
import
io.questdb.std.LongList
;
import
org.jetbrains.annotations.NotNull
;
import
org.jetbrains.annotations.Nullable
;
...
...
@@ -44,23 +42,22 @@ public class LatestByAllIndexedFilteredRecordCursorFactory extends AbstractTreeS
@NotNull
RecordMetadata
metadata
,
@NotNull
DataFrameCursorFactory
dataFrameCursorFactory
,
int
columnIndex
,
int
hashColumnIndex
,
int
hashColumnType
,
@Nullable
Function
filter
,
@NotNull
IntList
columnIndexes
,
@NotNull
CharSequenceHashSe
t
prefixes
@NotNull
LongLis
t
prefixes
)
{
super
(
metadata
,
dataFrameCursorFactory
,
configuration
);
this
.
prefixes
=
new
DirectLongList
(
Math
.
max
(
2
,
prefixes
.
size
()));
this
.
prefixes
=
new
DirectLongList
(
64
);
if
(
hashColumnIndex
>
-
1
&&
ColumnType
.
isGeoHash
(
hashColumnType
)
)
{
GeoHashes
.
fromStringToBits
(
prefixes
,
hashColumnType
,
this
.
prefixes
);
// copy into owned direct memory
for
(
int
i
=
0
;
i
<
prefixes
.
size
();
i
++
)
{
this
.
prefixes
.
add
(
prefixes
.
get
(
i
)
);
}
if
(
filter
==
null
)
{
this
.
cursor
=
new
LatestByAllIndexedRecordCursor
(
columnIndex
,
hashColumnIndex
,
hashColumnType
,
rows
,
columnIndexes
,
this
.
prefixes
);
this
.
cursor
=
new
LatestByAllIndexedRecordCursor
(
columnIndex
,
rows
,
columnIndexes
,
this
.
prefixes
);
}
else
{
this
.
cursor
=
new
LatestByAllIndexedFilteredRecordCursor
(
columnIndex
,
hashColumnIndex
,
hashColumnType
,
rows
,
filter
,
columnIndexes
,
this
.
prefixes
);
this
.
cursor
=
new
LatestByAllIndexedFilteredRecordCursor
(
columnIndex
,
rows
,
filter
,
columnIndexes
,
this
.
prefixes
);
}
}
...
...
core/src/main/java/io/questdb/griffin/engine/table/LatestByAllIndexedRecordCursor.java
浏览文件 @
cbeff753
...
...
@@ -46,8 +46,6 @@ import org.jetbrains.annotations.NotNull;
class
LatestByAllIndexedRecordCursor
extends
AbstractRecordListCursor
{
protected
final
DirectLongList
prefixes
;
private
final
int
columnIndex
;
private
final
int
hashColumnIndex
;
private
final
int
hashColumnType
;
private
final
SOUnboundedCountDownLatch
doneLatch
=
new
SOUnboundedCountDownLatch
();
protected
long
indexShift
=
0
;
protected
long
aIndex
;
...
...
@@ -55,16 +53,12 @@ class LatestByAllIndexedRecordCursor extends AbstractRecordListCursor {
public
LatestByAllIndexedRecordCursor
(
int
columnIndex
,
int
hashColumnIndex
,
int
hashColumnType
,
@NotNull
DirectLongList
rows
,
@NotNull
IntList
columnIndexes
,
@NotNull
DirectLongList
prefixes
)
{
super
(
rows
,
columnIndexes
);
this
.
columnIndex
=
columnIndex
;
this
.
hashColumnIndex
=
hashColumnIndex
;
this
.
hashColumnType
=
hashColumnType
;
this
.
prefixes
=
prefixes
;
}
...
...
@@ -118,9 +112,18 @@ class LatestByAllIndexedRecordCursor extends AbstractRecordListCursor {
LatestByArguments
.
setRowsSize
(
argsAddress
,
0
);
}
final
long
prefixesAddress
=
prefixes
.
getAddress
()
;
final
long
prefixesCount
=
prefixes
.
size
()
;
int
hashColumnIndex
=
-
1
;
int
hashColumnType
=
ColumnType
.
UNDEFINED
;
long
prefixesAddress
=
0
;
long
prefixesCount
=
0
;
if
(
this
.
prefixes
.
size
()
>
2
)
{
hashColumnIndex
=
(
int
)
prefixes
.
get
(
0
);
hashColumnType
=
(
int
)
prefixes
.
get
(
1
);
prefixesAddress
=
prefixes
.
getAddress
()
+
2
*
Long
.
BYTES
;
prefixesCount
=
prefixes
.
size
()
-
2
;
}
DataFrame
frame
;
// frame metadata is based on TableReader, which is "full" metadata
...
...
core/src/test/java/io/questdb/cairo/GeoHashesTest.java
浏览文件 @
cbeff753
...
...
@@ -481,19 +481,17 @@ public class GeoHashesTest {
}
@Test
public
void
test
FromStringToBit
s
()
throws
NumericException
{
public
void
test
BuildNormalizedPrefixesAndMask
s
()
throws
NumericException
{
final
int
cap
=
12
;
DirectLongList
bits
=
new
DirectLongList
(
cap
*
2
);
// hash and mask
CharSequenceHashSet
strh
=
new
CharSequenceHashSet
();
StringSink
sink
=
Misc
.
getThreadLocalBuilder
();
LongList
bits
=
new
LongList
(
cap
*
2
);
// hash and mask
int
columnType
=
ColumnType
.
getGeoHashTypeWithBits
(
5
*
cap
);
for
(
int
i
=
0
;
i
<
cap
;
i
++)
{
final
int
prec
=
(
i
%
3
)
+
3
;
final
long
h
=
rnd_geohash
(
prec
);
sink
.
clear
();
GeoHashes
.
appendChars
(
h
,
prec
,
sink
);
strh
.
add
(
sink
);
int
type
=
ColumnType
.
getGeoHashTypeWithBits
(
5
*
prec
);
GeoHashes
.
addNormalizedGeoPrefix
(
h
,
type
,
columnType
,
bits
);
}
GeoHashes
.
fromStringToBits
(
strh
,
ColumnType
.
getGeoHashTypeWithBits
(
cap
*
5
),
bits
);
for
(
int
i
=
0
;
i
<
bits
.
size
()
/
2
;
i
+=
2
)
{
final
long
b
=
bits
.
get
(
i
);
final
long
m
=
bits
.
get
(
i
+
1
);
...
...
@@ -502,41 +500,21 @@ public class GeoHashesTest {
}
@Test
public
void
testFromStringToBitsInvalidNull
()
{
final
int
cap
=
12
;
DirectLongList
bits
=
new
DirectLongList
(
cap
*
2
);
// hash and mask
CharSequenceHashSet
strh
=
new
CharSequenceHashSet
();
strh
.
add
(
""
);
strh
.
add
(
null
);
strh
.
add
(
"$invalid"
);
strh
.
add
(
"questdb.10"
);
GeoHashes
.
fromStringToBits
(
strh
,
ColumnType
.
getGeoHashTypeWithBits
(
cap
*
5
),
bits
);
Assert
.
assertEquals
(
0
,
bits
.
size
());
}
public
void
testPrefixPrecisionMismatch
()
throws
NumericException
{
final
int
cap
=
1
;
LongList
bits
=
new
LongList
(
cap
*
2
);
// hash and mask
final
long
h
=
rnd_geohash
(
5
);
final
long
p
=
rnd_geohash
(
7
);
@Test
public
void
testFromStringToBitsInvalidStrings
()
{
final
int
cap
=
12
;
DirectLongList
bits
=
new
DirectLongList
(
cap
*
2
);
// hash and mask
CharSequenceHashSet
strh
=
new
CharSequenceHashSet
();
strh
.
add
(
""
);
strh
.
add
(
"a medium sized banana"
);
GeoHashes
.
fromStringToBits
(
strh
,
ColumnType
.
getGeoHashTypeWithBits
(
cap
*
5
),
bits
);
int
pType
=
ColumnType
.
getGeoHashTypeWithBits
(
5
*
7
);
int
hType
=
ColumnType
.
getGeoHashTypeWithBits
(
5
*
5
);
try
{
GeoHashes
.
addNormalizedGeoPrefix
(
h
,
pType
,
hType
,
bits
);
}
catch
(
NumericException
ignored
)
{
}
Assert
.
assertEquals
(
0
,
bits
.
size
());
}
@Test
public
void
testFromStringToBitsSingle
()
{
final
int
cap
=
12
;
DirectLongList
bits
=
new
DirectLongList
(
cap
*
2
);
// hash and mask
CharSequenceHashSet
strh
=
new
CharSequenceHashSet
();
strh
.
add
(
"questdb"
);
GeoHashes
.
fromStringToBits
(
strh
,
ColumnType
.
getGeoHashTypeWithBits
(
cap
*
5
),
bits
);
Assert
.
assertEquals
(
2
,
bits
.
size
());
}
@Test
public
void
testFromBitStringTruncating
()
throws
NumericException
{
CharSequence
tooLongBitString
=
Chars
.
repeat
(
"1"
,
61
);
// truncates
...
...
core/src/test/java/io/questdb/griffin/SqlCodeGeneratorTest.java
浏览文件 @
cbeff753
...
...
@@ -1956,7 +1956,7 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest {
"2021-05-10T23:59:59.150000Z\tXXX\tf\n"
+
"2021-05-11T00:00:00.083000Z\tYYY\tz\n"
+
"2021-05-12T00:00:00.186000Z\tZZZ\tv\n"
,
"select * from pos latest by uuid where hash within(
'f', 'z', 'v'
)"
,
"select * from pos latest by uuid where hash within(
#f, #z, #v
)"
,
"time"
,
true
,
true
,
...
...
@@ -1974,7 +1974,7 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest {
"2021-05-10T23:59:59.150000Z\tXXX\tf9\n"
+
"2021-05-11T00:00:00.083000Z\tYYY\tz3\n"
+
"2021-05-12T00:00:00.186000Z\tZZZ\tve\n"
,
"select * from pos latest by uuid where hash within(
'f9', 'z3', 've'
)"
,
"select * from pos latest by uuid where hash within(
#f9, #z3, #ve
)"
,
"time"
,
true
,
true
,
...
...
@@ -1983,6 +1983,117 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest {
});
}
@Test
public
void
testLatestByAllIndexedGeoHash2cFn
()
throws
Exception
{
assertMemoryLeak
(
()
->
{
createGeoHashTable
(
2
);
assertQuery
(
"time\tuuid\thash\n"
+
"2021-05-10T23:59:59.150000Z\tXXX\tf9\n"
+
"2021-05-11T00:00:00.083000Z\tYYY\tz3\n"
+
"2021-05-12T00:00:00.186000Z\tZZZ\tve\n"
,
"select * from pos latest by uuid where hash within(make_geohash(-62, 53.4, 10), #z3, #ve)"
,
"time"
,
true
,
true
,
true
);
});
}
@Test
public
void
testLatestByAllIndexedGeoHashOutOfRangeFn
()
throws
Exception
{
assertMemoryLeak
(
()
->
{
createGeoHashTable
(
2
);
try
{
assertQuery
(
"time\tuuid\thash\n"
+
"2021-05-10T23:59:59.150000Z\tXXX\tf9\n"
+
"2021-05-11T00:00:00.083000Z\tYYY\tz3\n"
+
"2021-05-12T00:00:00.186000Z\tZZZ\tve\n"
,
"select * from pos latest by uuid where hash within(make_geohash(-620.0, 53.4, 10), #z3, #ve)"
,
"time"
,
true
,
true
,
true
);
}
catch
(
SqlException
ex
)
{
TestUtils
.
assertContains
(
ex
.
getFlyweightMessage
(),
"longitude must be in [-180.0..180.0] range"
);
}
});
}
@Test
public
void
testLatestByAllIndexedGeoHashStrCast
()
throws
Exception
{
assertMemoryLeak
(
()
->
{
createGeoHashTable
(
2
);
assertQuery
(
"time\tuuid\thash\n"
+
"2021-05-10T23:59:59.150000Z\tXXX\tf9\n"
+
"2021-05-11T00:00:00.083000Z\tYYY\tz3\n"
+
"2021-05-12T00:00:00.186000Z\tZZZ\tve\n"
,
"select * from pos latest by uuid where hash within(cast('f9' as geohash(2c)), #z3, #ve)"
,
"time"
,
true
,
true
,
true
);
});
}
@Test
public
void
testLatestByAllIndexedGeoHashLiteralExpected
()
throws
Exception
{
assertMemoryLeak
(
()
->
{
createGeoHashTable
(
2
);
try
{
assertQuery
(
"time\tuuid\thash\n"
+
"2021-05-10T23:59:59.150000Z\tXXX\tf9\n"
+
"2021-05-11T00:00:00.083000Z\tYYY\tz3\n"
+
"2021-05-12T00:00:00.186000Z\tZZZ\tve\n"
,
"select * from pos latest by uuid where hash within('z3', #z3, #ve)"
,
"time"
,
true
,
true
,
true
);
}
catch
(
SqlException
ex
)
{
TestUtils
.
assertContains
(
ex
.
getFlyweightMessage
(),
"GeoHash literal expected"
);
}
});
}
@Test
public
void
testLatestByAllIndexedGeoHashFnNonConst
()
throws
Exception
{
assertMemoryLeak
(
()
->
{
compiler
.
compile
(
"create table x as ("
+
"select"
+
" rnd_symbol(113, 4, 4, 2) s,"
+
" timestamp_sequence(500000000000L,100000000L) ts,"
+
" (rnd_double()*360.0 - 180.0) lon, "
+
" (rnd_double()*180.0 - 90.0) lat, "
+
" rnd_geohash(40) geo8"
+
" from long_sequence(1000)"
+
"), index(s) timestamp (ts) partition by DAY"
,
sqlExecutionContext
);
try
{
assertQuery
(
"time\tuuid\thash\n"
,
"select * from x latest by s where geo8 within(make_geohash(lon, lat, 40), #z3, #vegg)"
,
"ts"
,
true
,
true
,
true
);
}
catch
(
SqlException
ex
)
{
TestUtils
.
assertContains
(
ex
.
getFlyweightMessage
(),
"GeoHash const function expected"
);
}
});
}
@Test
public
void
testLatestByAllIndexedGeoHash4c
()
throws
Exception
{
assertMemoryLeak
(
...
...
@@ -1992,7 +2103,7 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest {
"2021-05-10T23:59:59.150000Z\tXXX\tf91t\n"
+
"2021-05-11T00:00:00.083000Z\tYYY\tz31w\n"
+
"2021-05-12T00:00:00.186000Z\tZZZ\tvepe\n"
,
"select * from pos latest by uuid where hash within(
'f91', 'z31w', 'vepe'
)"
,
"select * from pos latest by uuid where hash within(
#f91, #z31w, #vepe
)"
,
"time"
,
true
,
true
,
...
...
@@ -2010,7 +2121,7 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest {
"2021-05-10T23:59:59.150000Z\tXXX\tf91t48s7\n"
+
"2021-05-11T00:00:00.083000Z\tYYY\tz31wzd5w\n"
+
"2021-05-12T00:00:00.186000Z\tZZZ\tvepe7h62\n"
,
"select * from pos latest by uuid where hash within(
'f91', 'z31w', 'vepe7h'
)"
,
"select * from pos latest by uuid where hash within(
#f91, #z31w, #vepe7h
)"
,
"time"
,
true
,
true
,
...
...
@@ -2024,15 +2135,131 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest {
assertMemoryLeak
(
()
->
{
createGeoHashTable
(
2
);
assertQuery
(
"time\tuuid\thash\n"
+
"2021-05-10T23:59:59.150000Z\tXXX\tf9\n"
+
"2021-05-11T00:00:00.083000Z\tYYY\tz3\n"
,
"select * from pos latest by uuid where hash within('f9', 'z3', 'vepe7h')"
,
"time"
,
true
,
true
,
true
);
try
{
assertQuery
(
""
,
"select * from pos latest by uuid where hash within(#f9, #z3, #vepe7h)"
,
"time"
,
true
,
true
,
true
);
}
catch
(
SqlException
ex
)
{
TestUtils
.
assertContains
(
ex
.
getFlyweightMessage
(),
"prefix precision mismatch"
);
}
});
}
@Test
public
void
testLatestByAllIndexedGeoHashWithinEmpty
()
throws
Exception
{
assertMemoryLeak
(
()
->
{
createGeoHashTable
(
2
);
try
{
assertQuery
(
""
,
"select * from pos latest by uuid where hash within()"
,
"time"
,
true
,
true
,
true
);
}
catch
(
SqlException
ex
)
{
TestUtils
.
assertContains
(
ex
.
getFlyweightMessage
(),
"Too few arguments for 'within'"
);
}
});
}
@Test
public
void
testLatestByAllIndexedGeoHashWithinOr
()
throws
Exception
{
assertMemoryLeak
(
()
->
{
createGeoHashTable
(
2
);
try
{
assertQuery
(
""
,
"select * from pos latest by uuid where hash within(#f9) or hash within(#z3)"
,
"time"
,
true
,
true
,
true
);
}
catch
(
SqlException
ex
)
{
TestUtils
.
assertContains
(
ex
.
getFlyweightMessage
(),
"Multiple 'within' expressions not supported"
);
}
});
}
@Test
public
void
testLatestByAllIndexedGeoHashWithinColumnWrongType
()
throws
Exception
{
assertMemoryLeak
(
()
->
{
createGeoHashTable
(
2
);
try
{
assertQuery
(
""
,
"select * from pos latest by uuid where uuid within(#f9)"
,
"time"
,
true
,
true
,
true
);
}
catch
(
SqlException
ex
)
{
TestUtils
.
assertContains
(
ex
.
getFlyweightMessage
(),
"GeoHash column type expected"
);
}
});
}
@Test
public
void
testLatestByAllIndexedGeoHashWithinColumnNotLiteral
()
throws
Exception
{
assertMemoryLeak
(
()
->
{
createGeoHashTable
(
2
);
try
{
assertQuery
(
""
,
"select * from pos latest by uuid where 'hash' within(#f9)"
,
"time"
,
true
,
true
,
true
);
}
catch
(
SqlException
ex
)
{
TestUtils
.
assertContains
(
ex
.
getFlyweightMessage
(),
"unexpected token:"
);
}
});
}
@Test
public
void
testLatestByAllIndexedGeoHashWithinNullArg
()
throws
Exception
{
assertMemoryLeak
(
()
->
{
createGeoHashTable
(
2
);
try
{
assertQuery
(
""
,
"select * from pos latest by uuid where hash within(#f9, #z3, null)"
,
"time"
,
true
,
true
,
true
);
}
catch
(
SqlException
ex
)
{
TestUtils
.
assertContains
(
ex
.
getFlyweightMessage
(),
"GeoHash value expected"
);
}
});
}
@Test
public
void
testLatestByAllIndexedGeoHashWithinWrongCast
()
throws
Exception
{
assertMemoryLeak
(
()
->
{
createGeoHashTable
(
2
);
try
{
assertQuery
(
""
,
"select * from pos latest by uuid where hash within(cast('f91t' as geohash(4c)), #z3, null)"
,
"time"
,
true
,
true
,
true
);
}
catch
(
SqlException
ex
)
{
TestUtils
.
assertContains
(
ex
.
getFlyweightMessage
(),
"prefix precision mismatch"
);
}
});
}
...
...
@@ -2077,7 +2304,7 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest {
"time\tuuid\thash\n"
+
"2021-05-11T00:00:00.083000Z\tYYY\tz\n"
+
"2021-05-11T00:00:00.111000Z\tddd\tb\n"
,
"select * from pos latest by uuid where time in '2021-05-11' and hash within (
'z','b'
)"
,
"select * from pos latest by uuid where time in '2021-05-11' and hash within (
#z, #b
)"
,
"time"
,
true
,
true
,
...
...
@@ -2095,7 +2322,7 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest {
"time\tuuid\thash\n"
+
"2021-05-11T00:00:00.083000Z\tYYY\tz3\n"
+
"2021-05-11T00:00:00.111000Z\tddd\tbc\n"
,
"select * from pos latest by uuid where time in '2021-05-11' and hash within (
'z','b'
)"
,
"select * from pos latest by uuid where time in '2021-05-11' and hash within (
#z, #b
)"
,
"time"
,
true
,
true
,
...
...
@@ -2113,7 +2340,7 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest {
"time\tuuid\thash\n"
+
"2021-05-11T00:00:00.083000Z\tYYY\tz31w\n"
+
"2021-05-11T00:00:00.111000Z\tddd\tbcnk\n"
,
"select * from pos latest by uuid where time in '2021-05-11' and hash within (
'z','b'
)"
,
"select * from pos latest by uuid where time in '2021-05-11' and hash within (
#z, #b
)"
,
"time"
,
true
,
true
,
...
...
@@ -2130,7 +2357,7 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest {
assertQuery
(
"time\tuuid\thash\n"
+
"2021-05-11T00:00:00.083000Z\tYYY\tz31wzd5w\n"
,
"select * from pos latest by uuid where time in '2021-05-11' and hash within (
'z31','bbx'
)"
,
"select * from pos latest by uuid where time in '2021-05-11' and hash within (
#z31, #bbx
)"
,
"time"
,
true
,
true
,
...
...
@@ -2139,6 +2366,65 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest {
});
}
@Test
public
void
testLatestByAllIndexedGeoHashRnd6Bits
()
throws
Exception
{
assertMemoryLeak
(
()
->
{
createRndGeoHashBitsTable
();
assertQuery
(
"bits7\tts\n"
+
"1111111\t1970-01-16T21:43:20.000000Z\n"
+
"1111111\t1970-01-18T00:50:00.000000Z\n"
+
"1111111\t1970-01-18T00:55:00.000000Z\n"
+
"1111110\t1970-01-18T05:11:40.000000Z\n"
+
"1111110\t1970-01-18T07:10:00.000000Z\n"
+
"1111110\t1970-01-18T08:20:00.000000Z\n"
+
"1111111\t1970-01-18T08:28:20.000000Z\n"
,
"select bits7, ts from x latest by s where bits7 within(##111111)"
,
"ts"
,
true
,
true
,
true
);
});
}
@Test
public
void
testLatestByAllIndexedGeoHashRndLongBitsPrefix
()
throws
Exception
{
assertMemoryLeak
(
()
->
{
createRndGeoHashBitsTable
();
try
{
assertQuery
(
""
,
"select * from x latest by s where bits3 within(##111111)"
,
"ts"
,
true
,
true
,
true
);
}
catch
(
SqlException
ex
)
{
TestUtils
.
assertContains
(
ex
.
getFlyweightMessage
(),
"prefix precision mismatch"
);
}
});
}
@Test
public
void
testLatestByAllIndexedGeoHashRndLongBitsMask
()
throws
Exception
{
assertMemoryLeak
(
()
->
{
createRndGeoHashBitsTable
();
assertQuery
(
"i\ts\tts\tbits3\tbits7\tbits9\n"
+
"9384\tYFFD\t1970-01-17T15:31:40.000000Z\t101\t1110000\t101111011\n"
+
"9397\tMXUK\t1970-01-17T15:53:20.000000Z\t100\t1110001\t110001111\n"
,
"select * from x latest by s where bits7 within(#wt/5)"
,
//(##11100)",
"ts"
,
true
,
true
,
true
);
});
}
@Test
public
void
testLatestByAllIndexedGeoHashRnd1c
()
throws
Exception
{
assertMemoryLeak
(
...
...
@@ -2154,7 +2440,7 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest {
"y\t1970-01-18T07:41:40.000000Z\n"
+
"y\t1970-01-18T08:18:20.000000Z\n"
+
"z\t1970-01-18T08:35:00.000000Z\n"
,
"select geo1, ts from x latest by s where geo1 within(
'x', 'y', 'z'
)"
,
"select geo1, ts from x latest by s where geo1 within(
#x, #y, #z
)"
,
"ts"
,
true
,
true
,
...
...
@@ -2181,7 +2467,7 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest {
"yvh\t1970-01-18T06:55:00.000000Z\n"
+
"y1n\t1970-01-18T07:28:20.000000Z\n"
+
"zs4\t1970-01-18T08:03:20.000000Z\n"
,
"select geo2, ts from x latest by s where geo2 within(
'x', 'y', 'z'
)"
,
"select geo2, ts from x latest by s where geo2 within(
#x, #y, #z
)"
,
"ts"
,
true
,
true
,
...
...
@@ -2208,7 +2494,7 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest {
"xxusm\t1970-01-18T07:55:00.000000Z\n"
+
"x1dse\t1970-01-18T08:18:20.000000Z\n"
+
"zmt6j\t1970-01-18T08:38:20.000000Z\n"
,
"select geo4, ts from x latest by s where geo4 within(
'x', 'y', 'z'
)"
,
"select geo4, ts from x latest by s where geo4 within(
#x, #y, #z
)"
,
"ts"
,
true
,
true
,
...
...
@@ -2229,7 +2515,7 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest {
"ybsge\t1970-01-18T06:45:00.000000Z\n"
+
"z4t7w\t1970-01-18T07:45:00.000000Z\n"
+
"xxusm\t1970-01-18T07:55:00.000000Z\n"
,
"select geo4, ts from x latest by s where geo4 within(
'xx', 'y', 'z4'
)"
,
"select geo4, ts from x latest by s where geo4 within(
#xx, #y, #z4
)"
,
"ts"
,
true
,
true
,
...
...
@@ -2238,6 +2524,21 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest {
});
}
private
void
createRndGeoHashBitsTable
()
throws
SqlException
{
compiler
.
compile
(
"create table x as ("
+
"select"
+
" cast(x as int) i,"
+
" rnd_symbol(113, 4, 4, 2) s,"
+
" timestamp_sequence(500000000000L,100000000L) ts,"
+
" rnd_geohash(3) bits3,"
+
" rnd_geohash(7) bits7,"
+
" rnd_geohash(9) bits9"
+
" from long_sequence(10000)"
+
"), index(s) timestamp (ts) partition by DAY"
,
sqlExecutionContext
);
}
private
void
createRndGeoHashTable
()
throws
SqlException
{
compiler
.
compile
(
"create table x as ("
+
...
...
@@ -2258,7 +2559,7 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest {
private
void
createGeoHashTable
(
int
chars
)
throws
SqlException
{
compiler
.
compile
(
String
.
format
(
"create table pos(time timestamp, uuid symbol, hash geohash(%dc))"
,
chars
)
+
", index(uuid) timestamp(time) partition by DAY"
,
", index(uuid) timestamp(time) partition by DAY"
,
sqlExecutionContext
);
executeInsert
(
"insert into pos values('2021-05-10T23:59:59.150000Z','XXX','f91t48s7')"
);
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录