Skip to content

Q
Questdb

jobily / Questdb
大约 1 年 前同步成功

通知 1
Star 0
Fork 0
Q
Questdb

体验新版 GitCode，发现更多精彩内容 >>

未验证 提交 1c8acc11 编写于 作者: V Vlad Ilyushchenko 提交者： GitHub
浏览文件
操作

fix(griffin): fixed timestamp propagation issue for sampleBy. Fixed #749 (#758)

上级 7d713fa1
隐藏空白更改
内联 并排
Showing with 208 addition and 69 deletion
core/src/main/java/io/questdb/griffin/FunctionParser.java
浏览文件 @ 1c8acc11
......@@ -67,6 +67,48 @@ public class FunctionParser implements PostOrderTreeTraversalAlgo.Visitor {
this.functionFactoryCache = functionFactoryCache;
}
@NotNull
public static ScalarFunction createColumn(int position, CharSequence name, RecordMetadata metadata) throws SqlException {
final int index = metadata.getColumnIndexQuiet(name);
if (index == -1) {
throw SqlException.invalidColumn(position, name);
}
switch (metadata.getColumnType(index)) {
case ColumnType.BOOLEAN:
return new BooleanColumn(position, index);
case ColumnType.BYTE:
return new ByteColumn(position, index);
case ColumnType.SHORT:
return new ShortColumn(position, index);
case ColumnType.CHAR:
return new CharColumn(position, index);
case ColumnType.INT:
return new IntColumn(position, index);
case ColumnType.LONG:
return new LongColumn(position, index);
case ColumnType.FLOAT:
return new FloatColumn(position, index);
case ColumnType.DOUBLE:
return new DoubleColumn(position, index);
case ColumnType.STRING:
return new StrColumn(position, index);
case ColumnType.SYMBOL:
return new SymbolColumn(position, index, metadata.isSymbolTableStatic(index));
case ColumnType.BINARY:
return new BinColumn(position, index);
case ColumnType.DATE:
return new DateColumn(position, index);
case ColumnType.TIMESTAMP:
return new TimestampColumn(position, index);
case ColumnType.RECORD:
return new RecordColumn(position, index, metadata.getMetadata(index));
default:
return new Long256Column(position, index);
}
}
public Function createIndexParameter(int variableIndex, ExpressionNode node) throws SqlException {
Function function = getBindVariableService().getFunction(variableIndex);
if (function == null) {
......@@ -100,6 +142,22 @@ public class FunctionParser implements PostOrderTreeTraversalAlgo.Visitor {
return functionFactoryCache.isGroupBy(token);
}
public boolean isValidNoArgFunction(ExpressionNode node) {
final ObjList<FunctionFactoryDescriptor> overload = functionFactoryCache.getOverloadList(node.token);
if (overload == null) {
return false;
}
for (int i = 0, n = overload.size(); i < n; i++) {
FunctionFactoryDescriptor ffd = overload.getQuick(i);
if (ffd.getSigArgCount() == 0) {
return true;
}
}
return false;
}
/**
* Creates function instance. When node type is {@link ExpressionNode#LITERAL} a column or parameter
* function is returned. We will be using the supplied {@link #metadata} to resolve type of column. When node token
......@@ -301,44 +359,7 @@ public class FunctionParser implements PostOrderTreeTraversalAlgo.Visitor {
}
private Function createColumn(ExpressionNode node) throws SqlException {
final int index = metadata.getColumnIndexQuiet(node.token);
if (index == -1) {
throw SqlException.invalidColumn(node.position, node.token);
}
switch (metadata.getColumnType(index)) {
case ColumnType.BOOLEAN:
return new BooleanColumn(node.position, index);
case ColumnType.BYTE:
return new ByteColumn(node.position, index);
case ColumnType.SHORT:
return new ShortColumn(node.position, index);
case ColumnType.CHAR:
return new CharColumn(node.position, index);
case ColumnType.INT:
return new IntColumn(node.position, index);
case ColumnType.LONG:
return new LongColumn(node.position, index);
case ColumnType.FLOAT:
return new FloatColumn(node.position, index);
case ColumnType.DOUBLE:
return new DoubleColumn(node.position, index);
case ColumnType.STRING:
return new StrColumn(node.position, index);
case ColumnType.SYMBOL:
return new SymbolColumn(node.position, index, metadata.isSymbolTableStatic(index));
case ColumnType.BINARY:
return new BinColumn(node.position, index);
case ColumnType.DATE:
return new DateColumn(node.position, index);
case ColumnType.TIMESTAMP:
return new TimestampColumn(node.position, index);
case ColumnType.RECORD:
return new RecordColumn(node.position, index, metadata.getMetadata(index));
default:
return new Long256Column(node.position, index);
}
return createColumn(node.position, node.token, metadata);
}
private Function createConstant(ExpressionNode node) throws SqlException {
......@@ -725,22 +746,6 @@ public class FunctionParser implements PostOrderTreeTraversalAlgo.Visitor {
return bindVariableService;
}
public boolean isValidNoArgFunction(ExpressionNode node) {
final ObjList<FunctionFactoryDescriptor> overload = functionFactoryCache.getOverloadList(node.token);
if (overload == null) {
return false;
}
for (int i = 0, n = overload.size(); i < n; i++) {
FunctionFactoryDescriptor ffd = overload.getQuick(i);
if (ffd.getSigArgCount() == 0) {
return true;
}
}
return false;
}
static {
for (int i = 0, n = SqlCompiler.sqlControlSymbols.size(); i < n; i++) {
FunctionFactoryCache.invalidFunctionNames.add(SqlCompiler.sqlControlSymbols.getQuick(i));
......
core/src/main/java/io/questdb/griffin/SqlCodeGenerator.java
浏览文件 @ 1c8acc11
......@@ -246,6 +246,10 @@ public class SqlCodeGenerator implements Mutable {
return true;
}
private RecordMetadata calculateSetMetadata(RecordMetadata masterMetadata) {
return GenericRecordMetadata.removeTimestamp(masterMetadata);
}
@Nullable
private Function compileFilter(IntrinsicModel intrinsicModel, RecordMetadata readerMeta, SqlExecutionContext executionContext) throws SqlException {
if (intrinsicModel.filter != null) {
......@@ -2037,7 +2041,7 @@ public class SqlCodeGenerator implements Mutable {
for (int i = 0; i < columnCount; i++) {
final QueryColumn column = columns.getQuick(i);
ExpressionNode node = column.getAst();
final ExpressionNode node = column.getAst();
if (node.type == ExpressionNode.LITERAL && Chars.equalsNc(node.token, timestampColumn)) {
virtualMetadata.setTimestampIndex(i);
}
......@@ -2053,7 +2057,6 @@ public class SqlCodeGenerator implements Mutable {
}
functions.add(function);
if (function instanceof SymbolFunction) {
virtualMetadata.add(
new TableColumnMetadata(
......@@ -2076,6 +2079,37 @@ public class SqlCodeGenerator implements Mutable {
}
}
// if timestamp was required and present in the base model but
// not selected, we will need to add it
if (
executionContext.isTimestampRequired()
&& timestampColumn != null
&& virtualMetadata.getTimestampIndex() == -1
) {
final Function timestampFunction = FunctionParser.createColumn(
0, timestampColumn, metadata
);
functions.add(timestampFunction);
// here the base timestamp column name can name-clash with one of the
// functions, so we have to use bottomUpColumns to lookup alias we should
// be using. Bottom up column should have our timestamp because optimiser puts it there
for (int i = 0, n = model.getBottomUpColumns().size(); i < n; i++) {
QueryColumn qc = model.getBottomUpColumns().getQuick(i);
if (qc.getAst().type == LITERAL && Chars.equals(timestampColumn, qc.getAst().token)) {
virtualMetadata.setTimestampIndex(virtualMetadata.getColumnCount());
virtualMetadata.add(
new TableColumnMetadata(
Chars.toString(qc.getAlias()),
timestampFunction.getType(),
timestampFunction.getMetadata()
)
);
break;
}
}
}
return new VirtualRecordCursorFactory(virtualMetadata, functions, factory);
} catch (SqlException | CairoException e) {
factory.close();
......@@ -2519,10 +2553,6 @@ public class SqlCodeGenerator implements Mutable {
}
}
private RecordMetadata calculateSetMetadata(RecordMetadata masterMetadata) {
return GenericRecordMetadata.removeTimestamp(masterMetadata);
}
private RecordCursorFactory generateUnionAllFactory(
QueryModel model,
RecordCursorFactory masterFactory,
......
core/src/main/java/io/questdb/griffin/SqlOptimiser.java
浏览文件 @ 1c8acc11
......@@ -783,6 +783,50 @@ class SqlOptimiser {
}
}
private void createSelectColumn0(
CharSequence columnName,
ExpressionNode columnAst,
QueryModel validatingModel,
QueryModel translatingModel,
QueryModel innerModel,
QueryModel analyticModel
) throws SqlException {
// add duplicate column names only to group-by model
// taking into account that column is pre-aliased, e.g.
// "col, col" will look like "col, col col1"
LowerCaseCharSequenceObjHashMap<CharSequence> translatingAliasMap = translatingModel.getColumnNameToAliasMap();
int index = translatingAliasMap.keyIndex(columnAst.token);
if (index < 0) {
// column is already being referenced by translating model
final CharSequence translatedColumnName = translatingAliasMap.valueAtQuick(index);
final CharSequence innerAlias = createColumnAlias(columnName, innerModel);
final QueryColumn translatedColumn = nextColumn(innerAlias, translatedColumnName);
innerModel.addBottomUpColumn(translatedColumn);
// analytic model is used together with inner model
final CharSequence analyticAlias = createColumnAlias(innerAlias, analyticModel);
final QueryColumn analyticColumn = nextColumn(analyticAlias, innerAlias);
analyticModel.addBottomUpColumn(analyticColumn);
} else {
final CharSequence alias = createColumnAlias(columnName, translatingModel);
addColumnToTranslatingModel(
queryColumnPool.next().of(
alias,
columnAst
),
translatingModel,
validatingModel
);
final QueryColumn translatedColumn = nextColumn(alias);
// create column that references inner alias we just created
innerModel.addBottomUpColumn(translatedColumn);
analyticModel.addBottomUpColumn(translatedColumn);
}
}
private void createSelectColumnsForWildcard(
QueryColumn qc,
boolean hasJoins,
......@@ -2882,6 +2926,17 @@ class SqlOptimiser {
throw SqlException.$(0, "Analytic function is not allowed in context of aggregation. Use sub-query.");
}
if (sampleBy != null && baseModel.getTimestamp() != null && innerVirtualModel.getColumnNameToAliasMap().excludes(baseModel.getTimestamp().token)) {
createSelectColumn0(
baseModel.getTimestamp().token,
baseModel.getTimestamp(),
baseModel,
translatingModel,
innerVirtualModel,
analyticModel
);
}
// check if translating model is redundant, e.g.
// that it neither chooses between tables nor renames columns
boolean translationIsRedundant = /*cursorModel.getBottomUpColumns().size() == 0 &&*/ (useInnerModel || useGroupByModel || useAnalyticModel);
......
core/src/test/java/io/questdb/griffin/SqlCodeGeneratorTest.java
浏览文件 @ 1c8acc11
......@@ -162,6 +162,43 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest {
}
}
@Test
public void testNonAggFunctionWithAggFunctionSampleBy() throws Exception {
assertMemoryLeak(() -> assertQuery(
"day\tisin\tlast\n" +
"1\tcc\t0.7544827361952741\n",
"select day(ts), isin, last(start_price) from xetra where isin='cc' sample by 1d",
"create table xetra as (" +
"select" +
" rnd_symbol('aa', 'bb', 'cc') isin," +
" rnd_double() start_price," +
" timestamp_sequence(0, 1000000) ts" +
" from long_sequence(10000)" +
") timestamp(ts)",
null,
false
));
}
@Test
public void testNonAggFunctionWithAggFunctionSampleBySubQuery() throws Exception {
assertMemoryLeak(() -> assertQuery(
"day\tisin\tlast\n" +
"1\tcc\t0.7544827361952741\n",
// "select day(ts), isin, last(start_price) from xetra where isin='cc' sample by 1d",
"select day(ts), isin, last from (select ts, isin, last(start_price) from xetra where isin='cc' sample by 1d)",
"create table xetra as (" +
"select" +
" rnd_symbol('aa', 'bb', 'cc') isin," +
" rnd_double() start_price," +
" timestamp_sequence(0, 1000000) ts" +
" from long_sequence(10000)" +
") timestamp(ts)",
null,
false
));
}
@Test
public void testBindVariableInSelect() throws Exception {
assertMemoryLeak(() -> {
......
core/src/test/java/io/questdb/griffin/SqlParserTest.java
浏览文件 @ 1c8acc11
......@@ -85,6 +85,15 @@ public class SqlParserTest extends AbstractGriffinTest {
);
}
@Test
public void testDuplicateColumnGroupBy() throws SqlException {
assertQuery(
"select-group-by b, sum(a) sum, k1, k1 k from (select-choose [b, a, k k1] b, a, k k1, timestamp from (select [b, a, k] from x y timestamp (timestamp)) y) y sample by 3h",
"select b, sum(a), k k1, k from x y sample by 3h",
modelOf("x").col("a", ColumnType.DOUBLE).col("b", ColumnType.SYMBOL).col("k", ColumnType.TIMESTAMP).timestamp()
);
}
@Test
public void testAnalyticFunctionReferencesSameColumnAsVirtual() throws Exception {
assertQuery(
......@@ -1811,10 +1820,10 @@ public class SqlParserTest extends AbstractGriffinTest {
}
@Test
public void testDuplicateColumnGroupBy() throws SqlException {
public void testDuplicateColumnsVirtualAndGroupBySelect() throws SqlException {
assertQuery(
"select-group-by b, sum(a) sum, k1, k1 k from (select-choose [b, a, k k1] b, a, k k1 from (select [b, a, k] from x y timestamp (timestamp)) y) y sample by 3h",
"select b, sum(a), k k1, k from x y sample by 3h",
"select-group-by sum(b + a) sum, column, k1, k1 k from (select-virtual [a, b, a + b column, k1] a, b, a + b column, k1, k1 k, timestamp from (select-choose [a, b, k k1] a, b, k k1, timestamp from (select [a, b, k] from x timestamp (timestamp)))) sample by 1m",
"select sum(b+a), a+b, k k1, k from x sample by 1m",
modelOf("x").col("a", ColumnType.DOUBLE).col("b", ColumnType.SYMBOL).col("k", ColumnType.TIMESTAMP).timestamp()
);
}
......@@ -1829,11 +1838,14 @@ public class SqlParserTest extends AbstractGriffinTest {
}
@Test
public void testDuplicateColumnsVirtualAndGroupBySelect() throws SqlException {
public void testNonAggFunctionWithAggFunctionSampleBy() throws SqlException {
assertQuery(
"select-group-by sum(b + a) sum, column, k1, k1 k from (select-virtual [a, b, a + b column, k1] a, b, a + b column, k1, k1 k from (select-choose [a, b, k k1] a, b, k k1 from (select [a, b, k] from x timestamp (timestamp)))) sample by 1m",
"select sum(b+a), a+b, k k1, k from x sample by 1m",
modelOf("x").col("a", ColumnType.DOUBLE).col("b", ColumnType.SYMBOL).col("k", ColumnType.TIMESTAMP).timestamp()
"select-group-by day, isin, last(start_price) last from (select-virtual [day(ts) day, isin, start_price] day(ts) day, isin, start_price, ts from (select [ts, isin, start_price] from xetra timestamp (ts) where isin = 'DE000A0KRJS4')) sample by 1d",
"select day(ts), isin, last(start_price) from xetra where isin='DE000A0KRJS4' sample by 1d",
modelOf("xetra")
.timestamp("ts")
.col("isin", ColumnType.SYMBOL)
.col("start_price", ColumnType.DOUBLE)
);
}
......@@ -5419,7 +5431,7 @@ public class SqlParserTest extends AbstractGriffinTest {
@Test
public void testUnionKeepOrderByWhenSampleByPresent() throws SqlException {
assertQuery(
"select-choose x from (select-choose [x] x, t from (select [x] from a) union select-choose y, t from (select [y, t] from b) union all select-group-by k, sum(z) sum from (select-virtual ['a' k, z] 'a' k, z from (select-choose [t, z] z, t from (select [t, z] from c order by t)) timestamp (t)) sample by 6h) order by x",
"select-choose x from (select-choose [x] x, t from (select [x] from a) union select-choose y, t from (select [y, t] from b) union all select-group-by k, sum(z) sum from (select-virtual ['a' k, z] 'a' k, z, t from (select-choose [t, z] z, t from (select [t, z] from c order by t)) timestamp (t)) sample by 6h) order by x",
"select x from (select * from a union select * from b union all select 'a' k, sum(z) from (c order by t) timestamp(t) sample by 6h) order by x",
modelOf("a").col("x", ColumnType.INT).col("t", ColumnType.TIMESTAMP),
modelOf("b").col("y", ColumnType.INT).col("t", ColumnType.TIMESTAMP),
......@@ -5466,7 +5478,7 @@ public class SqlParserTest extends AbstractGriffinTest {
@Test
public void testUnionRemoveRedundantOrderBy() throws SqlException {
assertQuery(
"select-choose x from (select-choose [x] x, t from (select [x] from a) union select-choose y, t from (select [y, t] from b) union all select-group-by 1, sum(z) sum from (select-virtual [1 1, z] 1 1, z from (select-choose [t, z] z, t from (select [t, z] from c order by t)) timestamp (t)) sample by 6h) order by x",
"select-choose x from (select-choose [x] x, t from (select [x] from a) union select-choose y, t from (select [y, t] from b) union all select-group-by 1, sum(z) sum from (select-virtual [1 1, z] 1 1, z, t from (select-choose [t, z] z, t from (select [t, z] from c order by t)) timestamp (t)) sample by 6h) order by x",
"select x from (select * from a union select * from b union all select 1, sum(z) from (c order by t, t) timestamp(t) sample by 6h) order by x",
modelOf("a").col("x", ColumnType.INT).col("t", ColumnType.TIMESTAMP),
modelOf("b").col("y", ColumnType.INT).col("t", ColumnType.TIMESTAMP),
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或