Skip to content

Q
Questdb

jobily / Questdb
11 个月 前同步成功

通知 1
Star 0
Fork 0
Q
Questdb

体验新版 GitCode，发现更多精彩内容 >>

未验证 提交 d4c95b8f 编写于 作者: A Alex Pelagenko 提交者： GitHub
浏览文件
操作

feat(sql): support const expression in sample by period (#1519)

上级 352999e1
隐藏空白更改
内联 并排
Showing with 390 addition and 116 deletion
core/src/main/java/io/questdb/griffin/SqlCodeGenerator.java
浏览文件 @ d4c95b8f
......@@ -1297,7 +1297,8 @@ public class SqlCodeGenerator implements Mutable {
private RecordCursorFactory generateSampleBy(
QueryModel model,
SqlExecutionContext executionContext,
ExpressionNode sampleByNode
ExpressionNode sampleByNode,
ExpressionNode sampleByUnits
) throws SqlException {
executionContext.pushTimestampRequiredFlag(true);
try {
......@@ -1343,7 +1344,23 @@ public class SqlCodeGenerator implements Mutable {
final RecordMetadata metadata = factory.getMetadata();
final ObjList<ExpressionNode> sampleByFill = model.getSampleByFill();
final TimestampSampler timestampSampler = TimestampSamplerFactory.getInstance(sampleByNode.token, sampleByNode.position);
final TimestampSampler timestampSampler;
if (sampleByUnits == null) {
timestampSampler = TimestampSamplerFactory.getInstance(sampleByNode.token, sampleByNode.position);
} else {
Function sampleByPeriod = functionParser.parseFunction(
sampleByNode,
EmptyRecordMetadata.INSTANCE,
executionContext
);
if (!sampleByPeriod.isConstant() || (sampleByPeriod.getType() != ColumnType.LONG && sampleByPeriod.getType() != ColumnType.INT)) {
sampleByPeriod.close();
throw SqlException.$(sampleByNode.position, "sample by period must be a constant expression of INT or LONG type");
}
long period = sampleByPeriod.getLong(null);
sampleByPeriod.close();
timestampSampler = TimestampSamplerFactory.getInstance(period, sampleByUnits.token, sampleByUnits.position);
}
final int fillCount = sampleByFill.size();
try {
......@@ -1965,7 +1982,7 @@ public class SqlCodeGenerator implements Mutable {
final ExpressionNode sampleByNode = model.getSampleBy();
if (sampleByNode != null) {
return generateSampleBy(model, executionContext, sampleByNode);
return generateSampleBy(model, executionContext, sampleByNode, model.getSampleByUnit());
}
RecordCursorFactory factory = null;
......
core/src/main/java/io/questdb/griffin/SqlKeywords.java
浏览文件 @ d4c95b8f
......@@ -24,6 +24,8 @@
package io.questdb.griffin;
import io.questdb.griffin.model.ExpressionNode;
public class SqlKeywords {
public static final String CONCAT_FUNC_NAME = "concat";
public static final int CASE_KEYWORD_LENGTH = 4;
......
core/src/main/java/io/questdb/griffin/SqlParser.java
浏览文件 @ d4c95b8f
......@@ -152,6 +152,48 @@ public final class SqlParser {
throw SqlException.$((lexer.lastTokenPosition()), "'by' expected");
}
private void expectSample(GenericLexer lexer, QueryModel model) throws SqlException {
final ExpressionNode n = expr(lexer, (QueryModel) null);
if (isFullSampleByPeriod(n)) {
model.setSampleBy(n);
return;
}
// This is complex expression of sample by period. It must follow time unit interval
ExpressionNode periodUnit = expectLiteral(lexer);
if (periodUnit == null || periodUnit.type != ExpressionNode.LITERAL || !isValidSampleByPeriodLetter(periodUnit.token)) {
int lexerPosition = lexer.getUnparsed() == null ? lexer.getPosition() : lexer.lastTokenPosition();
throw SqlException.$(periodUnit != null ? periodUnit.position : lexerPosition, "one letter sample by period unit expected");
}
model.setSampleBy(n, periodUnit);
}
public static boolean isFullSampleByPeriod(ExpressionNode n) {
return n != null && (n.type == ExpressionNode.CONSTANT || (n.type == ExpressionNode.LITERAL && isValidSampleByPeriodLetter(n.token)));
}
private static boolean isValidSampleByPeriodLetter(CharSequence token) {
if (token.length() != 1) return false;
switch (token.charAt(0)) {
case 'T':
// millis
case 's':
// seconds
case 'm':
// minutes
case 'h':
// hours
case 'd':
// days
case 'M':
// months
case 'y':
return true;
default:
return false;
}
}
private ExpressionNode expectExpr(GenericLexer lexer) throws SqlException {
final ExpressionNode n = expr(lexer, (QueryModel) null);
if (n != null) {
......@@ -863,7 +905,7 @@ public final class SqlParser {
if (tok != null && isSampleKeyword(tok)) {
expectBy(lexer);
model.setSampleBy(expectLiteral(lexer));
expectSample(lexer, model);
tok = optTok(lexer);
if (tok != null && isFillKeyword(tok)) {
......
core/src/main/java/io/questdb/griffin/engine/groupby/TimestampSamplerFactory.java
浏览文件 @ d4c95b8f
......@@ -28,6 +28,7 @@ import io.questdb.griffin.SqlException;
import io.questdb.std.Numbers;
import io.questdb.std.NumericException;
import io.questdb.std.datetime.microtime.Timestamps;
import org.jetbrains.annotations.NotNull;
public final class TimestampSamplerFactory {
......@@ -77,31 +78,7 @@ public final class TimestampSamplerFactory {
}
}
switch (cs.charAt(k)) {
case 'T':
// millis
return new MicroTimestampSampler(Timestamps.MILLI_MICROS * n);
case 's':
// seconds
return new MicroTimestampSampler(Timestamps.SECOND_MICROS * n);
case 'm':
// minutes
return new MicroTimestampSampler(Timestamps.MINUTE_MICROS * n);
case 'h':
// hours
return new MicroTimestampSampler(Timestamps.HOUR_MICROS * n);
case 'd':
// days
return new MicroTimestampSampler(Timestamps.DAY_MICROS * n);
case 'M':
// months
return new MonthTimestampSampler(n);
case 'y':
return new YearTimestampSampler(n);
default:
break;
}
return createTimestampSampler(n, cs.charAt(k), position + k);
} catch (NumericException ignore) {
// we are parsing a pre-validated number
// but we have to deal with checked exception anyway
......@@ -110,4 +87,42 @@ public final class TimestampSamplerFactory {
throw SqlException.$(position + k, "unsupported interval qualifier");
}
public static TimestampSampler getInstance(long period, CharSequence units, int position) throws SqlException {
if (units.length() == 1) {
return createTimestampSampler(period, units.charAt(0), position);
}
// Just in case SqlParser will allow this in the future
throw SqlException.$(position, "expected one character interval qualifier");
}
@NotNull
private static TimestampSampler createTimestampSampler(long interval, char timeUnit, int position) throws SqlException {
switch (timeUnit) {
case 'T':
// millis
return new MicroTimestampSampler(Timestamps.MILLI_MICROS * interval);
case 's':
// seconds
return new MicroTimestampSampler(Timestamps.SECOND_MICROS * interval);
case 'm':
// minutes
return new MicroTimestampSampler(Timestamps.MINUTE_MICROS * interval);
case 'h':
// hours
return new MicroTimestampSampler(Timestamps.HOUR_MICROS * interval);
case 'd':
// days
return new MicroTimestampSampler(Timestamps.DAY_MICROS * interval);
case 'M':
// months
return new MonthTimestampSampler((int) interval);
case 'y':
return new YearTimestampSampler((int) interval);
default:
// Just in case SqlParser will allow this in the future
throw SqlException.$(position, "unsupported interval qualifier");
}
}
}
core/src/main/java/io/questdb/griffin/model/QueryModel.java
浏览文件 @ d4c95b8f
......@@ -100,6 +100,7 @@ public class QueryModel implements Mutable, ExecutionModel, AliasTranslator, Sin
private ExpressionNode alias;
private ExpressionNode timestamp;
private ExpressionNode sampleBy;
private ExpressionNode sampleByUnit;
private JoinContext context;
private ExpressionNode joinCriteria;
private int joinType;
......@@ -255,6 +256,7 @@ public class QueryModel implements Mutable, ExecutionModel, AliasTranslator, Sin
public void clearSampleBy() {
sampleBy = null;
sampleByUnit = null;
sampleByFill.clear();
sampleByTimezoneName = null;
sampleByOffset = null;
......@@ -311,6 +313,10 @@ public class QueryModel implements Mutable, ExecutionModel, AliasTranslator, Sin
return alias;
}
public ExpressionNode getSampleByUnit() {
return sampleByUnit;
}
public int getTableId() {
return tableId;
}
......@@ -525,6 +531,11 @@ public class QueryModel implements Mutable, ExecutionModel, AliasTranslator, Sin
this.sampleBy = sampleBy;
}
public void setSampleBy(ExpressionNode sampleBy, ExpressionNode sampleByUnit) {
this.sampleBy = sampleBy;
this.sampleByUnit = sampleByUnit;
}
public ExpressionNode getSampleByTimezoneName() {
return sampleByTimezoneName;
}
......@@ -659,6 +670,7 @@ public class QueryModel implements Mutable, ExecutionModel, AliasTranslator, Sin
public void moveSampleByFrom(QueryModel model) {
this.sampleBy = model.sampleBy;
this.sampleByUnit = model.sampleByUnit;
this.sampleByFill.clear();
this.sampleByFill.addAll(model.sampleByFill);
this.sampleByTimezoneName = model.sampleByTimezoneName;
......
core/src/test/java/io/questdb/griffin/SqlParserTest.java
浏览文件 @ d4c95b8f
......@@ -2887,17 +2887,17 @@ public class SqlParserTest extends AbstractSqlParserTest {
@Test
public void testInvalidGroupBy1() throws Exception {
assertSyntaxError("select x, y from tab sample by x,", 32, "unexpected");
assertSyntaxError("select x, y from tab sample by x,", 32, "literal expected");
}
@Test
public void testInvalidGroupBy2() throws Exception {
assertSyntaxError("select x, y from (tab sample by x,)", 33, "')' expected");
assertSyntaxError("select x, y from (tab sample by x,)", 33, "literal expected");
}
@Test
public void testInvalidGroupBy3() throws Exception {
assertSyntaxError("select x, y from tab sample by x, order by y", 32, "unexpected token: ,");
assertSyntaxError("select x, y from tab sample by x, order by y", 32, "literal expected");
}
@Test
......
core/src/test/java/io/questdb/griffin/engine/groupby/SampleByTest.java
浏览文件 @ d4c95b8f
......@@ -63,6 +63,99 @@ public class SampleByTest extends AbstractGriffinTest {
"Invalid column: c");
}
@Test
public void testBindVarsInPeriodSyntax() throws Exception {
testSampleByPeriodFails(
"select k, s, first(lat) lat, last(lon) lon from x where s in ('a') sample by $1 T align to calendar",
"select k, s, first(lat) lat, last(lon) lon from x where s in ('a') sample by $".length() - 1,
"sample by period must be a constant expression"
);
}
@Test
public void testGeohashFillNull() throws Exception {
assertQuery(
"s\tk\tfirst\tfirst1\tfirst2\tfirst3\n" +
"TJW\t1970-01-03T00:00:00.000000Z\t010\tc93\tfu3r7c\t5ewm40wx\n" +
"PSWH\t1970-01-03T00:00:00.000000Z\t\t\t\t\n" +
"TJW\t1970-01-03T00:30:00.000000Z\t\t\t\t\n" +
"PSWH\t1970-01-03T00:30:00.000000Z\t\t\t\t\n" +
"TJW\t1970-01-03T01:00:00.000000Z\t\t\t\t\n" +
"PSWH\t1970-01-03T01:00:00.000000Z\t110\ttk5\txn8nmw\t0n2gm6r7\n",
"select s, k, " +
"first(g1), " +
"first(g2), " +
"first(g4), " +
"first(g8) " +
"from x sample by 30m fill(NULL)",
"create table x as " +
"(" +
"select" +
" rnd_geohash(3) g1," +
" rnd_geohash(15) g2," +
" rnd_geohash(30) g4," +
" rnd_geohash(40) g8," +
" rnd_symbol(2,3,4,0) s, " +
" timestamp_sequence(172800000000, 3600000000) k" +
" from" +
" long_sequence(2)" +
") timestamp(k) partition by NONE",
"k",
false
);
}
@Test
public void testGeohashFillPrev() throws Exception {
assertQuery(
"s\tk\tfirst\tfirst1\tfirst2\tfirst3\n" +
"TJW\t1970-01-03T00:00:00.000000Z\t010\tc93\tfu3r7c\t5ewm40wx\n" +
"PSWH\t1970-01-03T00:00:00.000000Z\t\t\t\t\n" +
"TJW\t1970-01-03T00:30:00.000000Z\t010\tc93\tfu3r7c\t5ewm40wx\n" +
"PSWH\t1970-01-03T00:30:00.000000Z\t\t\t\t\n" +
"TJW\t1970-01-03T01:00:00.000000Z\t010\tc93\tfu3r7c\t5ewm40wx\n" +
"PSWH\t1970-01-03T01:00:00.000000Z\t110\ttk5\txn8nmw\t0n2gm6r7\n",
"select s, k, " +
"first(g1), " +
"first(g2), " +
"first(g4), " +
"first(g8) " +
"from x sample by 30m fill(PREV)",
"create table x as " +
"(" +
"select" +
" rnd_geohash(3) g1," +
" rnd_geohash(15) g2," +
" rnd_geohash(30) g4," +
" rnd_geohash(40) g8," +
" rnd_symbol(2,3,4,0) s, " +
" timestamp_sequence(172800000000, 3600000000) k" +
" from" +
" long_sequence(2)" +
") timestamp(k) partition by NONE",
"k",
false
);
}
@Test
public void testGeohashInterpolated() throws Exception {
assertFailure(
"select k, first(b) from x sample by 3h fill(linear)",
"create table x as " +
"(" +
"select" +
" rnd_double(0)*100 a," +
" rnd_geohash(30) b," +
" timestamp_sequence(172800000000, 3600000000) k" +
" from" +
" long_sequence(20)" +
") timestamp(k) partition by NONE",
10,
"Unsupported interpolation type: GEOHASH(6c)"
);
}
@Test
public void testGroupByAllTypes() throws Exception {
assertQuery("b\tsum\tsum1\tsum2\tsum3\tsum4\tsum5\n" +
......@@ -2091,90 +2184,6 @@ public class SampleByTest extends AbstractGriffinTest {
);
}
@Test
public void testGeohashInterpolated() throws Exception {
assertFailure(
"select k, first(b) from x sample by 3h fill(linear)",
"create table x as " +
"(" +
"select" +
" rnd_double(0)*100 a," +
" rnd_geohash(30) b," +
" timestamp_sequence(172800000000, 3600000000) k" +
" from" +
" long_sequence(20)" +
") timestamp(k) partition by NONE",
10,
"Unsupported interpolation type: GEOHASH(6c)"
);
}
@Test
public void testGeohashFillPrev() throws Exception {
assertQuery(
"s\tk\tfirst\tfirst1\tfirst2\tfirst3\n" +
"TJW\t1970-01-03T00:00:00.000000Z\t010\tc93\tfu3r7c\t5ewm40wx\n" +
"PSWH\t1970-01-03T00:00:00.000000Z\t\t\t\t\n" +
"TJW\t1970-01-03T00:30:00.000000Z\t010\tc93\tfu3r7c\t5ewm40wx\n" +
"PSWH\t1970-01-03T00:30:00.000000Z\t\t\t\t\n" +
"TJW\t1970-01-03T01:00:00.000000Z\t010\tc93\tfu3r7c\t5ewm40wx\n" +
"PSWH\t1970-01-03T01:00:00.000000Z\t110\ttk5\txn8nmw\t0n2gm6r7\n",
"select s, k, " +
"first(g1), " +
"first(g2), " +
"first(g4), " +
"first(g8) " +
"from x sample by 30m fill(PREV)",
"create table x as " +
"(" +
"select" +
" rnd_geohash(3) g1," +
" rnd_geohash(15) g2," +
" rnd_geohash(30) g4," +
" rnd_geohash(40) g8," +
" rnd_symbol(2,3,4,0) s, " +
" timestamp_sequence(172800000000, 3600000000) k" +
" from" +
" long_sequence(2)" +
") timestamp(k) partition by NONE",
"k",
false
);
}
@Test
public void testGeohashFillNull() throws Exception {
assertQuery(
"s\tk\tfirst\tfirst1\tfirst2\tfirst3\n" +
"TJW\t1970-01-03T00:00:00.000000Z\t010\tc93\tfu3r7c\t5ewm40wx\n" +
"PSWH\t1970-01-03T00:00:00.000000Z\t\t\t\t\n" +
"TJW\t1970-01-03T00:30:00.000000Z\t\t\t\t\n" +
"PSWH\t1970-01-03T00:30:00.000000Z\t\t\t\t\n" +
"TJW\t1970-01-03T01:00:00.000000Z\t\t\t\t\n" +
"PSWH\t1970-01-03T01:00:00.000000Z\t110\ttk5\txn8nmw\t0n2gm6r7\n",
"select s, k, " +
"first(g1), " +
"first(g2), " +
"first(g4), " +
"first(g8) " +
"from x sample by 30m fill(NULL)",
"create table x as " +
"(" +
"select" +
" rnd_geohash(3) g1," +
" rnd_geohash(15) g2," +
" rnd_geohash(30) g4," +
" rnd_geohash(40) g8," +
" rnd_symbol(2,3,4,0) s, " +
" timestamp_sequence(172800000000, 3600000000) k" +
" from" +
" long_sequence(2)" +
") timestamp(k) partition by NONE",
"k",
false
);
}
@Test
public void testSampleBadFunctionInterpolated() throws Exception {
assertFailure(
......@@ -8753,6 +8762,154 @@ public class SampleByTest extends AbstractGriffinTest {
);
}
@Test
public void testSamplePeriodInvalidWithNoUnits() throws Exception {
testSampleByPeriodFails(
"select sum(a), k from x sample by 300/10 align to calendar",
"select sum(a), k from x sample by 300/10 a".length() - 1,
"one letter sample by period unit expected"
);
}
@Test
public void testSamplePeriodInvalidWithNoUnits2() throws Exception {
testSampleByPeriodFails(
"select sum(a), k from x sample by 300/10",
"select sum(a), k from x sample by 300/10".length(),
"literal expected"
);
}
@Test
public void testSamplePeriodInvalidWithWrongUnit() throws Exception {
testSampleByPeriodFails(
"select sum(a), k from x sample by 300/10 milli",
"select sum(a), k from x sample by 300/10 m".length() - 1,
"one letter sample by period unit expected"
);
}
@Test
public void testSamplePeriodInvalidWithWrongUnitLetter() throws Exception {
testSampleByPeriodFails(
"select sum(a), k from x sample by 300/10 L",
"select sum(a), k from x sample by 300/10 L".length() - 1,
"one letter sample by period unit expected"
);
}
@Test
public void testSimpleArithmeticsInPeriod() throws Exception {
assertQuery("sum\tk\n",
"select sum(a), k from x sample by (10+20)m",
"create table x as " +
"(" +
"select" +
" rnd_double(0)*100 a," +
" rnd_symbol(5,4,4,1) b," +
" timestamp_sequence(172800000000, 3600000000) k" +
" from" +
" long_sequence(0)" +
") timestamp(k) partition by NONE",
"k",
false);
}
@Test
public void testSimpleArithmeticsInPeriod2() throws Exception {
assertQuery("sum\tk\n" +
"1592.0966416600525\t1970-01-03T00:00:00.000000Z\n" +
"1566.8131178120786\t1970-01-04T06:00:00.000000Z\n" +
"1393.2872924527742\t1970-01-05T12:00:00.000000Z\n" +
"584.4161505427071\t1970-01-06T18:00:00.000000Z\n",
"select sum(a), k from x sample by (10+20) h",
"create table x as " +
"(" +
"select" +
" rnd_double(0)*100 a," +
" rnd_symbol(5,4,4,1) b," +
" timestamp_sequence(172800000000, 3600000000) k" +
" from" +
" long_sequence(100)" +
") timestamp(k) partition by NONE",
"k",
false);
}
@Test
public void testSimpleArithmeticsInPeriod3() throws Exception {
assertQuery("sum\tk\n" +
"1592.0966416600525\t1970-01-03T00:00:00.000000Z\n" +
"1566.8131178120786\t1970-01-04T06:00:00.000000Z\n" +
"1393.2872924527742\t1970-01-05T12:00:00.000000Z\n" +
"584.4161505427071\t1970-01-06T18:00:00.000000Z\n",
"select sum(a), k from x sample by 300/10 h",
"create table x as " +
"(" +
"select" +
" rnd_double(0)*100 a," +
" rnd_symbol(5,4,4,1) b," +
" timestamp_sequence(172800000000, 3600000000) k" +
" from" +
" long_sequence(100)" +
") timestamp(k) partition by NONE",
"k",
false);
}
@Test
public void testSimpleLongArithmeticsInPeriod() throws Exception {
assertQuery("sum\tk\n",
"select sum(a), k from x sample by (1+2)*10L m align to calendar",
"create table x as " +
"(" +
"select" +
" rnd_double(0)*100 a," +
" rnd_symbol(5,4,4,1) b," +
" timestamp_sequence(172800000000, 3600000000) k" +
" from" +
" long_sequence(0)" +
") timestamp(k) partition by NONE",
"k",
false);
}
@Test
public void testWrongTypeInPeriodSyntax() throws Exception {
testSampleByPeriodFails(
"select k, s, first(lat) lat, last(lon) lon from x where s in ('a') sample by 1.0*3 T",
"select k, s, first(lat) lat, last(lon) lon from x where s in ('a') sample by 1.0*".length() - 1,
"sample by period must be a constant expression of INT or LONG type"
);
}
@Test
public void testWrongTypeInPeriodSyntax2() throws Exception {
testSampleByPeriodFails(
"select k, s, first(lat) lat, last(lon) lon from x where s in ('a') sample by '1T'",
"select k, s, first(lat) lat, last(lon) lon from x where s in ('a') sample by '".length() - 1,
"expected single letter qualifier"
);
}
@Test
public void testWrongTypeInPeriodSyntax3() throws Exception {
testSampleByPeriodFails(
"select k, s, first(lat) lat, last(lon) lon from x where s in ('a') sample by '1' T",
"select k, s, first(lat) lat, last(lon) lon from x where s in ('a') sample by '1' T".length() - 1,
"unexpected token: T"
);
}
@Test
public void testWrongTypeInUnit() throws Exception {
testSampleByPeriodFails(
"select k, s, first(lat) lat, last(lon) lon from x where s in ('a') sample by 10*3 mi",
"select k, s, first(lat) lat, last(lon) lon from x where s in ('a') sample by 10*3 m".length() - 1,
"one letter sample by period unit expected"
);
}
private void assertSampleByIndexQuery(String expected, String query, String insert) throws Exception {
assertSampleByIndexQuery(expected, query, insert, false);
}
......@@ -8812,4 +8969,33 @@ public class SampleByTest extends AbstractGriffinTest {
}
};
}
private void testSampleByPeriodFails(String query, int errorPosition, String errorContains) throws Exception {
assertMemoryLeak(() -> {
compiler.compile(
"create table x as " +
"(" +
"select" +
" rnd_double(1)*180 lat," +
" rnd_double(1)*180 lon," +
" rnd_symbol('a') s," +
" timestamp_sequence('2021-03-28T00:59:00.00000Z', 60*1000000L) k" +
" from" +
" long_sequence(100)" +
"), index(s) timestamp(k) partition by DAY",
sqlExecutionContext
);
try (
RecordCursorFactory ignored = compiler.compile(
query,
sqlExecutionContext
).getRecordCursorFactory()
) {
Assert.fail();
} catch (SqlException ex) {
TestUtils.assertContains(ex.getFlyweightMessage(), errorContains);
Assert.assertEquals(errorPosition, ex.getPosition());
}
});
}
}
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或