diff --git a/core/src/main/java/io/questdb/cairo/CairoEngine.java b/core/src/main/java/io/questdb/cairo/CairoEngine.java index a6092019617693bc91e190d3686da5226eb7a224..e6e58e3f7508c971f18eb36b2e5d161089c23041 100644 --- a/core/src/main/java/io/questdb/cairo/CairoEngine.java +++ b/core/src/main/java/io/questdb/cairo/CairoEngine.java @@ -287,7 +287,7 @@ public class CairoEngine implements Closeable, WriterSource { if (null == lockedReason) { boolean locked = readerPool.lock(tableName); if (locked) { - LOG.info().$("locked [table=`").$(tableName).$("`, thread=").$(Thread.currentThread().getId()).$(']').$(); + LOG.info().$("locked [table=`").utf8(tableName).$("`, thread=").$(Thread.currentThread().getId()).$(']').$(); return null; } writerPool.unlock(tableName); @@ -446,7 +446,7 @@ public class CairoEngine implements Closeable, WriterSource { ) { readerPool.unlock(tableName); writerPool.unlock(tableName, writer, newTable); - LOG.info().$("unlocked [table=`").$(tableName).$("`]").$(); + LOG.info().$("unlocked [table=`").utf8(tableName).$("`]").$(); } public void unlockReaders(CharSequence tableName) { diff --git a/core/src/main/java/io/questdb/cairo/O3OpenColumnJob.java b/core/src/main/java/io/questdb/cairo/O3OpenColumnJob.java index 8b2671c9f5e58b59ee5d9817d1bdaac79e021b08..8512c812ae1963bc9d7e0bbf2d16b1e59e941b91 100644 --- a/core/src/main/java/io/questdb/cairo/O3OpenColumnJob.java +++ b/core/src/main/java/io/questdb/cairo/O3OpenColumnJob.java @@ -164,7 +164,7 @@ public class O3OpenColumnJob extends AbstractQueueConsumerJob public static void openColumn(O3OpenColumnTask task, long cursor, Sequence subSeq, long tmpBuf) { final int openColumnMode = task.getOpenColumnMode(); - final CharSequence pathToTable = task.getPathToTable(); + final Path pathToTable = task.getPathToTable(); final int columnType = task.getColumnType(); final CharSequence columnName = task.getColumnName(); final long srcOooLo = task.getSrcOooLo(); @@ -251,7 +251,7 @@ public class O3OpenColumnJob extends AbstractQueueConsumerJob public static void openColumn( int openColumnMode, - CharSequence pathToTable, + Path pathToTable, CharSequence columnName, AtomicInteger columnCounter, AtomicInteger partCounter, diff --git a/core/src/main/java/io/questdb/cairo/O3PartitionJob.java b/core/src/main/java/io/questdb/cairo/O3PartitionJob.java index 718ecee25f3a647b794594bf3cc6a74f2d15b1c0..83295b1500f67705a6551328e71ff639e1d59f77 100644 --- a/core/src/main/java/io/questdb/cairo/O3PartitionJob.java +++ b/core/src/main/java/io/questdb/cairo/O3PartitionJob.java @@ -55,7 +55,7 @@ public class O3PartitionJob extends AbstractQueueConsumerJob { } public static void processPartition( - CharSequence pathToTable, + Path pathToTable, int partitionBy, ObjList columns, ObjList oooColumns, @@ -543,7 +543,7 @@ public class O3PartitionJob extends AbstractQueueConsumerJob { // find "current" partition boundary in the out of order data // once we know the boundary we can move on to calculating another one // srcOooHi is index inclusive of value - final CharSequence pathToTable = task.getPathToTable(); + final Path pathToTable = task.getPathToTable(); final int partitionBy = task.getPartitionBy(); final ObjList columns = task.getColumns(); final ObjList oooColumns = task.getO3Columns(); @@ -616,7 +616,7 @@ public class O3PartitionJob extends AbstractQueueConsumerJob { private static void publishOpenColumnTaskHarmonized( long cursor, int openColumnMode, - CharSequence pathToTable, + Path pathToTable, CharSequence columnName, AtomicInteger columnCounter, AtomicInteger partCounter, @@ -704,7 +704,7 @@ public class O3PartitionJob extends AbstractQueueConsumerJob { long txn, ObjList columns, ObjList oooColumns, - CharSequence pathToTable, + Path pathToTable, long srcOooLo, long srcOooHi, long srcOooMax, @@ -916,7 +916,7 @@ public class O3PartitionJob extends AbstractQueueConsumerJob { long tmpBuf, long cursor, int openColumnMode, - CharSequence pathToTable, + Path pathToTable, CharSequence columnName, AtomicInteger columnCounter, AtomicInteger partCounter, diff --git a/core/src/main/java/io/questdb/cairo/O3PurgeDiscoveryJob.java b/core/src/main/java/io/questdb/cairo/O3PurgeDiscoveryJob.java index 1c7e5abf6f4e0f5560c174fe0c22632cd4d579f5..3267f9eacafcb39c12bb7a9313698e04d379c70f 100644 --- a/core/src/main/java/io/questdb/cairo/O3PurgeDiscoveryJob.java +++ b/core/src/main/java/io/questdb/cairo/O3PurgeDiscoveryJob.java @@ -32,7 +32,6 @@ import io.questdb.mp.RingQueue; import io.questdb.mp.Sequence; import io.questdb.std.*; import io.questdb.std.str.MutableCharSink; -import io.questdb.std.str.NativeLPSZ; import io.questdb.std.str.Path; import io.questdb.std.str.StringSink; import io.questdb.tasks.O3PurgeDiscoveryTask; @@ -44,7 +43,7 @@ public class O3PurgeDiscoveryJob extends AbstractQueueConsumerJob purgeQueue; private final Sequence purgePubSeq; @@ -55,11 +54,11 @@ public class O3PurgeDiscoveryJob extends AbstractQueueConsumerJob purgeQueue, @Nullable Sequence purgePubSeq, @@ -93,7 +92,7 @@ public class O3PurgeDiscoveryJob extends AbstractQueueConsumerJob 0) { try { do { - processDir(sink, nativeLPSZ, tableName, txnList, ff.findName(p), ff.findType(p)); + processDir(sink, fileNameSink, tableName, txnList, ff.findName(p), ff.findType(p)); } while (ff.findNext(p) > 0); } finally { ff.findClose(p); @@ -163,24 +162,23 @@ public class O3PurgeDiscoveryJob extends AbstractQueueConsumerJob { - nativeLPSZ.of(file); - if (type == Files.DT_DIR && IGNORED_FILES.excludes(nativeLPSZ)) { + ff.iterateDir(path.$(), (pUtf8NameZ, type) -> { + if (Files.isDir(pUtf8NameZ, type)) { path.trimTo(rootLen); - path.concat(nativeLPSZ); + path.concat(pUtf8NameZ); other.trimTo(rootLen); - other.concat(nativeLPSZ); + other.concat(pUtf8NameZ); int plen = path.length(); renameFileOrLog(ff, dFile(path.trimTo(plen), columnName), dFile(other.trimTo(plen), newName)); renameFileOrLog(ff, iFile(path.trimTo(plen), columnName), iFile(other.trimTo(plen), newName)); diff --git a/core/src/main/java/io/questdb/cairo/mig/EngineMigration.java b/core/src/main/java/io/questdb/cairo/mig/EngineMigration.java index 1cec06083dc74645c367a15156259bac93ba4c07..4c3a782c4a333e6a8f6061f2979637bfadcca6b0 100644 --- a/core/src/main/java/io/questdb/cairo/mig/EngineMigration.java +++ b/core/src/main/java/io/questdb/cairo/mig/EngineMigration.java @@ -24,14 +24,16 @@ package io.questdb.cairo.mig; -import io.questdb.cairo.*; +import io.questdb.cairo.CairoConfiguration; +import io.questdb.cairo.CairoEngine; +import io.questdb.cairo.CairoException; +import io.questdb.cairo.TableUtils; import io.questdb.cairo.vm.Vm; import io.questdb.cairo.vm.api.MemoryARW; import io.questdb.cairo.vm.api.MemoryMARW; import io.questdb.log.Log; import io.questdb.log.LogFactory; import io.questdb.std.*; -import io.questdb.std.str.NativeLPSZ; import io.questdb.std.str.Path; import org.jetbrains.annotations.Nullable; @@ -148,66 +150,62 @@ public class EngineMigration { copyPath.of(root); final int rootLen = path.length(); - final NativeLPSZ nativeLPSZ = new NativeLPSZ(); - ff.iterateDir(path.$(), (name, type) -> { - if (type == Files.DT_DIR) { - nativeLPSZ.of(name); - if (Chars.notDots(nativeLPSZ)) { - path.trimTo(rootLen); - path.concat(nativeLPSZ); - copyPath.trimTo(rootLen); - copyPath.concat(nativeLPSZ); - final int plen = path.length(); - path.concat(TableUtils.META_FILE_NAME); - - if (ff.exists(path.$())) { - final long fd = openFileRWOrFail(ff, path); - try { - int currentTableVersion = TableUtils.readIntOrFail(ff, fd, META_OFFSET_VERSION, mem, path); - if (currentTableVersion < latestVersion) { - LOG.info() - .$("upgrading [path=").$(path) - .$(",fromVersion=").$(currentTableVersion) - .$(",toVersion=").$(latestVersion) - .I$(); - - copyPath.trimTo(plen); - backupFile(ff, path, copyPath, TableUtils.META_FILE_NAME, currentTableVersion); - - path.trimTo(plen); - context.of(path, copyPath, fd); - - for (int ver = currentTableVersion + 1; ver <= latestVersion; ver++) { - final MigrationAction migration = getMigrationToVersion(ver); - if (migration != null) { - try { - LOG.info().$("upgrading table [path=").$(path).$(",toVersion=").$(ver).I$(); - migration.migrate(context); - path.trimTo(plen); - } catch (Throwable e) { - LOG.error().$("failed to upgrade table path=") - .$(path.trimTo(plen)) - .$(", exception: ") - .$(e).$(); - throw e; - } - } + ff.iterateDir(path.$(), (pUtf8NameZ, type) -> { + if (Files.isDir(pUtf8NameZ, type)) { + path.trimTo(rootLen); + path.concat(pUtf8NameZ); + copyPath.trimTo(rootLen); + copyPath.concat(pUtf8NameZ); + final int plen = path.length(); + path.concat(TableUtils.META_FILE_NAME); + + if (ff.exists(path.$())) { + final long fd = openFileRWOrFail(ff, path); + try { + int currentTableVersion = TableUtils.readIntOrFail(ff, fd, META_OFFSET_VERSION, mem, path); + if (currentTableVersion < latestVersion) { + LOG.info() + .$("upgrading [path=").$(path) + .$(",fromVersion=").$(currentTableVersion) + .$(",toVersion=").$(latestVersion) + .I$(); + + copyPath.trimTo(plen); + backupFile(ff, path, copyPath, TableUtils.META_FILE_NAME, currentTableVersion); - TableUtils.writeIntOrFail( - ff, - fd, - META_OFFSET_VERSION, - ver, - mem, - path.trimTo(plen) - ); + path.trimTo(plen); + context.of(path, copyPath, fd); + + for (int ver = currentTableVersion + 1; ver <= latestVersion; ver++) { + final MigrationAction migration = getMigrationToVersion(ver); + if (migration != null) { + try { + LOG.info().$("upgrading table [path=").$(path).$(",toVersion=").$(ver).I$(); + migration.migrate(context); + path.trimTo(plen); + } catch (Throwable e) { + LOG.error().$("failed to upgrade table path=") + .$(path.trimTo(plen)) + .$(", exception: ") + .$(e).$(); + throw e; + } } + + TableUtils.writeIntOrFail( + ff, + fd, + META_OFFSET_VERSION, + ver, + mem, + path.trimTo(plen) + ); } - } finally { - ff.close(fd); - path.trimTo(plen); - copyPath.trimTo(plen); } + } finally { + ff.close(fd); + path.trimTo(plen); + copyPath.trimTo(plen); } } } diff --git a/core/src/main/java/io/questdb/cairo/pool/ReaderPool.java b/core/src/main/java/io/questdb/cairo/pool/ReaderPool.java index 42309d5708947105d62e5503640aacf635a907ff..153a44b296a20058e0ea42751f07e50e9515c2ee 100644 --- a/core/src/main/java/io/questdb/cairo/pool/ReaderPool.java +++ b/core/src/main/java/io/questdb/cairo/pool/ReaderPool.java @@ -269,7 +269,10 @@ public class ReaderPool extends AbstractPool implements ResourcePool ColumnType.GEO_HASH_MAX_BITS_LENGTH) { + throw SqlException.position(position) + .put("invalid GEOHASH type precision range, mast be [1, 60] bits, provided=") + .put(size); + } + return size; + } + private void addConcatArgs(ObjList args, ExpressionNode leaf) { if (leaf.type != ExpressionNode.FUNCTION || !isConcatFunction(leaf.token)) { args.add(leaf); @@ -152,48 +214,6 @@ public final class SqlParser { throw SqlException.$((lexer.lastTokenPosition()), "'by' expected"); } - private void expectSample(GenericLexer lexer, QueryModel model) throws SqlException { - final ExpressionNode n = expr(lexer, (QueryModel) null); - if (isFullSampleByPeriod(n)) { - model.setSampleBy(n); - return; - } - // This is complex expression of sample by period. It must follow time unit interval - ExpressionNode periodUnit = expectLiteral(lexer); - if (periodUnit == null || periodUnit.type != ExpressionNode.LITERAL || !isValidSampleByPeriodLetter(periodUnit.token)) { - int lexerPosition = lexer.getUnparsed() == null ? lexer.getPosition() : lexer.lastTokenPosition(); - throw SqlException.$(periodUnit != null ? periodUnit.position : lexerPosition, "one letter sample by period unit expected"); - } - model.setSampleBy(n, periodUnit); - } - - - public static boolean isFullSampleByPeriod(ExpressionNode n) { - return n != null && (n.type == ExpressionNode.CONSTANT || (n.type == ExpressionNode.LITERAL && isValidSampleByPeriodLetter(n.token))); - } - - private static boolean isValidSampleByPeriodLetter(CharSequence token) { - if (token.length() != 1) return false; - switch (token.charAt(0)) { - case 'T': - // millis - case 's': - // seconds - case 'm': - // minutes - case 'h': - // hours - case 'd': - // days - case 'M': - // months - case 'y': - return true; - default: - return false; - } - } - private ExpressionNode expectExpr(GenericLexer lexer) throws SqlException { final ExpressionNode n = expr(lexer, (QueryModel) null); if (n != null) { @@ -257,6 +277,21 @@ public final class SqlParser { throw SqlException.$((lexer.lastTokenPosition()), "'offset' expected"); } + private void expectSample(GenericLexer lexer, QueryModel model) throws SqlException { + final ExpressionNode n = expr(lexer, (QueryModel) null); + if (isFullSampleByPeriod(n)) { + model.setSampleBy(n); + return; + } + // This is complex expression of sample by period. It must follow time unit interval + ExpressionNode periodUnit = expectLiteral(lexer); + if (periodUnit == null || periodUnit.type != ExpressionNode.LITERAL || !isValidSampleByPeriodLetter(periodUnit.token)) { + int lexerPosition = lexer.getUnparsed() == null ? lexer.getPosition() : lexer.lastTokenPosition(); + throw SqlException.$(periodUnit != null ? periodUnit.position : lexerPosition, "one letter sample by period unit expected"); + } + model.setSampleBy(n, periodUnit); + } + private CharSequence expectTableNameOrSubQuery(GenericLexer lexer) throws SqlException { return tok(lexer, "table name or sub-query"); } @@ -1064,7 +1099,9 @@ public final class SqlParser { throw SqlException.$(lexer.lastTokenPosition(), "'into' expected"); } - model.setTableName(expectLiteral(lexer)); + tok = tok(lexer, "table name"); + + model.setTableName(nextLiteral(GenericLexer.assertNoDotsAndSlashes(GenericLexer.unquote(tok), lexer.lastTokenPosition()), lexer.lastTokenPosition())); tok = tok(lexer, "'(' or 'select'"); @@ -1686,39 +1723,6 @@ public final class SqlParser { return type; } - static int parseGeoHashBits(int position, int start, CharSequence sizeStr) throws SqlException { - assert start >= 0; - if (sizeStr.length() - start < 2) { - throw SqlException.position(position) - .put("invalid GEOHASH size, must be number followed by 'C' or 'B' character"); - } - int size; - try { - size = Numbers.parseInt(sizeStr, start, sizeStr.length() - 1); - } catch (NumericException e) { - throw SqlException.position(position) - .put("invalid GEOHASH size, must be number followed by 'C' or 'B' character"); - } - switch (sizeStr.charAt(sizeStr.length() - 1)) { - case 'C': - case 'c': - size *= 5; - break; - case 'B': - case 'b': - break; - default: - throw SqlException.position(position) - .put("invalid GEOHASH size units, must be 'c', 'C' for chars, or 'b', 'B' for bits"); - } - if (size < 1 || size > ColumnType.GEO_HASH_MAX_BITS_LENGTH) { - throw SqlException.position(position) - .put("invalid GEOHASH type precision range, mast be [1, 60] bits, provided=") - .put(size); - } - return size; - } - private @NotNull CharSequence tok(GenericLexer lexer, String expectedList) throws SqlException { final int pos = lexer.getPosition(); CharSequence tok = optTok(lexer); @@ -1743,12 +1747,10 @@ public final class SqlParser { case ')': case ',': case '`': -// case '"': case '\'': throw SqlException.position(pos).put("literal expected"); default: break; - } } diff --git a/core/src/main/java/io/questdb/griffin/engine/functions/catalogue/AbstractClassCatalogueFunctionFactory.java b/core/src/main/java/io/questdb/griffin/engine/functions/catalogue/AbstractClassCatalogueFunctionFactory.java index b09d1d9ba4fa5126e8244e42f679e32615f59825..b8fda6f723ecdc37bddd25de9a479ac2d3ae778d 100644 --- a/core/src/main/java/io/questdb/griffin/engine/functions/catalogue/AbstractClassCatalogueFunctionFactory.java +++ b/core/src/main/java/io/questdb/griffin/engine/functions/catalogue/AbstractClassCatalogueFunctionFactory.java @@ -33,7 +33,6 @@ import io.questdb.griffin.engine.functions.CursorFunction; import io.questdb.log.Log; import io.questdb.log.LogFactory; import io.questdb.std.*; -import io.questdb.std.str.NativeLPSZ; import io.questdb.std.str.Path; import io.questdb.std.str.StringSink; import org.jetbrains.annotations.Nullable; @@ -116,7 +115,7 @@ public abstract class AbstractClassCatalogueFunctionFactory implements FunctionF private final DelegatingRecordImpl record = new DelegatingRecordImpl(); private final DiskReadingRecord diskReadingRecord = new DiskReadingRecord(); private final StaticReadingRecord staticReadingRecord = new StaticReadingRecord(); - private final NativeLPSZ nativeLPSZ = new NativeLPSZ(); + private final StringSink sink = new StringSink(); private final int plimit; private final int[] intValues = new int[5]; private final long tempMem; @@ -188,11 +187,11 @@ public abstract class AbstractClassCatalogueFunctionFactory implements FunctionF private boolean next0() { do { - final long pname = ff.findName(findFileStruct); - nativeLPSZ.of(pname); - if (ff.findType(findFileStruct) == Files.DT_DIR && Chars.notDots(nativeLPSZ)) { + final long pUtf8NameZ = ff.findName(findFileStruct); + final long type = ff.findType(findFileStruct); + if (Files.isDir(pUtf8NameZ, type, sink)) { path.trimTo(plimit); - if (ff.exists(path.concat(pname).concat(TableUtils.META_FILE_NAME).$())) { + if (ff.exists(path.concat(pUtf8NameZ).concat(TableUtils.META_FILE_NAME).$())) { // open metadata file and read id long fd = ff.openRO(path); if (fd > -1) { @@ -251,7 +250,6 @@ public abstract class AbstractClassCatalogueFunctionFactory implements FunctionF } private class DiskReadingRecord implements Record { - private final StringSink utf8SinkA = new StringSink(); private final StringSink utf8SinkB = new StringSink(); @Override @@ -267,7 +265,7 @@ public abstract class AbstractClassCatalogueFunctionFactory implements FunctionF @Override public CharSequence getStr(int col) { if (col == 0) { - return getName(utf8SinkA); + return sink; } return null; } @@ -283,17 +281,16 @@ public abstract class AbstractClassCatalogueFunctionFactory implements FunctionF @Override public int getStrLen(int col) { if (col == 0) { - CharSequence cs = getStr(col); - return cs != null ? cs.length() : -1; + return sink.length(); } return -1; } @Nullable - private CharSequence getName(StringSink utf8SinkA) { - utf8SinkA.clear(); - if (Chars.utf8DecodeZ(ff.findName(findFileStruct), utf8SinkA)) { - return utf8SinkA; + private CharSequence getName(StringSink sink) { + sink.clear(); + if (Chars.utf8DecodeZ(ff.findName(findFileStruct), sink)) { + return sink; } else { return null; } diff --git a/core/src/main/java/io/questdb/griffin/engine/functions/catalogue/AttrDefCatalogueFunctionFactory.java b/core/src/main/java/io/questdb/griffin/engine/functions/catalogue/AttrDefCatalogueFunctionFactory.java index 644add19465bde0920e88b78951a1575017a55c8..90e0d5eb47538008ea0dc4c94d9a90d2b6202fdb 100644 --- a/core/src/main/java/io/questdb/griffin/engine/functions/catalogue/AttrDefCatalogueFunctionFactory.java +++ b/core/src/main/java/io/questdb/griffin/engine/functions/catalogue/AttrDefCatalogueFunctionFactory.java @@ -32,8 +32,8 @@ import io.questdb.griffin.engine.functions.CursorFunction; import io.questdb.log.Log; import io.questdb.log.LogFactory; import io.questdb.std.*; -import io.questdb.std.str.NativeLPSZ; import io.questdb.std.str.Path; +import io.questdb.std.str.StringSink; public class AttrDefCatalogueFunctionFactory implements FunctionFactory { @@ -96,7 +96,6 @@ public class AttrDefCatalogueFunctionFactory implements FunctionFactory { private final Path path; private final FilesFacade ff; private final AttrDefCatalogueCursor.DiskReadingRecord diskReadingRecord = new AttrDefCatalogueCursor.DiskReadingRecord(); - private final NativeLPSZ nativeLPSZ = new NativeLPSZ(); private final int plimit; private final long tempMem; private int tableId = -1; @@ -160,12 +159,12 @@ public class AttrDefCatalogueFunctionFactory implements FunctionFactory { do { if (readNextFileFromDisk) { foundMetadataFile = false; - final long pname = ff.findName(findFileStruct); + final long pUtf8NameZ = ff.findName(findFileStruct); if (hasNextFile) { - nativeLPSZ.of(pname); - if (ff.findType(findFileStruct) == Files.DT_DIR && Chars.notDots(nativeLPSZ)) { + final long type = ff.findType(findFileStruct); + if (Files.isDir(pUtf8NameZ, type)) { path.trimTo(plimit); - if (ff.exists(path.concat(pname).concat(TableUtils.META_FILE_NAME).$())) { + if (ff.exists(path.concat(pUtf8NameZ).concat(TableUtils.META_FILE_NAME).$())) { long fd = ff.openRO(path); if (fd > -1) { if (ff.read(fd, tempMem, Integer.BYTES, TableUtils.META_OFFSET_TABLE_ID) == Integer.BYTES) { diff --git a/core/src/main/java/io/questdb/griffin/engine/functions/catalogue/AttributeCatalogueFunctionFactory.java b/core/src/main/java/io/questdb/griffin/engine/functions/catalogue/AttributeCatalogueFunctionFactory.java index b8f673f4f5ffb3a25ec02c871c71440d125568e3..1aa5b818bfa68406919aad8c90b3b33f435bc621 100644 --- a/core/src/main/java/io/questdb/griffin/engine/functions/catalogue/AttributeCatalogueFunctionFactory.java +++ b/core/src/main/java/io/questdb/griffin/engine/functions/catalogue/AttributeCatalogueFunctionFactory.java @@ -33,8 +33,8 @@ import io.questdb.griffin.FunctionFactory; import io.questdb.griffin.SqlExecutionContext; import io.questdb.griffin.engine.functions.CursorFunction; import io.questdb.std.*; -import io.questdb.std.str.NativeLPSZ; import io.questdb.std.str.Path; +import io.questdb.std.str.StringSink; import static io.questdb.cutlass.pgwire.PGOids.PG_TYPE_TO_SIZE_MAP; @@ -106,7 +106,6 @@ public class AttributeCatalogueFunctionFactory implements FunctionFactory { private final Path path; private final FilesFacade ff; private final DiskReadingRecord diskReadingRecord = new DiskReadingRecord(); - private final NativeLPSZ nativeLPSZ = new NativeLPSZ(); private final int plimit; private final MemoryMR metaMem; private long findFileStruct = 0; @@ -171,14 +170,12 @@ public class AttributeCatalogueFunctionFactory implements FunctionFactory { do { if (readNextFileFromDisk) { foundMetadataFile = false; - final long pname = ff.findName(findFileStruct); + final long pUtf8NameZ = ff.findName(findFileStruct); if (hasNextFile) { - nativeLPSZ.of(pname); - if ( - ff.findType(findFileStruct) == Files.DT_DIR && Chars.notDots(nativeLPSZ) - ) { + final long type = ff.findType(findFileStruct); + if (Files.isDir(pUtf8NameZ, type)) { path.trimTo(plimit); - path.concat(pname); + path.concat(pUtf8NameZ); if (ff.exists(path.concat(TableUtils.META_FILE_NAME).$())) { foundMetadataFile = true; metaMem.smallFile(ff, path, MemoryTag.MMAP_DEFAULT); @@ -277,7 +274,7 @@ public class AttributeCatalogueFunctionFactory implements FunctionFactory { metadata.add(new TableColumnMetadata("atttypmod", 6, ColumnType.INT)); metadata.add(new TableColumnMetadata("attlen", 7, ColumnType.SHORT)); metadata.add(new TableColumnMetadata("attidentity", 8, ColumnType.CHAR)); - metadata.add(new TableColumnMetadata("attisdropped", 9,ColumnType.BOOLEAN)); + metadata.add(new TableColumnMetadata("attisdropped", 9, ColumnType.BOOLEAN)); METADATA = metadata; } } diff --git a/core/src/main/java/io/questdb/griffin/engine/functions/catalogue/TableMetadataCursorFactory.java b/core/src/main/java/io/questdb/griffin/engine/functions/catalogue/TableMetadataCursorFactory.java index b8859aac01b78985f220ce6a5adb3388bba1ca69..b9d12d205ecbb68be1a95dabb5b7bec05d979e36 100644 --- a/core/src/main/java/io/questdb/griffin/engine/functions/catalogue/TableMetadataCursorFactory.java +++ b/core/src/main/java/io/questdb/griffin/engine/functions/catalogue/TableMetadataCursorFactory.java @@ -32,8 +32,8 @@ import io.questdb.griffin.engine.functions.CursorFunction; import io.questdb.log.Log; import io.questdb.log.LogFactory; import io.questdb.std.*; -import io.questdb.std.str.NativeLPSZ; import io.questdb.std.str.Path; +import io.questdb.std.str.StringSink; import static io.questdb.cairo.TableUtils.META_FILE_NAME; @@ -110,7 +110,7 @@ public class TableMetadataCursorFactory implements FunctionFactory { } private class TableListRecordCursor implements RecordCursor { - private final NativeLPSZ nativeLPSZ = new NativeLPSZ(); + private final StringSink sink = new StringSink(); private final TableListRecord record = new TableListRecord(); private long findPtr = 0; private TableReaderMetadata metaReader; @@ -144,10 +144,8 @@ public class TableMetadataCursorFactory implements FunctionFactory { return false; } } - nativeLPSZ.of(ff.findName(findPtr)); - int type = ff.findType(findPtr); - if (type == Files.DT_DIR && nativeLPSZ.charAt(0) != '.') { - if (record.open(nativeLPSZ)) { + if (Files.isDir(ff.findName(findPtr), ff.findType(findPtr), sink)) { + if (record.open(sink)) { return true; } } @@ -211,7 +209,7 @@ public class TableMetadataCursorFactory implements FunctionFactory { @Override public CharSequence getStr(int col) { if (col == nameColumn) { - return nativeLPSZ; + return sink; } if (col == partitionByColumn) { return PartitionBy.toString(partitionBy); @@ -234,7 +232,7 @@ public class TableMetadataCursorFactory implements FunctionFactory { return getStr(col).length(); } - public boolean open(NativeLPSZ tableName) { + public boolean open(CharSequence tableName) { int pathLen = path.length(); try { path.chop$().concat(tableName).concat(META_FILE_NAME).$(); diff --git a/core/src/main/java/io/questdb/griffin/engine/table/TableListRecordCursorFactory.java b/core/src/main/java/io/questdb/griffin/engine/table/TableListRecordCursorFactory.java index d46fe2db5014665635481b5df0b38cdc81aa884c..fc044e0e519f5dfb90fb4609d08cd5cded1fc6e5 100644 --- a/core/src/main/java/io/questdb/griffin/engine/table/TableListRecordCursorFactory.java +++ b/core/src/main/java/io/questdb/griffin/engine/table/TableListRecordCursorFactory.java @@ -33,8 +33,8 @@ import io.questdb.cairo.sql.RecordMetadata; import io.questdb.griffin.SqlExecutionContext; import io.questdb.std.Files; import io.questdb.std.FilesFacade; -import io.questdb.std.str.NativeLPSZ; import io.questdb.std.str.Path; +import io.questdb.std.str.StringSink; public class TableListRecordCursorFactory implements RecordCursorFactory { private static final RecordMetadata METADATA; @@ -71,7 +71,7 @@ public class TableListRecordCursorFactory implements RecordCursorFactory { } private class TableListRecordCursor implements RecordCursor { - private final NativeLPSZ nativeLPSZ = new NativeLPSZ(); + private final StringSink sink = new StringSink(); private final TableListRecord record = new TableListRecord(); private long findPtr = 0; @@ -101,9 +101,7 @@ public class TableListRecordCursorFactory implements RecordCursorFactory { return false; } } - nativeLPSZ.of(ff.findName(findPtr)); - int type = ff.findType(findPtr); - if (type == Files.DT_DIR && nativeLPSZ.charAt(0) != '.') { + if (Files.isDir(ff.findName(findPtr), ff.findType(findPtr), sink)) { return true; } } @@ -138,7 +136,7 @@ public class TableListRecordCursorFactory implements RecordCursorFactory { @Override public CharSequence getStr(int col) { if (col == 0) { - return nativeLPSZ; + return sink; } return null; } diff --git a/core/src/main/java/io/questdb/std/Chars.java b/core/src/main/java/io/questdb/std/Chars.java index 5e89ad7f009617fbd1d74d5445fdee4e759b9e34..53d5da7d7d6a8785a262941b578ed14e31414719 100644 --- a/core/src/main/java/io/questdb/std/Chars.java +++ b/core/src/main/java/io/questdb/std/Chars.java @@ -485,17 +485,6 @@ public final class Chars { return value != null && value.length() > 0; } - public static boolean notDots(CharSequence value) { - final int len = value.length(); - if (len > 2) { - return true; - } - if (value.charAt(0) != '.') { - return true; - } - return len == 2 && value.charAt(1) != '.'; - } - public static CharSequence repeat(String s, int times) { return new CharSequence() { @Override diff --git a/core/src/main/java/io/questdb/std/Files.java b/core/src/main/java/io/questdb/std/Files.java index f1f83ca549c14731b60cb1f93f4a411d09db9fa4..f68ad6355eb92cf7ad0cdab7fbd50d539d05665f 100644 --- a/core/src/main/java/io/questdb/std/Files.java +++ b/core/src/main/java/io/questdb/std/Files.java @@ -26,6 +26,7 @@ package io.questdb.std; import io.questdb.std.str.LPSZ; import io.questdb.std.str.Path; +import io.questdb.std.str.StringSink; import java.io.File; import java.nio.charset.Charset; @@ -201,6 +202,41 @@ public final class Files { } } + public static boolean notDots(CharSequence value) { + final int len = value.length(); + if (len > 2) { + return true; + } + if (value.charAt(0) != '.') { + return true; + } + return len == 2 && value.charAt(1) != '.'; + } + + public static boolean notDots(long pUtf8NameZ) { + final byte b0 = Unsafe.getUnsafe().getByte(pUtf8NameZ); + + if (b0 != '.') { + return true; + } + + final byte b1 = Unsafe.getUnsafe().getByte(pUtf8NameZ + 1); + return b1 != 0 && (b1 != '.' || Unsafe.getUnsafe().getByte(pUtf8NameZ + 2) != 0); + } + + public static boolean isDir(long pUtf8NameZ, long type, StringSink nameSink) { + if (type == DT_DIR) { + nameSink.clear(); + Chars.utf8DecodeZ(pUtf8NameZ, nameSink); + return notDots(nameSink); + } + return false; + } + + public static boolean isDir(long pUtf8NameZ, long type) { + return type == DT_DIR && notDots(pUtf8NameZ); + } + public static long openAppend(LPSZ lpsz) { return bumpFileCount(openAppend(lpsz.address())); } diff --git a/core/src/main/java/io/questdb/std/FindVisitor.java b/core/src/main/java/io/questdb/std/FindVisitor.java index 7a5d013d85be1963f67181fc57b7b8c87ddf5e79..b0c2b0d31e7aebd80537558b936b7c23d0537b0c 100644 --- a/core/src/main/java/io/questdb/std/FindVisitor.java +++ b/core/src/main/java/io/questdb/std/FindVisitor.java @@ -26,5 +26,5 @@ package io.questdb.std; @FunctionalInterface public interface FindVisitor { - void onFind(long name, int type); + void onFind(long pUtf8NameZ, int type); } diff --git a/core/src/main/java/io/questdb/std/str/NativeLPSZ.java b/core/src/main/java/io/questdb/std/str/NativeLPSZ.java deleted file mode 100644 index 4df69de9a594c00e694eb328965bdacc87545679..0000000000000000000000000000000000000000 --- a/core/src/main/java/io/questdb/std/str/NativeLPSZ.java +++ /dev/null @@ -1,55 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2022 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.std.str; - -import io.questdb.std.Unsafe; - -/** - * Represents C LPSZ as Java' CharSequence. Bytes in native memory are - * interpreted as ASCII characters. Multi-byte characters are NOT decoded. - */ -public class NativeLPSZ extends AbstractCharSequence { - private long address; - private int len; - - @Override - public int length() { - return len; - } - - @Override - public char charAt(int index) { - return (char) Unsafe.getUnsafe().getByte(address + index); - } - - @SuppressWarnings("StatementWithEmptyBody") - public NativeLPSZ of(long address) { - this.address = address; - long p = address; - while (Unsafe.getUnsafe().getByte(p++) != 0) ; - this.len = (int) (p - address - 1); - return this; - } -} diff --git a/core/src/main/java/io/questdb/std/str/Path.java b/core/src/main/java/io/questdb/std/str/Path.java index 74743ccf5ebed82441034db68972948dac340f3b..bdc9f79d161d23a160da9a351018eda31ad097e0 100644 --- a/core/src/main/java/io/questdb/std/str/Path.java +++ b/core/src/main/java/io/questdb/std/str/Path.java @@ -70,6 +70,17 @@ public class Path extends AbstractCharSink implements Closeable, LPSZ { return PATH.get().of(root); } + /** + * Creates path from another instance of Path. The assumption is that + * the source path is already UTF8 encoded and does not require re-encoding. + * + * @param root path + * @return copy of root path + */ + public static Path getThreadLocal(Path root) { + return PATH.get().of(root); + } + public static Path getThreadLocal2(CharSequence root) { return PATH2.get().of(root); } @@ -82,11 +93,6 @@ public class Path extends AbstractCharSink implements Closeable, LPSZ { return this; } - public Path slash$() { - ensureSeparator(); - return $(); - } - @Override public long address() { return ptr; @@ -114,11 +120,11 @@ public class Path extends AbstractCharSink implements Closeable, LPSZ { return concat(str, 0, str.length()); } - public Path concat(long lpsz) { + public Path concat(long pUtf8NameZ) { ensureSeparator(); - long p = lpsz; + long p = pUtf8NameZ; while (true) { if (len + OVERHEAD >= capacity) { @@ -183,6 +189,18 @@ public class Path extends AbstractCharSink implements Closeable, LPSZ { return this; } + @Override + public Path put(int value) { + super.put(value); + return this; + } + + @Override + public Path put(long value) { + super.put(value); + return this; + } + @Override public CharSink put(char[] chars, int start, int len) { if (len + this.len >= capacity) { @@ -193,6 +211,15 @@ public class Path extends AbstractCharSink implements Closeable, LPSZ { return this; } + @Override + public void putUtf8Special(char c) { + if (c == '/' && Os.type == Os.WINDOWS) { + put('\\'); + } else { + put(c); + } + } + @Override public final int length() { return len; @@ -245,12 +272,6 @@ public class Path extends AbstractCharSink implements Closeable, LPSZ { return this; } - private void checkClosed() { - if (ptr == 0) { - this.ptr = this.wptr = Unsafe.malloc(capacity + 1, MemoryTag.NATIVE_DEFAULT); - } - } - public Path of(CharSequence str, int from, int to) { checkClosed(); this.wptr = ptr; @@ -258,36 +279,29 @@ public class Path extends AbstractCharSink implements Closeable, LPSZ { return concat(str, from, to); } - @Override - public Path put(long value) { - super.put(value); - return this; - } - - @Override - public Path put(int value) { - super.put(value); + public Path slash() { + ensureSeparator(); return this; } - @Override - public void putUtf8Special(char c) { - if (c == '/' && Os.type == Os.WINDOWS) { - put('\\'); - } else { - put(c); - } - } - - public Path slash() { + public Path slash$() { ensureSeparator(); - return this; + return $(); } @Override @NotNull public String toString() { - return ptr == 0 ? "" : AbstractCharSequence.getString(this); + if (ptr != 0) { + final CharSink b = Misc.getThreadLocalBuilder(); + if (Unsafe.getUnsafe().getByte(wptr - 1) == 0) { + Chars.utf8Decode(ptr, wptr - 1, b); + } else { + Chars.utf8Decode(ptr, wptr, b); + } + return b.toString(); + } + return ""; } public Path trimTo(int len) { @@ -296,6 +310,12 @@ public class Path extends AbstractCharSink implements Closeable, LPSZ { return this; } + private void checkClosed() { + if (ptr == 0) { + this.ptr = this.wptr = Unsafe.malloc(capacity + 1, MemoryTag.NATIVE_DEFAULT); + } + } + private void copy(CharSequence str, int from, int to) { encodeUtf8(str, from, to); } diff --git a/core/src/main/java/io/questdb/tasks/O3OpenColumnTask.java b/core/src/main/java/io/questdb/tasks/O3OpenColumnTask.java index c68bf7e927d3854a915d36e7178cc22b2d7fc69e..2718499cf2572e919126a1bb495dc9cec02bef45 100644 --- a/core/src/main/java/io/questdb/tasks/O3OpenColumnTask.java +++ b/core/src/main/java/io/questdb/tasks/O3OpenColumnTask.java @@ -26,11 +26,12 @@ package io.questdb.tasks; import io.questdb.cairo.BitmapIndexWriter; import io.questdb.cairo.TableWriter; +import io.questdb.std.str.Path; import java.util.concurrent.atomic.AtomicInteger; public class O3OpenColumnTask { - private CharSequence pathToTable; + private Path pathToTable; private AtomicInteger columnCounter; private AtomicInteger partCounter; private long txn; @@ -130,7 +131,7 @@ public class O3OpenColumnTask { return partitionTimestamp; } - public CharSequence getPathToTable() { + public Path getPathToTable() { return pathToTable; } @@ -228,7 +229,7 @@ public class O3OpenColumnTask { public void of( int openColumnMode, - CharSequence pathToTable, + Path pathToTable, CharSequence columnName, AtomicInteger columnCounter, AtomicInteger partCounter, diff --git a/core/src/main/java/io/questdb/tasks/O3PartitionTask.java b/core/src/main/java/io/questdb/tasks/O3PartitionTask.java index 3f27be6b88e018da76067085030b3f400c486636..f90b0207439181d77b6f882a4239bbc890c72dc5 100644 --- a/core/src/main/java/io/questdb/tasks/O3PartitionTask.java +++ b/core/src/main/java/io/questdb/tasks/O3PartitionTask.java @@ -29,11 +29,12 @@ import io.questdb.cairo.TableWriter; import io.questdb.cairo.vm.api.MemoryCARW; import io.questdb.cairo.vm.api.MemoryMAR; import io.questdb.std.ObjList; +import io.questdb.std.str.Path; import java.util.concurrent.atomic.AtomicInteger; public class O3PartitionTask { - private CharSequence pathToTable; + private Path pathToTable; private int partitionBy; private ObjList columns; private ObjList o3Columns; @@ -81,7 +82,7 @@ public class O3PartitionTask { return partitionTimestamp; } - public CharSequence getPathToTable() { + public Path getPathToTable() { return pathToTable; } @@ -130,7 +131,7 @@ public class O3PartitionTask { } public void of( - CharSequence path, + Path path, int partitionBy, ObjList columns, ObjList o3Columns, diff --git a/core/src/test/java/io/questdb/FilesTest.java b/core/src/test/java/io/questdb/FilesTest.java index dcdd939b80c27ae41622a094a370d95ced9bf112..029e957498d67cd7ddaa5dc7f581dafb520016e3 100644 --- a/core/src/test/java/io/questdb/FilesTest.java +++ b/core/src/test/java/io/questdb/FilesTest.java @@ -26,8 +26,8 @@ package io.questdb; import io.questdb.std.*; import io.questdb.std.datetime.millitime.DateFormatUtils; -import io.questdb.std.str.NativeLPSZ; import io.questdb.std.str.Path; +import io.questdb.std.str.StringSink; import io.questdb.test.tools.TestUtils; import org.junit.Assert; import org.junit.Rule; @@ -191,12 +191,14 @@ public class FilesTest { try (Path path = new Path().of(temp).$()) { try (Path cp = new Path()) { Assert.assertTrue(Files.touch(cp.of(temp).concat("a.txt").$())); - NativeLPSZ name = new NativeLPSZ(); + StringSink nameSink = new StringSink(); long pFind = Files.findFirst(path); Assert.assertTrue(pFind != 0); try { do { - names.add(name.of(Files.findName(pFind)).toString()); + nameSink.clear(); + Chars.utf8DecodeZ(Files.findName(pFind), nameSink); + names.add(nameSink.toString()); } while (Files.findNext(pFind) > 0); } finally { Files.findClose(pFind); diff --git a/core/src/test/java/io/questdb/cairo/TableWriterTest.java b/core/src/test/java/io/questdb/cairo/TableWriterTest.java index c891ba291d5da0a6a296a1bdb3e425f3922c950a..1406d98089ebfe463dd0a3b75c4e233a4f376b10 100644 --- a/core/src/test/java/io/questdb/cairo/TableWriterTest.java +++ b/core/src/test/java/io/questdb/cairo/TableWriterTest.java @@ -41,9 +41,7 @@ import io.questdb.std.datetime.DateLocale; import io.questdb.std.datetime.DateLocaleFactory; import io.questdb.std.datetime.microtime.TimestampFormatCompiler; import io.questdb.std.datetime.microtime.TimestampFormatUtils; -import io.questdb.std.datetime.millitime.DateFormatUtils; import io.questdb.std.str.LPSZ; -import io.questdb.std.str.NativeLPSZ; import io.questdb.std.str.Path; import io.questdb.test.tools.TestUtils; import org.junit.Assert; @@ -2974,7 +2972,7 @@ public class TableWriterTest extends AbstractCairoTest { private int getDirCount() { AtomicInteger count = new AtomicInteger(); try (Path path = new Path()) { - FF.iterateDir(path.of(root).concat(PRODUCT).$(), (name, type) -> { + FF.iterateDir(path.of(root).concat(PRODUCT).$(), (pUtf8NameZ, type) -> { if (type == Files.DT_DIR) { count.incrementAndGet(); } @@ -3475,17 +3473,14 @@ public class TableWriterTest extends AbstractCairoTest { writer.removeColumn("supplier"); - final NativeLPSZ lpsz = new NativeLPSZ(); try (Path path = new Path()) { path.of(root).concat(model.getName()); final int plen = path.length(); - FF.iterateDir(path.$(), (file, type) -> { - lpsz.of(file); - - if (type == Files.DT_DIR && !Files.isDots(lpsz)) { - Assert.assertFalse(FF.exists(path.trimTo(plen).concat(lpsz).concat("supplier.i").$())); - Assert.assertFalse(FF.exists(path.trimTo(plen).concat(lpsz).concat("supplier.d").$())); - Assert.assertFalse(FF.exists(path.trimTo(plen).concat(lpsz).concat("supplier.top").$())); + FF.iterateDir(path.$(), (pUtf8NameZ, type) -> { + if (Files.isDir(pUtf8NameZ, type)) { + Assert.assertFalse(FF.exists(path.trimTo(plen).concat(pUtf8NameZ).concat("supplier.i").$())); + Assert.assertFalse(FF.exists(path.trimTo(plen).concat(pUtf8NameZ).concat("supplier.d").$())); + Assert.assertFalse(FF.exists(path.trimTo(plen).concat(pUtf8NameZ).concat("supplier.top").$())); } }); } @@ -3556,30 +3551,28 @@ public class TableWriterTest extends AbstractCairoTest { writer.renameColumn("supplier", "sup"); - final NativeLPSZ lpsz = new NativeLPSZ(); try (Path path = new Path()) { path.of(root).concat(model.getName()); final int plen = path.length(); if (columnTypeTag == ColumnType.SYMBOL) { - Assert.assertFalse(FF.exists(path.trimTo(plen).concat(lpsz).concat("supplier.v").$())); - Assert.assertFalse(FF.exists(path.trimTo(plen).concat(lpsz).concat("supplier.o").$())); - Assert.assertFalse(FF.exists(path.trimTo(plen).concat(lpsz).concat("supplier.c").$())); - Assert.assertFalse(FF.exists(path.trimTo(plen).concat(lpsz).concat("supplier.k").$())); - Assert.assertTrue(FF.exists(path.trimTo(plen).concat(lpsz).concat("sup.v").$())); - Assert.assertTrue(FF.exists(path.trimTo(plen).concat(lpsz).concat("sup.o").$())); - Assert.assertTrue(FF.exists(path.trimTo(plen).concat(lpsz).concat("sup.c").$())); - Assert.assertTrue(FF.exists(path.trimTo(plen).concat(lpsz).concat("sup.k").$())); + Assert.assertFalse(FF.exists(path.trimTo(plen).concat("supplier.v").$())); + Assert.assertFalse(FF.exists(path.trimTo(plen).concat("supplier.o").$())); + Assert.assertFalse(FF.exists(path.trimTo(plen).concat("supplier.c").$())); + Assert.assertFalse(FF.exists(path.trimTo(plen).concat("supplier.k").$())); + Assert.assertTrue(FF.exists(path.trimTo(plen).concat("sup.v").$())); + Assert.assertTrue(FF.exists(path.trimTo(plen).concat("sup.o").$())); + Assert.assertTrue(FF.exists(path.trimTo(plen).concat("sup.c").$())); + Assert.assertTrue(FF.exists(path.trimTo(plen).concat("sup.k").$())); } path.trimTo(plen); - FF.iterateDir(path.$(), (file, type) -> { - lpsz.of(file); - if (type == Files.DT_DIR && !Files.isDots(lpsz)) { - Assert.assertFalse(FF.exists(path.trimTo(plen).concat(lpsz).concat("supplier.i").$())); - Assert.assertFalse(FF.exists(path.trimTo(plen).concat(lpsz).concat("supplier.d").$())); - Assert.assertFalse(FF.exists(path.trimTo(plen).concat(lpsz).concat("supplier.top").$())); - Assert.assertTrue(FF.exists(path.trimTo(plen).concat(lpsz).concat("sup.d").$())); + FF.iterateDir(path.$(), (pUtf8NameZ, type) -> { + if (Files.isDir(pUtf8NameZ, type)) { + Assert.assertFalse(FF.exists(path.trimTo(plen).concat(pUtf8NameZ).concat("supplier.i").$())); + Assert.assertFalse(FF.exists(path.trimTo(plen).concat(pUtf8NameZ).concat("supplier.d").$())); + Assert.assertFalse(FF.exists(path.trimTo(plen).concat(pUtf8NameZ).concat("supplier.top").$())); + Assert.assertTrue(FF.exists(path.trimTo(plen).concat(pUtf8NameZ).concat("sup.d").$())); if (columnTypeTag == ColumnType.BINARY || columnTypeTag == ColumnType.STRING) { - Assert.assertTrue(FF.exists(path.trimTo(plen).concat(lpsz).concat("sup.i").$())); + Assert.assertTrue(FF.exists(path.trimTo(plen).concat(pUtf8NameZ).concat("sup.i").$())); } } }); diff --git a/core/src/test/java/io/questdb/griffin/O3Test.java b/core/src/test/java/io/questdb/griffin/O3Test.java index da45c73b388b82779b7b3e51f372aaa29afc8493..b34fd3c18ee42a913c90f3c119fdfef319da51b3 100644 --- a/core/src/test/java/io/questdb/griffin/O3Test.java +++ b/core/src/test/java/io/questdb/griffin/O3Test.java @@ -291,6 +291,16 @@ public class O3Test extends AbstractO3Test { executeWithPool(0, O3Test::testColumnTopMidOOOData0); } + @Test + public void testColumnTopMidOOODataUtf8Contended() throws Exception { + executeWithPool(0, O3Test::testColumnTopMidOOODataUtf80); + } + + @Test + public void testColumnTopMidOOODataUtf8Parallel() throws Exception { + executeWithPool(4, O3Test::testColumnTopMidOOODataUtf80); + } + @Test public void testColumnTopMidOOODataParallel() throws Exception { executeWithPool(4, O3Test::testColumnTopMidOOOData0); @@ -4935,6 +4945,157 @@ public class O3Test extends AbstractO3Test { ); } + private static void testColumnTopMidOOODataUtf80( + CairoEngine engine, + SqlCompiler compiler, + SqlExecutionContext sqlExecutionContext + ) throws SqlException, URISyntaxException { + compiler.compile( + "create table 'привет от штиблет' as (" + + "select" + + " cast(x as int) i," + + " rnd_symbol('msft','ibm', 'googl') sym," + + " round(rnd_double(0)*100, 3) amt," + + " to_timestamp('2018-01', 'yyyy-MM') + x * 720000000 timestamp," + + " rnd_boolean() b," + + " rnd_str('ABC', 'CDE', null, 'XYZ') c," + + " rnd_double(2) d," + + " rnd_float(2) e," + + " rnd_short(10,1024) f," + + " rnd_date(to_date('2015', 'yyyy'), to_date('2016', 'yyyy'), 2) g," + + " rnd_symbol(4,4,4,2) ik," + + " rnd_long() j," + + " timestamp_sequence(500000000000L,100000000L) ts," + + " rnd_byte(2,50) l," + + " rnd_bin(10, 20, 2) m," + + " rnd_str(5,16,2) n," + + " rnd_char() t" + + " from long_sequence(500)" + + "), index(sym) timestamp (ts) partition by DAY", + sqlExecutionContext + ); + + compiler.compile("alter table 'привет от штиблет' add column v double", sqlExecutionContext); + compiler.compile("alter table 'привет от штиблет' add column v1 float", sqlExecutionContext); + compiler.compile("alter table 'привет от штиблет' add column v2 int", sqlExecutionContext); + compiler.compile("alter table 'привет от штиблет' add column v3 byte", sqlExecutionContext); + compiler.compile("alter table 'привет от штиблет' add column v4 short", sqlExecutionContext); + compiler.compile("alter table 'привет от штиблет' add column v5 boolean", sqlExecutionContext); + compiler.compile("alter table 'привет от штиблет' add column v6 date", sqlExecutionContext); + compiler.compile("alter table 'привет от штиблет' add column v7 timestamp", sqlExecutionContext); + compiler.compile("alter table 'привет от штиблет' add column v8 symbol", sqlExecutionContext); + compiler.compile("alter table 'привет от штиблет' add column v10 char", sqlExecutionContext); + compiler.compile("alter table 'привет от штиблет' add column v11 string", sqlExecutionContext); + compiler.compile("alter table 'привет от штиблет' add column v12 binary", sqlExecutionContext); + compiler.compile("alter table 'привет от штиблет' add column v9 long", sqlExecutionContext); + + compiler.compile( + "insert into 'привет от штиблет' " + + "select" + + " cast(x as int) i," + + " rnd_symbol('msft','ibm', 'googl') sym," + + " round(rnd_double(0)*100, 3) amt," + + " to_timestamp('2018-01', 'yyyy-MM') + x * 720000000 timestamp," + + " rnd_boolean() b," + + " rnd_str('ABC', 'CDE', null, 'XYZ') c," + + " rnd_double(2) d," + + " rnd_float(2) e," + + " rnd_short(10,1024) f," + + " rnd_date(to_date('2015', 'yyyy'), to_date('2016', 'yyyy'), 2) g," + + " rnd_symbol(4,4,4,2) ik," + + " rnd_long() j," + + " timestamp_sequence(549920000000L,100000000L) ts," + + " rnd_byte(2,50) l," + + " rnd_bin(10, 20, 2) m," + + " rnd_str(5,16,2) n," + + " rnd_char() t," + +// -------- new columns here --------------- + " rnd_double() v," + + " rnd_float() v1," + + " rnd_int() v2," + + " rnd_byte() v3," + + " rnd_short() v4," + + " rnd_boolean() v5," + + " rnd_date() v6," + + " rnd_timestamp(10,100000,356) v7," + + " rnd_symbol('AAA','BBB', null) v8," + + " rnd_char() v10," + + " rnd_str() v11," + + " rnd_bin() v12," + + " rnd_long() v9" + + " from long_sequence(1000)", + sqlExecutionContext + ); + + compiler.compile( + "create table append as (" + + "select" + + " cast(x as int) i," + + " rnd_symbol('msft','ibm', 'googl') sym," + + " round(rnd_double(0)*100, 3) amt," + + " to_timestamp('2018-01', 'yyyy-MM') + x * 720000000 timestamp," + + " rnd_boolean() b," + + " rnd_str('ABC', 'CDE', null, 'XYZ') c," + + " rnd_double(2) d," + + " rnd_float(2) e," + + " rnd_short(10,1024) f," + + " rnd_date(to_date('2015', 'yyyy'), to_date('2016', 'yyyy'), 2) g," + + " rnd_symbol(4,4,4,2) ik," + + " rnd_long() j," + + " timestamp_sequence(546600000000L,100000L) ts," + + " rnd_byte(2,50) l," + + " rnd_bin(10, 20, 2) m," + + " rnd_str(5,16,2) n," + + " rnd_char() t," + +// -------- new columns here --------------- + " rnd_double() v," + + " rnd_float() v1," + + " rnd_int() v2," + + " rnd_byte() v3," + + " rnd_short() v4," + + " rnd_boolean() v5," + + " rnd_date() v6," + + " rnd_timestamp(10,100000,356) v7," + + " rnd_symbol('AAA','BBB', null) v8," + + " rnd_char() v10," + + " rnd_str() v11," + + " rnd_bin() v12," + + " rnd_long() v9" + + " from long_sequence(100)" + + ") timestamp (ts) partition by DAY", + sqlExecutionContext + ); + + TestUtils.printSql(compiler, sqlExecutionContext, "select count() from ('привет от штиблет' union all append)", sink2); + TestUtils.printSql(compiler, sqlExecutionContext, "select max(ts) from ('привет от штиблет' union all append)", sink); + final String expectedMaxTimestamp = Chars.toString(sink); + compiler.compile("insert into 'привет от штиблет' select * from append", sqlExecutionContext); + + assertSqlResultAgainstFile( + compiler, + sqlExecutionContext, + "'привет от штиблет'", + "/o3/testColumnTopMidOOOData.txt" + ); + printSqlResult(compiler, sqlExecutionContext, "select count() from 'привет от штиблет'"); + TestUtils.assertEquals(sink2, sink); + + try ( + final TableWriter w = engine.getWriter( + sqlExecutionContext.getCairoSecurityContext(), + "привет от штиблет", + "test" + ) + ) { + sink.clear(); + sink.put("max\n"); + TimestampFormatUtils.appendDateTimeUSec(sink, w.getMaxTimestamp()); + sink.put('\n'); + TestUtils.assertEquals(expectedMaxTimestamp, sink); + Assert.assertEquals(0, w.getO3RowCount()); + } + } + private static void testPartitionedDataAppendOOData0( CairoEngine engine, SqlCompiler compiler, diff --git a/core/src/test/java/io/questdb/griffin/SqlCodeGeneratorTest.java b/core/src/test/java/io/questdb/griffin/SqlCodeGeneratorTest.java index 24ee048a7996b07ec2ab6a9fbc0325e1f2ba5dcf..671029a5dc97572de11e9316ccea4ff8ed064e9c 100644 --- a/core/src/test/java/io/questdb/griffin/SqlCodeGeneratorTest.java +++ b/core/src/test/java/io/questdb/griffin/SqlCodeGeneratorTest.java @@ -192,20 +192,20 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest { public void testBindVariableInIndexLookupList() throws Exception { assertMemoryLeak(() -> { compiler.compile("CREATE TABLE 'alcatel_traffic_tmp' (" + - "deviceName SYMBOL capacity 1000 index, " + - "time TIMESTAMP, " + - "slot SYMBOL, " + - "port SYMBOL, " + - "downStream DOUBLE, " + - "upStream DOUBLE" + - ") timestamp(time) partition by DAY", + "deviceName SYMBOL capacity 1000 index, " + + "time TIMESTAMP, " + + "slot SYMBOL, " + + "port SYMBOL, " + + "downStream DOUBLE, " + + "upStream DOUBLE" + + ") timestamp(time) partition by DAY", sqlExecutionContext); compiler.compile("create table src as (" + - " select rnd_symbol(15000, 4,4,0) sym, " + - " timestamp_sequence(0, 100000) ts, " + - " rnd_double() val " + - " from long_sequence(500)" + - ")", + " select rnd_symbol(15000, 4,4,0) sym, " + + " timestamp_sequence(0, 100000) ts, " + + " rnd_double() val " + + " from long_sequence(500)" + + ")", sqlExecutionContext); compiler.compile("insert into alcatel_traffic_tmp select sym, ts, sym, null, val, val from src", sqlExecutionContext); @@ -721,6 +721,38 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest { ); } + @Test + public void testEqGeoHashWhenOtherIsStr1() throws Exception { + assertMemoryLeak( + () -> { + createGeoHashTable(4); + assertQuery("time\tuuid\thash\n" + + "2021-05-10T23:59:59.439000Z\tbbb\tewef\n", + "select * from pos where hash = 'ewef'", + "time", + true, + true, + true + ); + }); + } + + @Test + public void testEqGeoHashWhenOtherIsStr2() throws Exception { + assertMemoryLeak( + () -> { + createGeoHashTable(4); + assertQuery("time\tuuid\thash\n" + + "2021-05-10T23:59:59.439000Z\tbbb\tewef\n", + "select * from pos where 'ewef' = hash", + "time", + true, + true, + true + ); + }); + } + @Test public void testFilterAPI() throws Exception { TestMatchFunctionFactory.clear(); @@ -1954,6 +1986,226 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest { ); } + @Test + public void testLatestByAllIndexed() throws Exception { + final String expected = "a\tb\tk\n" + + "23.90529010846525\tRXGZ\t1970-01-03T07:33:20.000000Z\n" + + "12.026122412833129\tHYRX\t1970-01-11T10:00:00.000000Z\n" + + "48.820511018586934\tVTJW\t1970-01-12T13:46:40.000000Z\n" + + "49.00510449885239\tPEHN\t1970-01-18T08:40:00.000000Z\n" + + "40.455469747939254\t\t1970-01-22T23:46:40.000000Z\n"; + assertQuery(expected, + "select * from x latest by b", + "create table x as " + + "(" + + "select" + + " rnd_double(0)*100 a," + + " rnd_symbol(5,4,4,1) b," + + " timestamp_sequence(0, 100000000000) k" + + " from" + + " long_sequence(20)" + + "), index(b) timestamp(k) partition by DAY", + "k", + "insert into x select * from (" + + " select" + + " rnd_double(0)*100," + + " 'VTJW'," + + " to_timestamp('2019', 'yyyy') t" + + " from long_sequence(1)" + + ") timestamp (t)", + "a\tb\tk\n" + + "23.90529010846525\tRXGZ\t1970-01-03T07:33:20.000000Z\n" + + "12.026122412833129\tHYRX\t1970-01-11T10:00:00.000000Z\n" + + "49.00510449885239\tPEHN\t1970-01-18T08:40:00.000000Z\n" + + "40.455469747939254\t\t1970-01-22T23:46:40.000000Z\n" + + "56.594291398612405\tVTJW\t2019-01-01T00:00:00.000000Z\n", + true, + true, + true + ); + } + + @Test + public void testLatestByAllIndexedConstantFilter() throws Exception { + final String expected = "a\tb\tk\n" + + "23.90529010846525\tRXGZ\t1970-01-03T07:33:20.000000Z\n" + + "12.026122412833129\tHYRX\t1970-01-11T10:00:00.000000Z\n" + + "48.820511018586934\tVTJW\t1970-01-12T13:46:40.000000Z\n" + + "49.00510449885239\tPEHN\t1970-01-18T08:40:00.000000Z\n" + + "40.455469747939254\t\t1970-01-22T23:46:40.000000Z\n"; + assertQuery(expected, + "select * from x latest by b where 5 > 2", + "create table x as " + + "(" + + "select" + + " rnd_double(0)*100 a," + + " rnd_symbol(5,4,4,1) b," + + " timestamp_sequence(0, 100000000000) k" + + " from long_sequence(20)" + + "), index(b) timestamp(k) partition by DAY", + "k", + "insert into x select * from (" + + " select" + + " rnd_double(0)*100," + + " 'VTJW'," + + " to_timestamp('2019', 'yyyy') t" + + " from long_sequence(1)" + + ") timestamp (t)", + "a\tb\tk\n" + + "23.90529010846525\tRXGZ\t1970-01-03T07:33:20.000000Z\n" + + "12.026122412833129\tHYRX\t1970-01-11T10:00:00.000000Z\n" + + "49.00510449885239\tPEHN\t1970-01-18T08:40:00.000000Z\n" + + "40.455469747939254\t\t1970-01-22T23:46:40.000000Z\n" + + "56.594291398612405\tVTJW\t2019-01-01T00:00:00.000000Z\n", + true, + true, + true + ); + } + + @Test + public void testLatestByAllIndexedFilter() throws Exception { + final String expected = "a\tk\tb\n" + + "78.83065830055033\t1970-01-04T11:20:00.000000Z\tVTJW\n" + + "51.85631921367574\t1970-01-19T12:26:40.000000Z\tCPSW\n" + + "50.25890936351257\t1970-01-20T16:13:20.000000Z\tRXGZ\n" + + "72.604681060764\t1970-01-22T23:46:40.000000Z\t\n"; + assertQuery(expected, + "select a,k,b from x latest by b where a > 40", + "create table x as " + + "(" + + "select" + + " timestamp_sequence(0, 100000000000) k," + + " rnd_double(0)*100 a1," + + " rnd_double(0)*100 a2," + + " rnd_double(0)*100 a3," + + " rnd_double(0)*100 a," + + " rnd_symbol(5,4,4,1) b" + + " from long_sequence(20)" + + "), index(b) timestamp(k) partition by DAY", + "k", + "insert into x select * from (" + + " select" + + " to_timestamp('2019', 'yyyy') t," + + " rnd_double(0)*100," + + " rnd_double(0)*100," + + " rnd_double(0)*100," + + " 46.578761277152225," + + " 'VTJW'" + + " from long_sequence(1)" + + ") timestamp (t)", + "a\tk\tb\n" + + "51.85631921367574\t1970-01-19T12:26:40.000000Z\tCPSW\n" + + "50.25890936351257\t1970-01-20T16:13:20.000000Z\tRXGZ\n" + + "72.604681060764\t1970-01-22T23:46:40.000000Z\t\n" + + "46.578761277152225\t2019-01-01T00:00:00.000000Z\tVTJW\n", + true, + true, + true + ); + } + + @Test + public void testLatestByAllIndexedFilterBySymbol() throws Exception { + final String expected = "a\tb\tc\tk\n" + + "67.52509547112409\tCPSW\tSXUX\t1970-01-21T20:00:00.000000Z\n"; + assertQuery(expected, + "select * from x latest by b where c = 'SXUX'", + "create table x as " + + "(" + + "select" + + " rnd_double(0)*100 a," + + " rnd_symbol(5,4,4,1) b," + + " rnd_symbol(5,4,4,1) c," + + " timestamp_sequence(0, 100000000000) k" + + " from long_sequence(20)" + + "), index(b) timestamp(k) partition by DAY", + "k", + "insert into x select * from (" + + " select" + + " rnd_double(0)*100," + + " 'VTJW'," + + " 'SXUX'," + + " to_timestamp('2019', 'yyyy') t" + + " from long_sequence(1)" + + ") timestamp (t)", + "a\tb\tc\tk\n" + + "94.41658975532606\tVTJW\tSXUX\t2019-01-01T00:00:00.000000Z\n", + true, + true, + true + ); + } + + @Test + public void testLatestByAllIndexedFilterColumnDereference() throws Exception { + final String expected = "b\tk\n" + + "RXGZ\t1970-01-12T13:46:40.000000Z\n"; + assertQuery(expected, + "select b,k from x latest by b where b = 'RXGZ' and k < '1970-01-22'", + "create table x as " + + "(" + + "select" + + " timestamp_sequence(0, 100000000000) k," + + " rnd_symbol(5,4,4,1) b," + + " rnd_double(0)*100 a1," + + " rnd_double(0)*100 a2" + + " from long_sequence(20)" + + ") timestamp(k) partition by DAY", + "k", + true, + true, + false + ); + } + + @Test + public void testLatestByAllIndexedFilteredMultiplePartitions() throws Exception { + assertMemoryLeak( + () -> { + compiler.compile("create table trips(id int, vendor symbol index, ts timestamp) timestamp(ts) partition by DAY", sqlExecutionContext); + // insert three partitions + compiler.compile( + "insert into trips select " + + "rnd_int(), " + + "rnd_symbol('KK','ZZ', 'TT'), " + + "timestamp_sequence(0, 100000L) " + + "from long_sequence(1000)", + sqlExecutionContext + ); + + // cast('1970-01-02' as timestamp) produces incorrect timestamp + compiler.compile( + "insert into trips select " + + "rnd_int(), " + + "rnd_symbol('DD','QQ', 'TT'), " + + "timestamp_sequence(to_timestamp('1970-01-02', 'yyyy-MM-dd'), 100000L) " + + "from long_sequence(1000)", + sqlExecutionContext + ); + + compiler.compile( + "insert into trips select " + + "rnd_int(), " + + "rnd_symbol('PP','QQ', 'CC'), " + + "timestamp_sequence(to_timestamp('1970-01-03', 'yyyy-MM-dd'), 100000L) " + + "from long_sequence(1000)", + sqlExecutionContext + ); + + TestUtils.assertSql( + compiler, + sqlExecutionContext, + "trips latest by vendor where id > 0 order by ts", + sink, + "id\tvendor\tts\n" + + "1878619626\tKK\t1970-01-01T00:01:39.200000Z\n" + + "371958898\tDD\t1970-01-02T00:01:39.900000Z\n" + + "1699760758\tPP\t1970-01-03T00:01:39.100000Z\n" + ); + } + ); + } @Test public void testLatestByAllIndexedGeoHash1c() throws Exception { @@ -2010,100 +2262,7 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest { } @Test - public void testLatestByAllIndexedGeoHashOutOfRangeFn() throws Exception { - assertMemoryLeak( - () -> { - createGeoHashTable(2); - try { - assertQuery("time\tuuid\thash\n" + - "2021-05-10T23:59:59.150000Z\tXXX\tf9\n" + - "2021-05-11T00:00:00.083000Z\tYYY\tz3\n" + - "2021-05-12T00:00:00.186000Z\tZZZ\tve\n", - "select * from pos latest by uuid where hash within(make_geohash(-620.0, 53.4, 10), #z3, #ve)", - "time", - true, - true, - true - ); - } catch (SqlException ex) { - TestUtils.assertContains(ex.getFlyweightMessage(), "longitude must be in [-180.0..180.0] range"); - } - }); - } - - @Test - public void testLatestByAllIndexedGeoHashStrCast() throws Exception { - assertMemoryLeak( - () -> { - createGeoHashTable(2); - assertQuery("time\tuuid\thash\n" + - "2021-05-10T23:59:59.150000Z\tXXX\tf9\n" + - "2021-05-11T00:00:00.083000Z\tYYY\tz3\n" + - "2021-05-12T00:00:00.186000Z\tZZZ\tve\n", - "select * from pos latest by uuid where hash within(cast('f9' as geohash(2c)), #z3, #ve)", - "time", - true, - true, - true - ); - }); - } - - @Test - public void testLatestByAllIndexedGeoHashLiteralExpected() throws Exception { - assertMemoryLeak( - () -> { - createGeoHashTable(2); - try { - assertQuery("time\tuuid\thash\n" + - "2021-05-10T23:59:59.150000Z\tXXX\tf9\n" + - "2021-05-11T00:00:00.083000Z\tYYY\tz3\n" + - "2021-05-12T00:00:00.186000Z\tZZZ\tve\n", - "select * from pos latest by uuid where hash within('z3', #z3, #ve)", - "time", - true, - true, - true - ); - } catch (SqlException ex) { - TestUtils.assertContains(ex.getFlyweightMessage(), "GeoHash literal expected"); - } - }); - } - - @Test - public void testLatestByAllIndexedGeoHashFnNonConst() throws Exception { - assertMemoryLeak( - () -> { - compiler.compile( - "create table x as (" + - "select" + - " rnd_symbol(113, 4, 4, 2) s," + - " timestamp_sequence(500000000000L,100000000L) ts," + - " (rnd_double()*360.0 - 180.0) lon, " + - " (rnd_double()*180.0 - 90.0) lat, " + - " rnd_geohash(40) geo8" + - " from long_sequence(1000)" + - "), index(s) timestamp (ts) partition by DAY", - sqlExecutionContext - ); - try { - assertQuery("time\tuuid\thash\n", - "select * from x latest by s where geo8 within(make_geohash(lon, lat, 40), #z3, #vegg)", - "ts", - true, - true, - true - ); - } catch (SqlException ex) { - TestUtils.assertContains(ex.getFlyweightMessage(), "GeoHash const function expected"); - } - }); - } - - - @Test - public void testLatestByAllIndexedGeoHash4c() throws Exception { + public void testLatestByAllIndexedGeoHash4c() throws Exception { assertMemoryLeak( () -> { createGeoHashTable(4); @@ -2158,128 +2317,173 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest { } @Test - public void testLatestByAllIndexedGeoHashWithinEmpty() throws Exception { + public void testLatestByAllIndexedGeoHashFnNonConst() throws Exception { assertMemoryLeak( () -> { - createGeoHashTable(2); + compiler.compile( + "create table x as (" + + "select" + + " rnd_symbol(113, 4, 4, 2) s," + + " timestamp_sequence(500000000000L,100000000L) ts," + + " (rnd_double()*360.0 - 180.0) lon, " + + " (rnd_double()*180.0 - 90.0) lat, " + + " rnd_geohash(40) geo8" + + " from long_sequence(1000)" + + "), index(s) timestamp (ts) partition by DAY", + sqlExecutionContext + ); try { - assertQuery("", - "select * from pos latest by uuid where hash within()", - "time", + assertQuery("time\tuuid\thash\n", + "select * from x latest by s where geo8 within(make_geohash(lon, lat, 40), #z3, #vegg)", + "ts", true, true, true ); } catch (SqlException ex) { - TestUtils.assertContains(ex.getFlyweightMessage(), "Too few arguments for 'within'"); + TestUtils.assertContains(ex.getFlyweightMessage(), "GeoHash const function expected"); } }); } @Test - public void testLatestByAllIndexedGeoHashWithinOr() throws Exception { + public void testLatestByAllIndexedGeoHashLiteralExpected() throws Exception { assertMemoryLeak( () -> { createGeoHashTable(2); try { - assertQuery("", - "select * from pos latest by uuid where hash within(#f9) or hash within(#z3)", + assertQuery("time\tuuid\thash\n" + + "2021-05-10T23:59:59.150000Z\tXXX\tf9\n" + + "2021-05-11T00:00:00.083000Z\tYYY\tz3\n" + + "2021-05-12T00:00:00.186000Z\tZZZ\tve\n", + "select * from pos latest by uuid where hash within('z3', #z3, #ve)", "time", true, true, true ); } catch (SqlException ex) { - TestUtils.assertContains(ex.getFlyweightMessage(), "Multiple 'within' expressions not supported"); + TestUtils.assertContains(ex.getFlyweightMessage(), "GeoHash literal expected"); } }); } @Test - public void testLatestByAllIndexedGeoHashWithinColumnWrongType() throws Exception { + public void testLatestByAllIndexedGeoHashOutOfRangeFn() throws Exception { assertMemoryLeak( () -> { createGeoHashTable(2); try { - assertQuery("", - "select * from pos latest by uuid where uuid within(#f9)", + assertQuery("time\tuuid\thash\n" + + "2021-05-10T23:59:59.150000Z\tXXX\tf9\n" + + "2021-05-11T00:00:00.083000Z\tYYY\tz3\n" + + "2021-05-12T00:00:00.186000Z\tZZZ\tve\n", + "select * from pos latest by uuid where hash within(make_geohash(-620.0, 53.4, 10), #z3, #ve)", "time", true, true, true ); } catch (SqlException ex) { - TestUtils.assertContains(ex.getFlyweightMessage(), "GeoHash column type expected"); + TestUtils.assertContains(ex.getFlyweightMessage(), "longitude must be in [-180.0..180.0] range"); } }); } @Test - public void testLatestByAllIndexedGeoHashWithinColumnNotLiteral() throws Exception { + public void testLatestByAllIndexedGeoHashRnd1c() throws Exception { assertMemoryLeak( () -> { - createGeoHashTable(2); - try { - assertQuery("", - "select * from pos latest by uuid where 'hash' within(#f9)", - "time", - true, - true, - true - ); - } catch (SqlException ex) { - TestUtils.assertContains(ex.getFlyweightMessage(), "unexpected token:"); - } + createRndGeoHashTable(); + assertQuery( + "geo1\tts\n" + + "x\t1970-01-17T21:43:20.000000Z\n" + + "x\t1970-01-18T02:38:20.000000Z\n" + + "y\t1970-01-18T03:03:20.000000Z\n" + + "x\t1970-01-18T03:06:40.000000Z\n" + + "y\t1970-01-18T05:53:20.000000Z\n" + + "y\t1970-01-18T07:41:40.000000Z\n" + + "y\t1970-01-18T08:18:20.000000Z\n" + + "z\t1970-01-18T08:35:00.000000Z\n", + "select geo1, ts from x latest by s where geo1 within(#x, #y, #z)", + "ts", + true, + true, + true + ); }); } @Test - public void testLatestByAllIndexedGeoHashWithinNullArg() throws Exception { + public void testLatestByAllIndexedGeoHashRnd2c() throws Exception { assertMemoryLeak( () -> { - createGeoHashTable(2); - try { - assertQuery("", - "select * from pos latest by uuid where hash within(#f9, #z3, null)", - "time", - true, - true, - true - ); - } catch (SqlException ex) { - TestUtils.assertContains(ex.getFlyweightMessage(), "GeoHash value expected"); - } + createRndGeoHashTable(); + assertQuery( + "geo2\tts\n" + + "z7g\t1970-01-17T18:45:00.000000Z\n" + + "xzu\t1970-01-17T21:06:40.000000Z\n" + + "yyg\t1970-01-18T01:36:40.000000Z\n" + + "yds\t1970-01-18T01:56:40.000000Z\n" + + "yjx\t1970-01-18T05:03:20.000000Z\n" + + "ymx\t1970-01-18T05:53:20.000000Z\n" + + "y8x\t1970-01-18T06:45:00.000000Z\n" + + "y25\t1970-01-18T06:48:20.000000Z\n" + + "yvh\t1970-01-18T06:55:00.000000Z\n" + + "y1n\t1970-01-18T07:28:20.000000Z\n" + + "zs4\t1970-01-18T08:03:20.000000Z\n", + "select geo2, ts from x latest by s where geo2 within(#x, #y, #z)", + "ts", + true, + true, + true + ); }); } @Test - public void testLatestByAllIndexedGeoHashWithinWrongCast() throws Exception { + public void testLatestByAllIndexedGeoHashRnd4c() throws Exception { assertMemoryLeak( () -> { - createGeoHashTable(2); - try { - assertQuery("", - "select * from pos latest by uuid where hash within(cast('f91t' as geohash(4c)), #z3, null)", - "time", - true, - true, - true - ); - } catch (SqlException ex) { - TestUtils.assertContains(ex.getFlyweightMessage(), "prefix precision mismatch"); - } + createRndGeoHashTable(); + assertQuery( + "geo4\tts\n" + + "zd4gu\t1970-01-17T20:06:40.000000Z\n" + + "xwnjg\t1970-01-18T01:36:40.000000Z\n" + + "yv6gp\t1970-01-18T02:48:20.000000Z\n" + + "z4wbx\t1970-01-18T05:51:40.000000Z\n" + + "zejr0\t1970-01-18T06:43:20.000000Z\n" + + "ybsge\t1970-01-18T06:45:00.000000Z\n" + + "zdhfv\t1970-01-18T06:53:20.000000Z\n" + + "z4t7w\t1970-01-18T07:45:00.000000Z\n" + + "xxusm\t1970-01-18T07:55:00.000000Z\n" + + "x1dse\t1970-01-18T08:18:20.000000Z\n" + + "zmt6j\t1970-01-18T08:38:20.000000Z\n", + "select geo4, ts from x latest by s where geo4 within(#x, #y, #z)", + "ts", + true, + true, + true + ); }); } @Test - public void testEqGeoHashWhenOtherIsStr1() throws Exception { + public void testLatestByAllIndexedGeoHashRnd6Bits() throws Exception { assertMemoryLeak( () -> { - createGeoHashTable(4); - assertQuery("time\tuuid\thash\n" + - "2021-05-10T23:59:59.439000Z\tbbb\tewef\n", - "select * from pos where hash = 'ewef'", - "time", + createRndGeoHashBitsTable(); + assertQuery( + "bits7\tts\n" + + "1111111\t1970-01-16T21:43:20.000000Z\n" + + "1111111\t1970-01-18T00:50:00.000000Z\n" + + "1111111\t1970-01-18T00:55:00.000000Z\n" + + "1111110\t1970-01-18T05:11:40.000000Z\n" + + "1111110\t1970-01-18T07:10:00.000000Z\n" + + "1111110\t1970-01-18T08:20:00.000000Z\n" + + "1111111\t1970-01-18T08:28:20.000000Z\n", + "select bits7, ts from x latest by s where bits7 within(##111111)", + "ts", true, true, true @@ -2288,13 +2492,73 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest { } @Test - public void testEqGeoHashWhenOtherIsStr2() throws Exception { + public void testLatestByAllIndexedGeoHashRnd8c() throws Exception { assertMemoryLeak( () -> { - createGeoHashTable(4); + createRndGeoHashTable(); + assertQuery( + "geo4\tts\n" + + "yv6gp\t1970-01-18T02:48:20.000000Z\n" + + "z4wbx\t1970-01-18T05:51:40.000000Z\n" + + "ybsge\t1970-01-18T06:45:00.000000Z\n" + + "z4t7w\t1970-01-18T07:45:00.000000Z\n" + + "xxusm\t1970-01-18T07:55:00.000000Z\n", + "select geo4, ts from x latest by s where geo4 within(#xx, #y, #z4)", + "ts", + true, + true, + true + ); + }); + } + + @Test + public void testLatestByAllIndexedGeoHashRndLongBitsMask() throws Exception { + assertMemoryLeak( + () -> { + createRndGeoHashBitsTable(); + assertQuery("i\ts\tts\tbits3\tbits7\tbits9\n" + + "9384\tYFFD\t1970-01-17T15:31:40.000000Z\t101\t1110000\t101111011\n" + + "9397\tMXUK\t1970-01-17T15:53:20.000000Z\t100\t1110001\t110001111\n", + "select * from x latest by s where bits7 within(#wt/5)",//(##11100)", + "ts", + true, + true, + true + ); + }); + } + + @Test + public void testLatestByAllIndexedGeoHashRndLongBitsPrefix() throws Exception { + assertMemoryLeak( + () -> { + createRndGeoHashBitsTable(); + try { + assertQuery("", + "select * from x latest by s where bits3 within(##111111)", + "ts", + true, + true, + true + ); + } catch (SqlException ex) { + TestUtils.assertContains(ex.getFlyweightMessage(), "prefix precision mismatch"); + } + + }); + } + + @Test + public void testLatestByAllIndexedGeoHashStrCast() throws Exception { + assertMemoryLeak( + () -> { + createGeoHashTable(2); assertQuery("time\tuuid\thash\n" + - "2021-05-10T23:59:59.439000Z\tbbb\tewef\n", - "select * from pos where 'ewef' = hash", + "2021-05-10T23:59:59.150000Z\tXXX\tf9\n" + + "2021-05-11T00:00:00.083000Z\tYYY\tz3\n" + + "2021-05-12T00:00:00.186000Z\tZZZ\tve\n", + "select * from pos latest by uuid where hash within(cast('f9' as geohash(2c)), #z3, #ve)", "time", true, true, @@ -2375,316 +2639,182 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest { } @Test - public void testLatestByAllIndexedGeoHashRnd6Bits() throws Exception { + public void testLatestByAllIndexedGeoHashWithinColumnNotLiteral() throws Exception { assertMemoryLeak( () -> { - createRndGeoHashBitsTable(); - assertQuery( - "bits7\tts\n" + - "1111111\t1970-01-16T21:43:20.000000Z\n" + - "1111111\t1970-01-18T00:50:00.000000Z\n" + - "1111111\t1970-01-18T00:55:00.000000Z\n" + - "1111110\t1970-01-18T05:11:40.000000Z\n" + - "1111110\t1970-01-18T07:10:00.000000Z\n" + - "1111110\t1970-01-18T08:20:00.000000Z\n" + - "1111111\t1970-01-18T08:28:20.000000Z\n", - "select bits7, ts from x latest by s where bits7 within(##111111)", - "ts", - true, - true, - true - ); + createGeoHashTable(2); + try { + assertQuery("", + "select * from pos latest by uuid where 'hash' within(#f9)", + "time", + true, + true, + true + ); + } catch (SqlException ex) { + TestUtils.assertContains(ex.getFlyweightMessage(), "unexpected token:"); + } }); } @Test - public void testLatestByAllIndexedGeoHashRndLongBitsPrefix() throws Exception { + public void testLatestByAllIndexedGeoHashWithinColumnWrongType() throws Exception { assertMemoryLeak( () -> { - createRndGeoHashBitsTable(); + createGeoHashTable(2); try { assertQuery("", - "select * from x latest by s where bits3 within(##111111)", - "ts", + "select * from pos latest by uuid where uuid within(#f9)", + "time", true, true, true ); } catch (SqlException ex) { - TestUtils.assertContains(ex.getFlyweightMessage(), "prefix precision mismatch"); + TestUtils.assertContains(ex.getFlyweightMessage(), "GeoHash column type expected"); } - }); } @Test - public void testLatestByAllIndexedGeoHashRndLongBitsMask() throws Exception { + public void testLatestByAllIndexedGeoHashWithinEmpty() throws Exception { assertMemoryLeak( () -> { - createRndGeoHashBitsTable(); - assertQuery("i\ts\tts\tbits3\tbits7\tbits9\n" + - "9384\tYFFD\t1970-01-17T15:31:40.000000Z\t101\t1110000\t101111011\n" + - "9397\tMXUK\t1970-01-17T15:53:20.000000Z\t100\t1110001\t110001111\n", - "select * from x latest by s where bits7 within(#wt/5)",//(##11100)", - "ts", - true, - true, - true - ); + createGeoHashTable(2); + try { + assertQuery("", + "select * from pos latest by uuid where hash within()", + "time", + true, + true, + true + ); + } catch (SqlException ex) { + TestUtils.assertContains(ex.getFlyweightMessage(), "Too few arguments for 'within'"); + } }); } @Test - public void testLatestByAllIndexedGeoHashRnd1c() throws Exception { + public void testLatestByAllIndexedGeoHashWithinNullArg() throws Exception { assertMemoryLeak( () -> { - createRndGeoHashTable(); - assertQuery( - "geo1\tts\n" + - "x\t1970-01-17T21:43:20.000000Z\n" + - "x\t1970-01-18T02:38:20.000000Z\n" + - "y\t1970-01-18T03:03:20.000000Z\n" + - "x\t1970-01-18T03:06:40.000000Z\n" + - "y\t1970-01-18T05:53:20.000000Z\n" + - "y\t1970-01-18T07:41:40.000000Z\n" + - "y\t1970-01-18T08:18:20.000000Z\n" + - "z\t1970-01-18T08:35:00.000000Z\n", - "select geo1, ts from x latest by s where geo1 within(#x, #y, #z)", - "ts", - true, - true, - true - ); + createGeoHashTable(2); + try { + assertQuery("", + "select * from pos latest by uuid where hash within(#f9, #z3, null)", + "time", + true, + true, + true + ); + } catch (SqlException ex) { + TestUtils.assertContains(ex.getFlyweightMessage(), "GeoHash value expected"); + } }); } @Test - public void testLatestByAllIndexedGeoHashRnd2c() throws Exception { + public void testLatestByAllIndexedGeoHashWithinOr() throws Exception { assertMemoryLeak( () -> { - createRndGeoHashTable(); - assertQuery( - "geo2\tts\n" + - "z7g\t1970-01-17T18:45:00.000000Z\n" + - "xzu\t1970-01-17T21:06:40.000000Z\n" + - "yyg\t1970-01-18T01:36:40.000000Z\n" + - "yds\t1970-01-18T01:56:40.000000Z\n" + - "yjx\t1970-01-18T05:03:20.000000Z\n" + - "ymx\t1970-01-18T05:53:20.000000Z\n" + - "y8x\t1970-01-18T06:45:00.000000Z\n" + - "y25\t1970-01-18T06:48:20.000000Z\n" + - "yvh\t1970-01-18T06:55:00.000000Z\n" + - "y1n\t1970-01-18T07:28:20.000000Z\n" + - "zs4\t1970-01-18T08:03:20.000000Z\n", - "select geo2, ts from x latest by s where geo2 within(#x, #y, #z)", - "ts", - true, - true, - true - ); + createGeoHashTable(2); + try { + assertQuery("", + "select * from pos latest by uuid where hash within(#f9) or hash within(#z3)", + "time", + true, + true, + true + ); + } catch (SqlException ex) { + TestUtils.assertContains(ex.getFlyweightMessage(), "Multiple 'within' expressions not supported"); + } }); } @Test - public void testLatestByAllIndexedGeoHashRnd4c() throws Exception { + public void testLatestByAllIndexedGeoHashWithinWrongCast() throws Exception { assertMemoryLeak( () -> { - createRndGeoHashTable(); - assertQuery( - "geo4\tts\n" + - "zd4gu\t1970-01-17T20:06:40.000000Z\n" + - "xwnjg\t1970-01-18T01:36:40.000000Z\n" + - "yv6gp\t1970-01-18T02:48:20.000000Z\n" + - "z4wbx\t1970-01-18T05:51:40.000000Z\n" + - "zejr0\t1970-01-18T06:43:20.000000Z\n" + - "ybsge\t1970-01-18T06:45:00.000000Z\n" + - "zdhfv\t1970-01-18T06:53:20.000000Z\n" + - "z4t7w\t1970-01-18T07:45:00.000000Z\n" + - "xxusm\t1970-01-18T07:55:00.000000Z\n" + - "x1dse\t1970-01-18T08:18:20.000000Z\n" + - "zmt6j\t1970-01-18T08:38:20.000000Z\n", - "select geo4, ts from x latest by s where geo4 within(#x, #y, #z)", - "ts", - true, - true, - true - ); + createGeoHashTable(2); + try { + assertQuery("", + "select * from pos latest by uuid where hash within(cast('f91t' as geohash(4c)), #z3, null)", + "time", + true, + true, + true + ); + } catch (SqlException ex) { + TestUtils.assertContains(ex.getFlyweightMessage(), "prefix precision mismatch"); + } }); } @Test - public void testLatestByAllIndexedGeoHashRnd8c() throws Exception { + public void testLatestByAllIndexedListMultiplePartitions() throws Exception { assertMemoryLeak( () -> { - createRndGeoHashTable(); - assertQuery( - "geo4\tts\n" + - "yv6gp\t1970-01-18T02:48:20.000000Z\n" + - "z4wbx\t1970-01-18T05:51:40.000000Z\n" + - "ybsge\t1970-01-18T06:45:00.000000Z\n" + - "z4t7w\t1970-01-18T07:45:00.000000Z\n" + - "xxusm\t1970-01-18T07:55:00.000000Z\n", - "select geo4, ts from x latest by s where geo4 within(#xx, #y, #z4)", - "ts", - true, - true, - true + compiler.compile("create table trips(id int, vendor symbol index, ts timestamp) timestamp(ts) partition by DAY", sqlExecutionContext); + // insert three partitions + compiler.compile( + "insert into trips select " + + "rnd_int(), " + + "rnd_symbol('KK','ZZ', 'TT'), " + + "timestamp_sequence(0, 100000L) " + + "from long_sequence(1000)", + sqlExecutionContext ); - }); - } - private void createRndGeoHashBitsTable() throws SqlException { - compiler.compile( - "create table x as (" + - "select" + - " cast(x as int) i," + - " rnd_symbol(113, 4, 4, 2) s," + - " timestamp_sequence(500000000000L,100000000L) ts," + - " rnd_geohash(3) bits3," + - " rnd_geohash(7) bits7," + - " rnd_geohash(9) bits9" + - " from long_sequence(10000)" + - "), index(s) timestamp (ts) partition by DAY", - sqlExecutionContext - ); - } - - private void createRndGeoHashTable() throws SqlException { - compiler.compile( - "create table x as (" + - "select" + - " cast(x as int) i," + - " rnd_symbol(113, 4, 4, 2) s," + - " timestamp_sequence(500000000000L,100000000L) ts," + - " rnd_geohash(5) geo1," + - " rnd_geohash(15) geo2," + - " rnd_geohash(25) geo4," + - " rnd_geohash(40) geo8" + - " from long_sequence(10000)" + - "), index(s) timestamp (ts) partition by DAY", - sqlExecutionContext - ); - } - - private void createGeoHashTable(int chars) throws SqlException { - compiler.compile( - String.format("create table pos(time timestamp, uuid symbol, hash geohash(%dc))", chars) + - ", index(uuid) timestamp(time) partition by DAY", - sqlExecutionContext - ); - executeInsert("insert into pos values('2021-05-10T23:59:59.150000Z','XXX','f91t48s7')"); - executeInsert("insert into pos values('2021-05-10T23:59:59.322000Z','ddd','bbqyzfp6')"); - executeInsert("insert into pos values('2021-05-10T23:59:59.351000Z','bbb','9egcyrxq')"); - executeInsert("insert into pos values('2021-05-10T23:59:59.439000Z','bbb','ewef1vk8')"); - executeInsert("insert into pos values('2021-05-10T00:00:00.016000Z','aaa','vb2wg49h')"); - executeInsert("insert into pos values('2021-05-10T00:00:00.042000Z','ccc','bft3gn89')"); - executeInsert("insert into pos values('2021-05-10T00:00:00.055000Z','aaa','z6cf5j85')"); - executeInsert("insert into pos values('2021-05-11T00:00:00.066000Z','ddd','vcunv6j7')"); - executeInsert("insert into pos values('2021-05-11T00:00:00.072000Z','ccc','edez0n5y')"); - executeInsert("insert into pos values('2021-05-11T00:00:00.074000Z','aaa','fds32zgc')"); - executeInsert("insert into pos values('2021-05-11T00:00:00.083000Z','YYY','z31wzd5w')"); - executeInsert("insert into pos values('2021-05-11T00:00:00.092000Z','ddd','v9nwc4ny')"); - executeInsert("insert into pos values('2021-05-11T00:00:00.107000Z','ccc','f6yb1yx9')"); - executeInsert("insert into pos values('2021-05-11T00:00:00.111000Z','ddd','bcnktpnw')"); - executeInsert("insert into pos values('2021-05-11T00:00:00.123000Z','aaa','z3t2we5z')"); - executeInsert("insert into pos values('2021-05-11T00:00:00.127000Z','aaa','bgn1yt4y')"); - executeInsert("insert into pos values('2021-05-11T00:00:00.144000Z','aaa','fuetk3k6')"); - executeInsert("insert into pos values('2021-05-12T00:00:00.167000Z','ccc','bchx5x14')"); - executeInsert("insert into pos values('2021-05-12T00:00:00.167000Z','ZZZ','bbxwb5jj')"); - executeInsert("insert into pos values('2021-05-12T00:00:00.186000Z','ZZZ','vepe7h62')"); - executeInsert("insert into pos values('2021-05-12T00:00:00.241000Z','bbb','bchxpmmg')"); - executeInsert("insert into pos values('2021-05-12T00:00:00.245000Z','ddd','f90z3bs5')"); - executeInsert("insert into pos values('2021-05-12T00:00:00.247000Z','bbb','bftqreuh')"); - executeInsert("insert into pos values('2021-05-12T00:00:00.295000Z','ddd','u2rqgy9s')"); - executeInsert("insert into pos values('2021-05-12T00:00:00.304000Z','aaa','w23bhjd2')"); - } + // cast('1970-01-02' as timestamp) produces incorrect timestamp + compiler.compile( + "insert into trips select " + + "rnd_int(), " + + "rnd_symbol('DD','QQ', 'TT'), " + + "timestamp_sequence(to_timestamp('1970-01-02', 'yyyy-MM-dd'), 100000L) " + + "from long_sequence(1000)", + sqlExecutionContext + ); - @Test - public void testLatestByAllIndexed() throws Exception { - final String expected = "a\tb\tk\n" + - "23.90529010846525\tRXGZ\t1970-01-03T07:33:20.000000Z\n" + - "12.026122412833129\tHYRX\t1970-01-11T10:00:00.000000Z\n" + - "48.820511018586934\tVTJW\t1970-01-12T13:46:40.000000Z\n" + - "49.00510449885239\tPEHN\t1970-01-18T08:40:00.000000Z\n" + - "40.455469747939254\t\t1970-01-22T23:46:40.000000Z\n"; - assertQuery(expected, - "select * from x latest by b", - "create table x as " + - "(" + - "select" + - " rnd_double(0)*100 a," + - " rnd_symbol(5,4,4,1) b," + - " timestamp_sequence(0, 100000000000) k" + - " from" + - " long_sequence(20)" + - "), index(b) timestamp(k) partition by DAY", - "k", - "insert into x select * from (" + - " select" + - " rnd_double(0)*100," + - " 'VTJW'," + - " to_timestamp('2019', 'yyyy') t" + - " from long_sequence(1)" + - ") timestamp (t)", - "a\tb\tk\n" + - "23.90529010846525\tRXGZ\t1970-01-03T07:33:20.000000Z\n" + - "12.026122412833129\tHYRX\t1970-01-11T10:00:00.000000Z\n" + - "49.00510449885239\tPEHN\t1970-01-18T08:40:00.000000Z\n" + - "40.455469747939254\t\t1970-01-22T23:46:40.000000Z\n" + - "56.594291398612405\tVTJW\t2019-01-01T00:00:00.000000Z\n", - true, - true, - true - ); - } + compiler.compile( + "insert into trips select " + + "rnd_int(), " + + "rnd_symbol('PP','QQ', 'CC'), " + + "timestamp_sequence(to_timestamp('1970-01-03', 'yyyy-MM-dd'), 100000L) " + + "from long_sequence(1000)", + sqlExecutionContext + ); - @Test - public void testLatestByAllIndexedConstantFilter() throws Exception { - final String expected = "a\tb\tk\n" + - "23.90529010846525\tRXGZ\t1970-01-03T07:33:20.000000Z\n" + - "12.026122412833129\tHYRX\t1970-01-11T10:00:00.000000Z\n" + - "48.820511018586934\tVTJW\t1970-01-12T13:46:40.000000Z\n" + - "49.00510449885239\tPEHN\t1970-01-18T08:40:00.000000Z\n" + - "40.455469747939254\t\t1970-01-22T23:46:40.000000Z\n"; - assertQuery(expected, - "select * from x latest by b where 5 > 2", - "create table x as " + - "(" + - "select" + - " rnd_double(0)*100 a," + - " rnd_symbol(5,4,4,1) b," + - " timestamp_sequence(0, 100000000000) k" + - " from long_sequence(20)" + - "), index(b) timestamp(k) partition by DAY", - "k", - "insert into x select * from (" + - " select" + - " rnd_double(0)*100," + - " 'VTJW'," + - " to_timestamp('2019', 'yyyy') t" + - " from long_sequence(1)" + - ") timestamp (t)", - "a\tb\tk\n" + - "23.90529010846525\tRXGZ\t1970-01-03T07:33:20.000000Z\n" + - "12.026122412833129\tHYRX\t1970-01-11T10:00:00.000000Z\n" + - "49.00510449885239\tPEHN\t1970-01-18T08:40:00.000000Z\n" + - "40.455469747939254\t\t1970-01-22T23:46:40.000000Z\n" + - "56.594291398612405\tVTJW\t2019-01-01T00:00:00.000000Z\n", - true, - true, - true + TestUtils.assertSql( + compiler, + sqlExecutionContext, + "trips latest by vendor where vendor in ('KK', 'ZZ', 'TT', 'DD', 'PP', 'QQ', 'CC') order by ts", + sink, + "id\tvendor\tts\n" + + "-1243990650\tZZ\t1970-01-01T00:01:39.900000Z\n" + + "1878619626\tKK\t1970-01-01T00:01:39.200000Z\n" + + "371958898\tDD\t1970-01-02T00:01:39.900000Z\n" + + "-774731115\tTT\t1970-01-02T00:01:39.800000Z\n" + + "-1808277542\tCC\t1970-01-03T00:01:39.900000Z\n" + + "-610460127\tQQ\t1970-01-03T00:01:39.500000Z\n" + + "1699760758\tPP\t1970-01-03T00:01:39.100000Z\n" + ); + } ); } @Test - public void testLatestByAllIndexedFilter() throws Exception { + public void testLatestByAllIndexedMixed() throws Exception { final String expected = "a\tk\tb\n" + "78.83065830055033\t1970-01-04T11:20:00.000000Z\tVTJW\n" + + "2.6836863013701473\t1970-01-13T17:33:20.000000Z\tHYRX\n" + + "9.76683471072458\t1970-01-14T21:20:00.000000Z\tPEHN\n" + "51.85631921367574\t1970-01-19T12:26:40.000000Z\tCPSW\n" + "50.25890936351257\t1970-01-20T16:13:20.000000Z\tRXGZ\n" + "72.604681060764\t1970-01-22T23:46:40.000000Z\t\n"; assertQuery(expected, - "select a,k,b from x latest by b where a > 40", + "select a,k,b from x latest by b", "create table x as " + "(" + "select" + @@ -2694,7 +2824,8 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest { " rnd_double(0)*100 a3," + " rnd_double(0)*100 a," + " rnd_symbol(5,4,4,1) b" + - " from long_sequence(20)" + + " from" + + " long_sequence(20)" + "), index(b) timestamp(k) partition by DAY", "k", "insert into x select * from (" + @@ -2703,15 +2834,17 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest { " rnd_double(0)*100," + " rnd_double(0)*100," + " rnd_double(0)*100," + - " 46.578761277152225," + + " rnd_double(0)*100," + " 'VTJW'" + " from long_sequence(1)" + ") timestamp (t)", "a\tk\tb\n" + + "2.6836863013701473\t1970-01-13T17:33:20.000000Z\tHYRX\n" + + "9.76683471072458\t1970-01-14T21:20:00.000000Z\tPEHN\n" + "51.85631921367574\t1970-01-19T12:26:40.000000Z\tCPSW\n" + "50.25890936351257\t1970-01-20T16:13:20.000000Z\tRXGZ\n" + "72.604681060764\t1970-01-22T23:46:40.000000Z\t\n" + - "46.578761277152225\t2019-01-01T00:00:00.000000Z\tVTJW\n", + "6.578761277152223\t2019-01-01T00:00:00.000000Z\tVTJW\n", true, true, true @@ -2719,53 +2852,38 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest { } @Test - public void testLatestByAllIndexedFilterColumnDereference() throws Exception { - final String expected = "b\tk\n" + - "RXGZ\t1970-01-12T13:46:40.000000Z\n"; - assertQuery(expected, - "select b,k from x latest by b where b = 'RXGZ' and k < '1970-01-22'", - "create table x as " + - "(" + - "select" + - " timestamp_sequence(0, 100000000000) k," + - " rnd_symbol(5,4,4,1) b," + - " rnd_double(0)*100 a1," + - " rnd_double(0)*100 a2" + - " from long_sequence(20)" + - ") timestamp(k) partition by DAY", - "k", - true, - true, - false - ); - } - - @Test - public void testLatestByAllIndexedFilterBySymbol() throws Exception { - final String expected = "a\tb\tc\tk\n" + - "67.52509547112409\tCPSW\tSXUX\t1970-01-21T20:00:00.000000Z\n"; + public void testLatestByAllIndexedMixedColumns() throws Exception { + final String expected = "k\ta\n" + + "1970-01-03T07:33:20.000000Z\t23.90529010846525\n" + + "1970-01-11T10:00:00.000000Z\t12.026122412833129\n" + + "1970-01-12T13:46:40.000000Z\t48.820511018586934\n" + + "1970-01-18T08:40:00.000000Z\t49.00510449885239\n" + + "1970-01-22T23:46:40.000000Z\t40.455469747939254\n"; assertQuery(expected, - "select * from x latest by b where c = 'SXUX'", + "select k,a from x latest by b", "create table x as " + "(" + "select" + " rnd_double(0)*100 a," + " rnd_symbol(5,4,4,1) b," + - " rnd_symbol(5,4,4,1) c," + " timestamp_sequence(0, 100000000000) k" + - " from long_sequence(20)" + + " from" + + " long_sequence(20)" + "), index(b) timestamp(k) partition by DAY", "k", "insert into x select * from (" + " select" + " rnd_double(0)*100," + " 'VTJW'," + - " 'SXUX'," + " to_timestamp('2019', 'yyyy') t" + " from long_sequence(1)" + ") timestamp (t)", - "a\tb\tc\tk\n" + - "94.41658975532606\tVTJW\tSXUX\t2019-01-01T00:00:00.000000Z\n", + "k\ta\n" + + "1970-01-03T07:33:20.000000Z\t23.90529010846525\n" + + "1970-01-11T10:00:00.000000Z\t12.026122412833129\n" + + "1970-01-18T08:40:00.000000Z\t49.00510449885239\n" + + "1970-01-22T23:46:40.000000Z\t40.455469747939254\n" + + "2019-01-01T00:00:00.000000Z\t56.594291398612405\n", true, true, true @@ -2773,7 +2891,7 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest { } @Test - public void testLatestByAllIndexedFilteredMultiplePartitions() throws Exception { + public void testLatestByAllIndexedMultiplePartitions() throws Exception { assertMemoryLeak( () -> { compiler.compile("create table trips(id int, vendor symbol index, ts timestamp) timestamp(ts) partition by DAY", sqlExecutionContext); @@ -2809,202 +2927,16 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest { TestUtils.assertSql( compiler, sqlExecutionContext, - "trips latest by vendor where id > 0 order by ts", + "trips latest by vendor order by ts", sink, "id\tvendor\tts\n" + "1878619626\tKK\t1970-01-01T00:01:39.200000Z\n" + + "-1243990650\tZZ\t1970-01-01T00:01:39.900000Z\n" + + "-774731115\tTT\t1970-01-02T00:01:39.800000Z\n" + "371958898\tDD\t1970-01-02T00:01:39.900000Z\n" + - "1699760758\tPP\t1970-01-03T00:01:39.100000Z\n" - ); - } - ); - } - - @Test - public void testLatestByAllIndexedListMultiplePartitions() throws Exception { - assertMemoryLeak( - () -> { - compiler.compile("create table trips(id int, vendor symbol index, ts timestamp) timestamp(ts) partition by DAY", sqlExecutionContext); - // insert three partitions - compiler.compile( - "insert into trips select " + - "rnd_int(), " + - "rnd_symbol('KK','ZZ', 'TT'), " + - "timestamp_sequence(0, 100000L) " + - "from long_sequence(1000)", - sqlExecutionContext - ); - - // cast('1970-01-02' as timestamp) produces incorrect timestamp - compiler.compile( - "insert into trips select " + - "rnd_int(), " + - "rnd_symbol('DD','QQ', 'TT'), " + - "timestamp_sequence(to_timestamp('1970-01-02', 'yyyy-MM-dd'), 100000L) " + - "from long_sequence(1000)", - sqlExecutionContext - ); - - compiler.compile( - "insert into trips select " + - "rnd_int(), " + - "rnd_symbol('PP','QQ', 'CC'), " + - "timestamp_sequence(to_timestamp('1970-01-03', 'yyyy-MM-dd'), 100000L) " + - "from long_sequence(1000)", - sqlExecutionContext - ); - - TestUtils.assertSql( - compiler, - sqlExecutionContext, - "trips latest by vendor where vendor in ('KK', 'ZZ', 'TT', 'DD', 'PP', 'QQ', 'CC') order by ts", - sink, - "id\tvendor\tts\n" + - "-1243990650\tZZ\t1970-01-01T00:01:39.900000Z\n" + - "1878619626\tKK\t1970-01-01T00:01:39.200000Z\n" + - "371958898\tDD\t1970-01-02T00:01:39.900000Z\n" + - "-774731115\tTT\t1970-01-02T00:01:39.800000Z\n" + - "-1808277542\tCC\t1970-01-03T00:01:39.900000Z\n" + - "-610460127\tQQ\t1970-01-03T00:01:39.500000Z\n" + - "1699760758\tPP\t1970-01-03T00:01:39.100000Z\n" - ); - } - ); - } - - @Test - public void testLatestByAllIndexedMixed() throws Exception { - final String expected = "a\tk\tb\n" + - "78.83065830055033\t1970-01-04T11:20:00.000000Z\tVTJW\n" + - "2.6836863013701473\t1970-01-13T17:33:20.000000Z\tHYRX\n" + - "9.76683471072458\t1970-01-14T21:20:00.000000Z\tPEHN\n" + - "51.85631921367574\t1970-01-19T12:26:40.000000Z\tCPSW\n" + - "50.25890936351257\t1970-01-20T16:13:20.000000Z\tRXGZ\n" + - "72.604681060764\t1970-01-22T23:46:40.000000Z\t\n"; - assertQuery(expected, - "select a,k,b from x latest by b", - "create table x as " + - "(" + - "select" + - " timestamp_sequence(0, 100000000000) k," + - " rnd_double(0)*100 a1," + - " rnd_double(0)*100 a2," + - " rnd_double(0)*100 a3," + - " rnd_double(0)*100 a," + - " rnd_symbol(5,4,4,1) b" + - " from" + - " long_sequence(20)" + - "), index(b) timestamp(k) partition by DAY", - "k", - "insert into x select * from (" + - " select" + - " to_timestamp('2019', 'yyyy') t," + - " rnd_double(0)*100," + - " rnd_double(0)*100," + - " rnd_double(0)*100," + - " rnd_double(0)*100," + - " 'VTJW'" + - " from long_sequence(1)" + - ") timestamp (t)", - "a\tk\tb\n" + - "2.6836863013701473\t1970-01-13T17:33:20.000000Z\tHYRX\n" + - "9.76683471072458\t1970-01-14T21:20:00.000000Z\tPEHN\n" + - "51.85631921367574\t1970-01-19T12:26:40.000000Z\tCPSW\n" + - "50.25890936351257\t1970-01-20T16:13:20.000000Z\tRXGZ\n" + - "72.604681060764\t1970-01-22T23:46:40.000000Z\t\n" + - "6.578761277152223\t2019-01-01T00:00:00.000000Z\tVTJW\n", - true, - true, - true - ); - } - - @Test - public void testLatestByAllIndexedMixedColumns() throws Exception { - final String expected = "k\ta\n" + - "1970-01-03T07:33:20.000000Z\t23.90529010846525\n" + - "1970-01-11T10:00:00.000000Z\t12.026122412833129\n" + - "1970-01-12T13:46:40.000000Z\t48.820511018586934\n" + - "1970-01-18T08:40:00.000000Z\t49.00510449885239\n" + - "1970-01-22T23:46:40.000000Z\t40.455469747939254\n"; - assertQuery(expected, - "select k,a from x latest by b", - "create table x as " + - "(" + - "select" + - " rnd_double(0)*100 a," + - " rnd_symbol(5,4,4,1) b," + - " timestamp_sequence(0, 100000000000) k" + - " from" + - " long_sequence(20)" + - "), index(b) timestamp(k) partition by DAY", - "k", - "insert into x select * from (" + - " select" + - " rnd_double(0)*100," + - " 'VTJW'," + - " to_timestamp('2019', 'yyyy') t" + - " from long_sequence(1)" + - ") timestamp (t)", - "k\ta\n" + - "1970-01-03T07:33:20.000000Z\t23.90529010846525\n" + - "1970-01-11T10:00:00.000000Z\t12.026122412833129\n" + - "1970-01-18T08:40:00.000000Z\t49.00510449885239\n" + - "1970-01-22T23:46:40.000000Z\t40.455469747939254\n" + - "2019-01-01T00:00:00.000000Z\t56.594291398612405\n", - true, - true, - true - ); - } - - @Test - public void testLatestByAllIndexedMultiplePartitions() throws Exception { - assertMemoryLeak( - () -> { - compiler.compile("create table trips(id int, vendor symbol index, ts timestamp) timestamp(ts) partition by DAY", sqlExecutionContext); - // insert three partitions - compiler.compile( - "insert into trips select " + - "rnd_int(), " + - "rnd_symbol('KK','ZZ', 'TT'), " + - "timestamp_sequence(0, 100000L) " + - "from long_sequence(1000)", - sqlExecutionContext - ); - - // cast('1970-01-02' as timestamp) produces incorrect timestamp - compiler.compile( - "insert into trips select " + - "rnd_int(), " + - "rnd_symbol('DD','QQ', 'TT'), " + - "timestamp_sequence(to_timestamp('1970-01-02', 'yyyy-MM-dd'), 100000L) " + - "from long_sequence(1000)", - sqlExecutionContext - ); - - compiler.compile( - "insert into trips select " + - "rnd_int(), " + - "rnd_symbol('PP','QQ', 'CC'), " + - "timestamp_sequence(to_timestamp('1970-01-03', 'yyyy-MM-dd'), 100000L) " + - "from long_sequence(1000)", - sqlExecutionContext - ); - - TestUtils.assertSql( - compiler, - sqlExecutionContext, - "trips latest by vendor order by ts", - sink, - "id\tvendor\tts\n" + - "1878619626\tKK\t1970-01-01T00:01:39.200000Z\n" + - "-1243990650\tZZ\t1970-01-01T00:01:39.900000Z\n" + - "-774731115\tTT\t1970-01-02T00:01:39.800000Z\n" + - "371958898\tDD\t1970-01-02T00:01:39.900000Z\n" + - "1699760758\tPP\t1970-01-03T00:01:39.100000Z\n" + - "-610460127\tQQ\t1970-01-03T00:01:39.500000Z\n" + - "-1808277542\tCC\t1970-01-03T00:01:39.900000Z\n" + "1699760758\tPP\t1970-01-03T00:01:39.100000Z\n" + + "-610460127\tQQ\t1970-01-03T00:01:39.500000Z\n" + + "-1808277542\tCC\t1970-01-03T00:01:39.900000Z\n" ); } ); @@ -3141,6 +3073,78 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest { ); } + @Test + public void testLatestByAllValueIndexedColumn() throws Exception { + assertMemoryLeak(() -> { + compiler.compile("create table balances(\n" + + "cust_id SYMBOL index,\n" + + "balance_ccy SYMBOL,\n" + + "balance DOUBLE,\n" + + "timestamp TIMESTAMP\n" + + ")\n" + + "timestamp(timestamp)", sqlExecutionContext); + + executeInsert("insert into balances values ('c1', 'USD', 1500, '2021-09-14T17:35:01.000000Z')"); + executeInsert("insert into balances values ('c1', 'USD', 900.75, '2021-09-14T17:35:02.000000Z')"); + executeInsert("insert into balances values ('c1', 'EUR', 880.2, '2021-09-14T17:35:03.000000Z')"); + executeInsert("insert into balances values ('c1', 'EUR', 782, '2021-09-14T17:35:04.000000Z')"); + executeInsert("insert into balances values ('c2', 'USD', 900, '2021-09-14T17:35:05.000000Z')"); + executeInsert("insert into balances values ('c2', 'USD', 190.75, '2021-09-14T17:35:06.000000Z')"); + executeInsert("insert into balances values ('c2', 'EUR', 890.2, '2021-09-14T17:35:07.000000Z')"); + executeInsert("insert into balances values ('c2', 'EUR', 1000, '2021-09-14T17:35:08.000000Z')"); + + TestUtils.assertSql( + compiler, + sqlExecutionContext, + "SELECT * FROM balances LATEST BY cust_id, balance_ccy \n" + + "WHERE cust_id = 'c1' and balance_ccy='EUR'", + sink, + "cust_id\tbalance_ccy\tbalance\ttimestamp\n" + + "c1\tEUR\t782.0\t2021-09-14T17:35:04.000000Z\n" + ); + }); + } + + @Test + public void testLatestByFilteredBySymbolInAllIndexed() throws Exception { + testLatestByFilteredBySymbolIn("create table x (\n" + + " ts timestamp,\n" + + " node symbol index,\n" + + " metric symbol index,\n" + + " value long) \n" + + " timestamp(ts) partition by day"); + } + + @Test + public void testLatestByFilteredBySymbolInNoIndexes() throws Exception { + testLatestByFilteredBySymbolIn("create table x (\n" + + " ts timestamp,\n" + + " node symbol,\n" + + " metric symbol,\n" + + " value long) \n" + + " timestamp(ts) partition by day"); + } + + @Test + public void testLatestByFilteredSymbolInPartiallyIndexed1() throws Exception { + testLatestByFilteredBySymbolIn("create table x (\n" + + " ts timestamp,\n" + + " node symbol index,\n" + + " metric symbol,\n" + + " value long) \n" + + " timestamp(ts) partition by day"); + } + + @Test + public void testLatestByFilteredSymbolInPartiallyIndexed2() throws Exception { + testLatestByFilteredBySymbolIn("create table x (\n" + + " ts timestamp,\n" + + " node symbol,\n" + + " metric symbol index,\n" + + " value long) \n" + + " timestamp(ts) partition by day"); + } + @Test public void testLatestByIOFailure() throws Exception { assertMemoryLeak(() -> { @@ -3198,13 +3202,35 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest { }); } + @Ignore + // TODO: fix, where is applied after latest by, the optimized I suspect @Test - public void testLatestByKeyValue() throws Exception { - // no index - assertQuery("a\tb\tk\n" + - "23.90529010846525\tRXGZ\t1970-01-03T07:33:20.000000Z\n", - "select * from x latest by b where b = 'RXGZ'", - "create table x as " + + public void testLatestByIsApplicableToSubQueriesNoDesignatedTimestamp() throws Exception { + assertMemoryLeak(() -> { + compiler.compile("create table tab(" + + " id symbol, " + + " name symbol, " + + " value double, " + + " other_ts timestamp, " + + " ts timestamp" + + ")", sqlExecutionContext); + executeInsert("insert into tab values ('d1', 'c1', 101.1, '2021-10-15T11:31:35.878Z', '2021-10-05T11:31:35.878Z')"); + executeInsert("insert into tab values ('d2', 'c1', 111.7, '2021-10-16T17:31:35.878Z', '2021-10-06T15:31:35.878Z')"); + assertSql( + "(tab latest by id where name in (select distinct name from tab where name != 'c2')) timestamp(other_ts)", + "id\tname\tvalue\tother_ts\tts\n" + + "d1\tc1\t101.4\t2021-10-15 14:31:35.878\t2021-10-05 14:31:35.878\n" + + "d2\tc1\t111.7\t2021-10-16 17:31:35.878\t2021-10-06 15:31:35.878\n"); + }); + } + + @Test + public void testLatestByKeyValue() throws Exception { + // no index + assertQuery("a\tb\tk\n" + + "23.90529010846525\tRXGZ\t1970-01-03T07:33:20.000000Z\n", + "select * from x latest by b where b = 'RXGZ'", + "create table x as " + "(" + "select " + " rnd_double(0)*100 a," + @@ -3778,6 +3804,92 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest { Assert.assertTrue(TestMatchFunctionFactory.assertAPI()); } + @Test + public void testLatestByMultiColumnPlusFilter0() throws Exception { + testLatestByMultiColumnPlusFilter("create table tab(" + + " id symbol, " + + " name symbol, " + + " value double, " + + " ts timestamp" + + ")"); + } + + @Test + public void testLatestByMultiColumnPlusFilter1() throws Exception { + testLatestByMultiColumnPlusFilter("create table tab(" + + " id symbol, " + + " name symbol, " + + " value double, " + + " ts timestamp" + + ") timestamp(ts) partition by DAY"); + } + + @Ignore("LatestByAllIndexedFilteredRecordCursorFactory applies filter after latest by is executed") + @Test + public void testLatestByMultiColumnPlusFilter2() throws Exception { + testLatestByMultiColumnPlusFilter("create table tab(" + + " id symbol index, " + + " name symbol, " + + " value double, " + + " ts timestamp" + + ") timestamp(ts) partition by DAY"); + } + + @Ignore("LatestByAllIndexedFilteredRecordCursorFactory applies filter after latest by is executed") + @Test + public void testLatestByMultiColumnPlusFilter3() throws Exception { + testLatestByMultiColumnPlusFilter("create table tab(" + + " id symbol, " + + " name symbol index, " + + " value double, " + + " ts timestamp" + + ") timestamp(ts) partition by DAY"); + } + + @Ignore("LatestByAllIndexedFilteredRecordCursorFactory applies filter after latest by is executed") + @Test + public void testLatestByMultiColumnPlusFilter4() throws Exception { + testLatestByMultiColumnPlusFilter("create table tab(" + + " id symbol index, " + + " name symbol index, " + + " value double, " + + " ts timestamp" + + ") timestamp(ts) partition by DAY"); + } + + @Ignore("LatestByAllIndexedFilteredRecordCursorFactory applies filter after latest by is executed") + @Test + public void testLatestByMultiColumnPlusFilter5() throws Exception { + testLatestByMultiColumnPlusFilter("create table tab(" + + " id symbol index, " + + " name symbol index, " + + " value double, " + + " ts timestamp" + + ")"); + } + + @Ignore("LatestByAllIndexedFilteredRecordCursorFactory applies filter after latest by is executed") + @Test + public void testLatestByMultiColumnPlusFilter6() throws Exception { + testLatestByMultiColumnPlusFilter("create table tab(" + + " id symbol, " + + " name symbol index, " + + " value double, " + + " ts timestamp" + + ")"); + } + + @Ignore("LatestByAllIndexedFilteredRecordCursorFactory applies filter after latest by is executed") + @Test + public void testLatestByMultiColumnPlusFilter7() throws Exception { + testLatestByMultiColumnPlusFilter("create table tab(" + + " id symbol index, " + + " name symbol, " + + " value double, " + + " ts timestamp" + + ")"); + } + @Test public void testLatestByMultipleColumns() throws Exception { assertQuery("cust_id\tbalance_ccy\tbalance\tstatus\ttimestamp\n", @@ -3835,6 +3947,46 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest { "Invalid column"); } + @Test + public void testLatestBySelectAllFilteredBySymbolInAllIndexed() throws Exception { + testLatestBySelectAllFilteredBySymbolIn("create table x (\n" + + " ts timestamp,\n" + + " node symbol index,\n" + + " metric symbol index,\n" + + " value long) \n" + + " timestamp(ts) partition by day"); + } + + @Test + public void testLatestBySelectAllFilteredBySymbolInNoIndexes() throws Exception { + testLatestBySelectAllFilteredBySymbolIn("create table x (\n" + + " ts timestamp,\n" + + " node symbol,\n" + + " metric symbol,\n" + + " value long) \n" + + " timestamp(ts) partition by day"); + } + + @Test + public void testLatestBySelectAllFilteredBySymbolInPartiallyIndexed1() throws Exception { + testLatestBySelectAllFilteredBySymbolIn("create table x (\n" + + " ts timestamp,\n" + + " node symbol index,\n" + + " metric symbol,\n" + + " value long) \n" + + " timestamp(ts) partition by day"); + } + + @Test + public void testLatestBySelectAllFilteredBySymbolInPartiallyIndexed2() throws Exception { + testLatestBySelectAllFilteredBySymbolIn("create table x (\n" + + " ts timestamp,\n" + + " node symbol,\n" + + " metric symbol index,\n" + + " value long) \n" + + " timestamp(ts) partition by day"); + } + @Test public void testLatestBySubQuery() throws Exception { // no index @@ -4179,6 +4331,74 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest { ); } + @Test + public void testLatestBySupportedColumnTypes0() throws Exception { + testLatestBySupportedColumnTypes( + "create table tab (" + + " boolean boolean, " + + " short short, " + + " int int, " + + " long long, " + + " long256 long256, " + + " char char, " + + " string string, " + + " symbol symbol, " + + " ts timestamp" + + ")", + null); + } + + @Test + public void testLatestBySupportedColumnTypes1() throws Exception { + testLatestBySupportedColumnTypes( + "create table tab (" + + " boolean boolean, " + + " short short, " + + " int int, " + + " long long, " + + " long256 long256, " + + " char char, " + + " string string, " + + " symbol symbol, " + + " ts timestamp" + + ") timestamp(ts) partition by DAY", + "ts"); + } + + @Test + public void testLatestBySupportedColumnTypes2() throws Exception { + testLatestBySupportedColumnTypes( + "create table tab (" + + " boolean boolean, " + + " short short, " + + " int int, " + + " long long, " + + " long256 long256, " + + " char char, " + + " string string, " + + " symbol symbol index, " + + " ts timestamp" + + ")", + null); + } + + @Test + public void testLatestBySupportedColumnTypes3() throws Exception { + testLatestBySupportedColumnTypes( + "create table tab (" + + " boolean boolean, " + + " short short, " + + " int int, " + + " long long, " + + " long256 long256, " + + " char char, " + + " string string, " + + " symbol symbol index, " + + " ts timestamp" + + ") timestamp(ts) partition by DAY", + "ts"); + } + @Test public void testLatestByTimestampInclusion() throws Exception { assertQuery("ts\tmarket_type\tavg\n" + @@ -4200,259 +4420,194 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest { } @Test - public void testLatestByValue() throws Exception { - // no index - assertQuery("a\tb\tk\n" + - "65.08594025855301\tHNR\t1970-01-02T03:46:40.000000Z\n", - "select * from x latest by b where b = 'HNR'", - "create table x as " + - "(" + - "select " + - " rnd_double(0)*100 a," + - " rnd_str(2,4,4) b," + - " timestamp_sequence(0, 100000000000) k" + - " from" + - " long_sequence(20)" + - ") timestamp(k) partition by DAY", - "k", - "insert into x select * from (" + - "select" + - " rnd_double(0)*100," + - " 'HNR'," + - " to_timestamp('1971', 'yyyy') t" + - " from long_sequence(1)" + - ") timestamp(t)", - "a\tb\tk\n" + - "34.56897991538844\tHNR\t1971-01-01T00:00:00.000000Z\n", - true, - true, - false, - true); - } - - @Test - public void testLatestByAllValueIndexedColumn() throws Exception { + public void testLatestByTsIsPickedAtRuntimeNoDesignated() throws Exception { assertMemoryLeak(() -> { - compiler.compile("create table balances(\n" + - "cust_id SYMBOL index,\n" + - "balance_ccy SYMBOL,\n" + - "balance DOUBLE,\n" + - "timestamp TIMESTAMP\n" + - ")\n" + - "timestamp(timestamp)", sqlExecutionContext); - - executeInsert("insert into balances values ('c1', 'USD', 1500, '2021-09-14T17:35:01.000000Z')"); - executeInsert("insert into balances values ('c1', 'USD', 900.75, '2021-09-14T17:35:02.000000Z')"); - executeInsert("insert into balances values ('c1', 'EUR', 880.2, '2021-09-14T17:35:03.000000Z')"); - executeInsert("insert into balances values ('c1', 'EUR', 782, '2021-09-14T17:35:04.000000Z')"); - executeInsert("insert into balances values ('c2', 'USD', 900, '2021-09-14T17:35:05.000000Z')"); - executeInsert("insert into balances values ('c2', 'USD', 190.75, '2021-09-14T17:35:06.000000Z')"); - executeInsert("insert into balances values ('c2', 'EUR', 890.2, '2021-09-14T17:35:07.000000Z')"); - executeInsert("insert into balances values ('c2', 'EUR', 1000, '2021-09-14T17:35:08.000000Z')"); - - TestUtils.assertSql( - compiler, - sqlExecutionContext, - "SELECT * FROM balances LATEST BY cust_id, balance_ccy \n" + - "WHERE cust_id = 'c1' and balance_ccy='EUR'", - sink, - "cust_id\tbalance_ccy\tbalance\ttimestamp\n" + - "c1\tEUR\t782.0\t2021-09-14T17:35:04.000000Z\n" - ); + compiler.compile("create table tab(" + + " id symbol, " + + " name symbol, " + + " value double, " + + " other_ts timestamp, " + + " ts timestamp" + + ")", sqlExecutionContext); + executeInsert("insert into tab values ('d1', 'c1', 101.1, '2021-10-15T11:31:35.878Z', '2021-10-05T13:31:35.878Z')"); + executeInsert("insert into tab values ('d1', 'c1', 101.2, '2021-10-15T12:31:35.878Z', '2021-10-05T12:31:35.878Z')"); + executeInsert("insert into tab values ('d1', 'c1', 101.3, '2021-10-15T13:31:35.878Z', '2021-10-05T17:31:35.878Z')"); + executeInsert("insert into tab values ('d1', 'c1', 101.4, '2021-10-15T14:31:35.878Z', '2021-10-05T14:31:35.878Z')"); + executeInsert("insert into tab values ('d1', 'c2', 102.1, '2021-10-15T15:31:35.878Z', '2021-10-04T11:31:35.878Z')"); + executeInsert("insert into tab values ('d1', 'c2', 102.2, '2021-10-15T16:31:35.878Z', '2021-10-03T12:31:35.878Z')"); + executeInsert("insert into tab values ('d1', 'c2', 102.3, '2021-10-15T17:31:35.878Z', '2021-10-02T13:31:35.878Z')"); + executeInsert("insert into tab values ('d1', 'c2', 102.4, '2021-10-15T11:31:35.878Z', '2021-09-05T14:31:35.878Z')"); + executeInsert("insert into tab values ('d1', 'c2', 102.5, '2021-10-15T12:31:35.878Z', '2021-01-05T15:31:35.878Z')"); + executeInsert("insert into tab values ('d2', 'c1', 201.1, '2021-10-15T13:31:35.878Z', '2021-10-05T11:31:35.878Z')"); + executeInsert("insert into tab values ('d2', 'c1', 201.2, '2021-10-15T14:31:35.878Z', '2021-10-05T12:31:35.878Z')"); + executeInsert("insert into tab values ('d2', 'c1', 201.3, '2021-10-15T15:31:35.878Z', '2021-10-25T13:31:35.878Z')"); + executeInsert("insert into tab values ('d2', 'c1', 201.4, '2021-10-15T16:31:35.878Z', '2021-10-05T14:31:35.878Z')"); + executeInsert("insert into tab values ('d2', 'c2', 401.1, '2021-10-16T17:31:35.878Z', '2021-10-26T11:31:35.878Z')"); + executeInsert("insert into tab values ('d2', 'c1', 401.2, '2021-10-16T11:31:35.878Z', '2021-10-06T12:31:35.878Z')"); + executeInsert("insert into tab values ('d2', 'c1', 111.7, '2021-10-16T17:31:35.878Z', '2021-10-26T15:31:35.878Z')"); + assertSql( + "(tab latest by id where name in ('c2')) timestamp(other_ts)", + "id\tname\tvalue\tother_ts\tts\n" + + "d1\tc2\t102.5\t2021-10-15T12:31:35.878000Z\t2021-01-05T15:31:35.878000Z\n" + + "d2\tc2\t401.1\t2021-10-16T17:31:35.878000Z\t2021-10-26T11:31:35.878000Z\n"); }); } + @Ignore() + // TODO: if the table has a designated timestamp, it becomes sticky + // on latest by, and we cannot change it explicitly. + // The query: (tab latest by id where name in ('c2')) timestamp(ts) + // - only interested in rows with name == 'c2' + // - for these, we want the latest row based on the ts column, + // using id as unique key for the whole row @Test - public void testLatestByFilteredBySymbolInNoIndexes() throws Exception { - testLatestByFilteredBySymbolIn("create table x (\n" + - " ts timestamp,\n" + - " node symbol,\n" + - " metric symbol,\n" + - " value long) \n" + - " timestamp(ts) partition by day"); + public void testLatestByTsIsPickedAtRuntimeOtherThanDesignated() throws Exception { + assertMemoryLeak(() -> { + compiler.compile("create table tab(" + + " id symbol, " + + " name symbol, " + + " value double, " + + " other_ts timestamp, " + + " ts timestamp" + + ") timestamp(other_ts) partition by day", sqlExecutionContext); + executeInsert("insert into tab values ('d1', 'c1', 101.1, '2021-10-15T11:31:35.878Z', '2021-10-05T13:31:35.878Z')"); + executeInsert("insert into tab values ('d1', 'c1', 101.2, '2021-10-15T12:31:35.878Z', '2021-10-05T12:31:35.878Z')"); + executeInsert("insert into tab values ('d1', 'c1', 101.3, '2021-10-15T13:31:35.878Z', '2021-10-05T17:31:35.878Z')"); + executeInsert("insert into tab values ('d1', 'c1', 101.4, '2021-10-15T14:31:35.878Z', '2021-10-05T14:31:35.878Z')"); + executeInsert("insert into tab values ('d1', 'c2', 102.1, '2021-10-15T15:31:35.878Z', '2021-10-04T11:31:35.878Z')"); + executeInsert("insert into tab values ('d1', 'c2', 102.2, '2021-10-15T16:31:35.878Z', '2021-10-03T12:31:35.878Z')"); + executeInsert("insert into tab values ('d1', 'c2', 102.3, '2021-10-15T17:31:35.878Z', '2021-10-02T13:31:35.878Z')"); + executeInsert("insert into tab values ('d1', 'c2', 102.4, '2021-10-15T11:31:35.878Z', '2021-09-05T14:31:35.878Z')"); + executeInsert("insert into tab values ('d1', 'c2', 102.5, '2021-10-15T12:31:35.878Z', '2021-01-05T15:31:35.878Z')"); + executeInsert("insert into tab values ('d2', 'c1', 201.1, '2021-10-15T13:31:35.878Z', '2021-10-05T11:31:35.878Z')"); + executeInsert("insert into tab values ('d2', 'c1', 201.2, '2021-10-15T14:31:35.878Z', '2021-10-05T12:31:35.878Z')"); + executeInsert("insert into tab values ('d2', 'c1', 201.3, '2021-10-15T15:31:35.878Z', '2021-10-25T13:31:35.878Z')"); + executeInsert("insert into tab values ('d2', 'c1', 201.4, '2021-10-15T16:31:35.878Z', '2021-10-05T14:31:35.878Z')"); + executeInsert("insert into tab values ('d2', 'c2', 401.1, '2021-10-16T17:31:35.878Z', '2021-10-26T11:31:35.878Z')"); + executeInsert("insert into tab values ('d2', 'c1', 401.2, '2021-10-16T11:31:35.878Z', '2021-10-06T12:31:35.878Z')"); + executeInsert("insert into tab values ('d2', 'c1', 111.7, '2021-10-16T17:31:35.878Z', '2021-10-26T15:31:35.878Z')"); + assertSql( + "(tab latest by id where name in ('c2')) timestamp(ts)", + "id\tname\tvalue\tother_ts\tts\n" + + "d1\tc2\t102.10000000000001\t2021-10-15T17:31:35.878000Z\t2021-10-04T11:31:35.878000Z\n" + + "d2\tc2\t401.1\t2021-10-16T17:31:35.878000Z\t2021-10-26T11:31:35.878000Z\n"); + }); } @Test - public void testLatestByFilteredSymbolInPartiallyIndexed1() throws Exception { - testLatestByFilteredBySymbolIn("create table x (\n" + - " ts timestamp,\n" + - " node symbol index,\n" + - " metric symbol,\n" + - " value long) \n" + - " timestamp(ts) partition by day"); + public void testLatestByUnsupportedColumnTypes() throws Exception { + // unsupported: [BYTE, DATE, TIMESTAMP, FLOAT, DOUBLE, GEOBYTE, GEOSHORT, GEOINT, GEOLONG, BINARY] + CharSequence createTableDDL = "create table comprehensive as (" + + " select" + + " rnd_byte(2,50) byte, " + + " rnd_date(to_date('2020', 'yyyy'), to_date('2021', 'yyyy'), 2) date, " + + " rnd_timestamp(to_timestamp('2020', 'yyyy'), to_timestamp('2021', 'yyyy'), 2) timestamp, " + + " rnd_float(2) float, " + + " rnd_double(2) double, " + + " rnd_geohash(5) gbyte, " + + " rnd_geohash(15) gshort, " + + " rnd_geohash(30) gint, " + + " rnd_geohash(60) glong, " + + " rnd_bin(10, 20, 2) binary, " + + " timestamp_sequence(0, 1000000000) ts" + + " from long_sequence(10)" + + ") timestamp(ts) partition by DAY"; + CharSequence expectedTail = "invalid type, only [BOOLEAN, SHORT, INT, LONG, LONG256, CHAR, STRING, SYMBOL] are supported in LATEST BY"; + assertFailure( + "comprehensive latest by byte", + createTableDDL, + 24, + "byte (BYTE): " + expectedTail); + for (String[] nameType : new String[][]{ + {"date", "DATE"}, + {"timestamp", "TIMESTAMP"}, + {"float", "FLOAT"}, + {"double", "DOUBLE"}, + {"gbyte", "GEOHASH(1c)"}, + {"gshort", "GEOHASH(3c)"}, + {"gint", "GEOHASH(6c)"}, + {"glong", "GEOHASH(12c)"}, + {"binary", "BINARY"}, + {"ts", "TIMESTAMP"}}) { + assertFailure( + "comprehensive latest by " + nameType[0], + null, + 24, + String.format("%s (%s): %s", nameType[0], nameType[1], expectedTail)); + } } @Test - public void testLatestByFilteredSymbolInPartiallyIndexed2() throws Exception { - testLatestByFilteredBySymbolIn("create table x (\n" + - " ts timestamp,\n" + - " node symbol,\n" + - " metric symbol index,\n" + - " value long) \n" + - " timestamp(ts) partition by day"); + public void testLatestByValue() throws Exception { + // no index + assertQuery("a\tb\tk\n" + + "65.08594025855301\tHNR\t1970-01-02T03:46:40.000000Z\n", + "select * from x latest by b where b = 'HNR'", + "create table x as " + + "(" + + "select " + + " rnd_double(0)*100 a," + + " rnd_str(2,4,4) b," + + " timestamp_sequence(0, 100000000000) k" + + " from" + + " long_sequence(20)" + + ") timestamp(k) partition by DAY", + "k", + "insert into x select * from (" + + "select" + + " rnd_double(0)*100," + + " 'HNR'," + + " to_timestamp('1971', 'yyyy') t" + + " from long_sequence(1)" + + ") timestamp(t)", + "a\tb\tk\n" + + "34.56897991538844\tHNR\t1971-01-01T00:00:00.000000Z\n", + true, + true, + false, + true); } @Test - public void testLatestByFilteredBySymbolInAllIndexed() throws Exception { - testLatestByFilteredBySymbolIn("create table x (\n" + - " ts timestamp,\n" + - " node symbol index,\n" + - " metric symbol index,\n" + - " value long) \n" + - " timestamp(ts) partition by day"); - } - - private void testLatestByFilteredBySymbolIn(String ddl) throws Exception { + public void testLeftJoinDoesNotRequireTimestamp() throws Exception { assertMemoryLeak(() -> { - compiler.compile(ddl, sqlExecutionContext); + compiler.compile("CREATE TABLE sensors (ID LONG, make STRING, city STRING);", sqlExecutionContext); + compiler.compile( + "INSERT INTO sensors\n" + + "SELECT\n" + + " x ID, --increasing integer\n" + + " rnd_str('Eberle', 'Honeywell', 'Omron', 'United Automation', 'RS Pro') make,\n" + + " rnd_str('New York', 'Miami', 'Boston', 'Chicago', 'San Francisco') city\n" + + "FROM long_sequence(10000) x;", + sqlExecutionContext + ); - executeInsert("insert into x values ('2021-11-17T17:35:01.000000Z', 'node1', 'cpu', 1)"); - executeInsert("insert into x values ('2021-11-17T17:35:02.000000Z', 'node1', 'cpu', 10)"); - executeInsert("insert into x values ('2021-11-17T17:35:03.000000Z', 'node1', 'cpu', 100)"); - executeInsert("insert into x values ('2021-11-17T17:35:01.000000Z', 'node2', 'cpu', 7)"); - executeInsert("insert into x values ('2021-11-17T17:35:02.000000Z', 'node2', 'cpu', 15)"); - executeInsert("insert into x values ('2021-11-17T17:35:03.000000Z', 'node2', 'cpu', 75)"); - executeInsert("insert into x values ('2021-11-17T17:35:01.000000Z', 'node3', 'cpu', 5)"); - executeInsert("insert into x values ('2021-11-17T17:35:02.000000Z', 'node3', 'cpu', 20)"); - executeInsert("insert into x values ('2021-11-17T17:35:03.000000Z', 'node3', 'cpu', 25)"); - executeInsert("insert into x values ('2021-11-17T17:35:01.000000Z', 'node1', 'memory', 20)"); - executeInsert("insert into x values ('2021-11-17T17:35:02.000000Z', 'node1', 'memory', 200)"); - executeInsert("insert into x values ('2021-11-17T17:35:03.000000Z', 'node1', 'memory', 2000)"); - executeInsert("insert into x values ('2021-11-17T17:35:01.000000Z', 'node2', 'memory', 30)"); - executeInsert("insert into x values ('2021-11-17T17:35:02.000000Z', 'node2', 'memory', 300)"); - executeInsert("insert into x values ('2021-11-17T17:35:03.000000Z', 'node2', 'memory', 3000)"); - executeInsert("insert into x values ('2021-11-17T17:35:01.000000Z', 'node3', 'memory', 40)"); - executeInsert("insert into x values ('2021-11-17T17:35:02.000000Z', 'node3', 'memory', 400)"); - executeInsert("insert into x values ('2021-11-17T17:35:03.000000Z', 'node3', 'memory', 4000)"); + compiler.compile( + "CREATE TABLE readings\n" + + "AS(\n" + + " SELECT\n" + + " x ID,\n" + + " timestamp_sequence(to_timestamp('2019-10-17T00:00:00', 'yyyy-MM-ddTHH:mm:ss'), rnd_long(1,10,2) * 100000L) ts,\n" + + " rnd_double(0)*8 + 15 temp,\n" + + " rnd_long(0, 10000, 0) sensorId\n" + + " FROM long_sequence(10000000) x)\n" + + "TIMESTAMP(ts)\n" + + "PARTITION BY MONTH;", + sqlExecutionContext + ); TestUtils.assertSql( compiler, sqlExecutionContext, - "select metric, sum(value) from x latest by node \n" + - "where node in ('node1', 'node2') and metric in ('cpu')", - sink, - "metric\tsum\n" + - "cpu\t175\n" - ); - }); - } - - @Test - public void testLatestBySelectAllFilteredBySymbolInNoIndexes() throws Exception { - testLatestBySelectAllFilteredBySymbolIn("create table x (\n" + - " ts timestamp,\n" + - " node symbol,\n" + - " metric symbol,\n" + - " value long) \n" + - " timestamp(ts) partition by day"); - } - - @Test - public void testLatestBySelectAllFilteredBySymbolInPartiallyIndexed1() throws Exception { - testLatestBySelectAllFilteredBySymbolIn("create table x (\n" + - " ts timestamp,\n" + - " node symbol index,\n" + - " metric symbol,\n" + - " value long) \n" + - " timestamp(ts) partition by day"); - } - - @Test - public void testLatestBySelectAllFilteredBySymbolInPartiallyIndexed2() throws Exception { - testLatestBySelectAllFilteredBySymbolIn("create table x (\n" + - " ts timestamp,\n" + - " node symbol,\n" + - " metric symbol index,\n" + - " value long) \n" + - " timestamp(ts) partition by day"); - } - - @Test - public void testLatestBySelectAllFilteredBySymbolInAllIndexed() throws Exception { - testLatestBySelectAllFilteredBySymbolIn("create table x (\n" + - " ts timestamp,\n" + - " node symbol index,\n" + - " metric symbol index,\n" + - " value long) \n" + - " timestamp(ts) partition by day"); - } - - private void testLatestBySelectAllFilteredBySymbolIn(String ddl) throws Exception { - assertMemoryLeak(() -> { - compiler.compile(ddl, sqlExecutionContext); - - executeInsert("insert into x values ('2021-11-17T17:35:01.000000Z', 'node1', 'cpu', 1)"); - executeInsert("insert into x values ('2021-11-17T17:35:02.000000Z', 'node1', 'cpu', 10)"); - executeInsert("insert into x values ('2021-11-17T17:35:03.000000Z', 'node1', 'cpu', 100)"); - executeInsert("insert into x values ('2021-11-17T17:35:01.000000Z', 'node2', 'cpu', 7)"); - executeInsert("insert into x values ('2021-11-17T17:35:02.000000Z', 'node2', 'cpu', 15)"); - executeInsert("insert into x values ('2021-11-17T17:35:03.000000Z', 'node2', 'cpu', 75)"); - executeInsert("insert into x values ('2021-11-17T17:35:01.000000Z', 'node3', 'cpu', 5)"); - executeInsert("insert into x values ('2021-11-17T17:35:02.000000Z', 'node3', 'cpu', 20)"); - executeInsert("insert into x values ('2021-11-17T17:35:03.000000Z', 'node3', 'cpu', 25)"); - executeInsert("insert into x values ('2021-11-17T17:35:01.000000Z', 'node1', 'memory', 20)"); - executeInsert("insert into x values ('2021-11-17T17:35:02.000000Z', 'node1', 'memory', 200)"); - executeInsert("insert into x values ('2021-11-17T17:35:03.000000Z', 'node1', 'memory', 2000)"); - executeInsert("insert into x values ('2021-11-17T17:35:01.000000Z', 'node2', 'memory', 30)"); - executeInsert("insert into x values ('2021-11-17T17:35:02.000000Z', 'node2', 'memory', 300)"); - executeInsert("insert into x values ('2021-11-17T17:35:03.000000Z', 'node2', 'memory', 3000)"); - executeInsert("insert into x values ('2021-11-17T17:35:01.000000Z', 'node3', 'memory', 40)"); - executeInsert("insert into x values ('2021-11-17T17:35:02.000000Z', 'node3', 'memory', 400)"); - executeInsert("insert into x values ('2021-11-17T17:35:03.000000Z', 'node3', 'memory', 4000)"); - - TestUtils.assertSql( - compiler, - sqlExecutionContext, - "select * from x latest by node, metric \n" + - "where node in ('node2') and metric in ('cpu', 'memory')", - sink, - "ts\tnode\tmetric\tvalue\n" + - "2021-11-17T17:35:03.000000Z\tnode2\tcpu\t75\n" + - "2021-11-17T17:35:03.000000Z\tnode2\tmemory\t3000\n" - ); - }); - } - - @Test - public void testLeftJoinDoesNotRequireTimestamp() throws Exception { - assertMemoryLeak(() -> { - compiler.compile("CREATE TABLE sensors (ID LONG, make STRING, city STRING);", sqlExecutionContext); - compiler.compile( - "INSERT INTO sensors\n" + - "SELECT\n" + - " x ID, --increasing integer\n" + - " rnd_str('Eberle', 'Honeywell', 'Omron', 'United Automation', 'RS Pro') make,\n" + - " rnd_str('New York', 'Miami', 'Boston', 'Chicago', 'San Francisco') city\n" + - "FROM long_sequence(10000) x;", - sqlExecutionContext - ); - - compiler.compile( - "CREATE TABLE readings\n" + - "AS(\n" + - " SELECT\n" + - " x ID,\n" + - " timestamp_sequence(to_timestamp('2019-10-17T00:00:00', 'yyyy-MM-ddTHH:mm:ss'), rnd_long(1,10,2) * 100000L) ts,\n" + - " rnd_double(0)*8 + 15 temp,\n" + - " rnd_long(0, 10000, 0) sensorId\n" + - " FROM long_sequence(10000000) x)\n" + - "TIMESTAMP(ts)\n" + - "PARTITION BY MONTH;", - sqlExecutionContext - ); - - TestUtils.assertSql( - compiler, - sqlExecutionContext, - "SELECT ts, a.city, a.make, avg(temp)\n" + - "FROM readings timestamp(ts)\n" + - "JOIN\n" + - " (SELECT ID sensId, city, make\n" + - " FROM sensors\n" + - " WHERE city='Miami' AND make='Omron') a\n" + - "ON readings.sensorId = a.sensId\n" + - "WHERE ts in '2019-10-21;1d'\n" + - "SAMPLE BY 1h;", + "SELECT ts, a.city, a.make, avg(temp)\n" + + "FROM readings timestamp(ts)\n" + + "JOIN\n" + + " (SELECT ID sensId, city, make\n" + + " FROM sensors\n" + + " WHERE city='Miami' AND make='Omron') a\n" + + "ON readings.sensorId = a.sensId\n" + + "WHERE ts in '2019-10-21;1d'\n" + + "SAMPLE BY 1h;", sink, "ts\tcity\tmake\tavg\n" + "2019-10-21T00:00:15.500000Z\tMiami\tOmron\t18.932522082097226\n" + @@ -6112,6 +6267,91 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest { ); } + @Ignore("result order is currently dependent on stability of sorting method") + // TODO: this is broken, the expected result order for "select * from tab" in the presence + // of repeated timestamps needs to be predefined and consistent, one of two alternatives: + // 1.- most recent insert for a given timestamp first: + // "d2\tc1\t201.10000000000002\t2021-10-05T11:31:35.878000Z\n" + + // "d1\tc2\t102.10000000000001\t2021-10-05T11:31:35.878000Z\n" + + // "d1\tc1\t101.10000000000001\t2021-10-05T11:31:35.878000Z\n" + + // "d2\tc1\t201.20000000000002\t2021-10-05T12:31:35.878000Z\n" + + // "d1\tc2\t102.2\t2021-10-05T12:31:35.878000Z\n" + + // "d1\tc1\t101.2\t2021-10-05T12:31:35.878000Z\n" + + // "d2\tc1\t201.3\t2021-10-05T13:31:35.878000Z\n" + + // "d1\tc2\t102.30000000000001\t2021-10-05T13:31:35.878000Z\n" + + // "d1\tc1\t101.30000000000001\t2021-10-05T13:31:35.878000Z\n" + + // "d2\tc1\t201.4\t2021-10-05T14:31:35.878000Z\n" + + // "d1\tc2\t102.4\t2021-10-05T14:31:35.878000Z\n" + + // "d1\tc1\t101.4\t2021-10-05T14:31:35.878000Z\n" + + // "d1\tc2\t102.5\t2021-10-05T15:31:35.878000Z\n" + + // "d2\tc2\t401.1\t2021-10-06T11:31:35.878000Z\n" + + // "d2\tc1\t401.20000000000005\t2021-10-06T12:31:35.878000Z\n" + + // "d2\tc1\t111.7\t2021-10-06T15:31:35.878000Z\n" + // 2.- least recent insert for a given timestamp first: + // "d1\tc1\t101.10000000000001\t2021-10-05T11:31:35.878000Z\n" + + // "d1\tc2\t102.10000000000001\t2021-10-05T11:31:35.878000Z\n" + + // "d2\tc1\t201.10000000000002\t2021-10-05T11:31:35.878000Z\n" + + // "d1\tc1\t101.2\t2021-10-05T12:31:35.878000Z\n" + + // "d1\tc2\t102.2\t2021-10-05T12:31:35.878000Z\n" + + // "d2\tc1\t201.20000000000002\t2021-10-05T12:31:35.878000Z\n" + + // "d1\tc1\t101.30000000000001\t2021-10-05T13:31:35.878000Z\n" + + // "d1\tc2\t102.30000000000001\t2021-10-05T13:31:35.878000Z\n" + + // "d2\tc1\t201.3\t2021-10-05T13:31:35.878000Z\n" + + // "d1\tc1\t101.4\t2021-10-05T14:31:35.878000Z\n" + + // "d1\tc2\t102.4\t2021-10-05T14:31:35.878000Z\n" + + // "d2\tc1\t201.4\t2021-10-05T14:31:35.878000Z\n" + + // "d1\tc2\t102.5\t2021-10-05T15:31:35.878000Z\n" + + // "d2\tc2\t401.1\t2021-10-06T11:31:35.878000Z\n" + + // "d2\tc1\t401.20000000000005\t2021-10-06T12:31:35.878000Z\n" + + // "d2\tc1\t111.7\t2021-10-06T15:31:35.878000Z\n" + // in the assertSql that follows, option #2 has been taken. + @Test + public void testSelectExpectedOrder() throws Exception { + assertMemoryLeak(() -> { + compiler.compile("create table tab(" + + " id symbol index, " + + " name symbol index, " + + " value double, " + + " ts timestamp" + + ") timestamp(ts) partition by DAY", sqlExecutionContext); + executeInsert("insert into tab values ('d1', 'c1', 101.1, '2021-10-05T11:31:35.878Z')"); + executeInsert("insert into tab values ('d1', 'c1', 101.2, '2021-10-05T12:31:35.878Z')"); + executeInsert("insert into tab values ('d1', 'c1', 101.3, '2021-10-05T13:31:35.878Z')"); + executeInsert("insert into tab values ('d1', 'c1', 101.4, '2021-10-05T14:31:35.878Z')"); + executeInsert("insert into tab values ('d1', 'c2', 102.1, '2021-10-05T11:31:35.878Z')"); + executeInsert("insert into tab values ('d1', 'c2', 102.2, '2021-10-05T12:31:35.878Z')"); + executeInsert("insert into tab values ('d1', 'c2', 102.3, '2021-10-05T13:31:35.878Z')"); + executeInsert("insert into tab values ('d1', 'c2', 102.4, '2021-10-05T14:31:35.878Z')"); + executeInsert("insert into tab values ('d1', 'c2', 102.5, '2021-10-05T15:31:35.878Z')"); + executeInsert("insert into tab values ('d2', 'c1', 201.1, '2021-10-05T11:31:35.878Z')"); + executeInsert("insert into tab values ('d2', 'c1', 201.2, '2021-10-05T12:31:35.878Z')"); + executeInsert("insert into tab values ('d2', 'c1', 201.3, '2021-10-05T13:31:35.878Z')"); + executeInsert("insert into tab values ('d2', 'c1', 201.4, '2021-10-05T14:31:35.878Z')"); + executeInsert("insert into tab values ('d2', 'c2', 401.1, '2021-10-06T11:31:35.878Z')"); + executeInsert("insert into tab values ('d2', 'c1', 401.2, '2021-10-06T12:31:35.878Z')"); + executeInsert("insert into tab values ('d2', 'c1', 111.7, '2021-10-06T15:31:35.878Z')"); + assertSql( + "tab", + "id\tname\tvalue\tts\n" + + "d1\tc1\t101.10000000000001\t2021-10-05T11:31:35.878000Z\n" + + "d1\tc2\t102.10000000000001\t2021-10-05T11:31:35.878000Z\n" + + "d2\tc1\t201.10000000000002\t2021-10-05T11:31:35.878000Z\n" + + "d1\tc1\t101.2\t2021-10-05T12:31:35.878000Z\n" + + "d1\tc2\t102.2\t2021-10-05T12:31:35.878000Z\n" + + "d2\tc1\t201.20000000000002\t2021-10-05T12:31:35.878000Z\n" + + "d1\tc1\t101.30000000000001\t2021-10-05T13:31:35.878000Z\n" + + "d1\tc2\t102.30000000000001\t2021-10-05T13:31:35.878000Z\n" + + "d2\tc1\t201.3\t2021-10-05T13:31:35.878000Z\n" + + "d1\tc1\t101.4\t2021-10-05T14:31:35.878000Z\n" + + "d1\tc2\t102.4\t2021-10-05T14:31:35.878000Z\n" + + "d2\tc1\t201.4\t2021-10-05T14:31:35.878000Z\n" + + "d1\tc2\t102.5\t2021-10-05T15:31:35.878000Z\n" + + "d2\tc2\t401.1\t2021-10-06T11:31:35.878000Z\n" + + "d2\tc1\t401.20000000000005\t2021-10-06T12:31:35.878000Z\n" + + "d2\tc1\t111.7\t2021-10-06T15:31:35.878000Z\n"); + }); + } + @Test public void testSelectFromAliasedTable() throws Exception { assertMemoryLeak(() -> { @@ -6368,6 +6608,34 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest { ); } + @Test + public void testUtf8TableName() throws Exception { + assertMemoryLeak( + () -> { + compiler.compile("CREATE TABLE 'привет от штиблет' (f0 STRING, штиблет STRING, f2 STRING);", sqlExecutionContext); + TestUtils.assertSql( + compiler, + sqlExecutionContext, + "tables()", + sink, + "id\tname\tdesignatedTimestamp\tpartitionBy\tmaxUncommittedRows\tcommitLag\n" + + "1\tпривет от штиблет\t\tNONE\t1000\t0\n" + ); + + TestUtils.assertSql( + compiler, + sqlExecutionContext, + "show columns from 'привет от штиблет'", + sink, + "column\ttype\tindexed\tindexBlockCapacity\tsymbolCached\tsymbolCapacity\tdesignated\n" + + "f0\tSTRING\tfalse\t0\tfalse\t0\tfalse\n" + + "штиблет\tSTRING\tfalse\t0\tfalse\t0\tfalse\n" + + "f2\tSTRING\tfalse\t0\tfalse\t0\tfalse\n" + ); + } + ); + } + @Test public void testVectorAggregateOnSparsePartitions() throws Exception { final String expected = "a\tk\n"; @@ -6558,376 +6826,181 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest { ); } - @Test - public void testLatestByUnsupportedColumnTypes() throws Exception { - // unsupported: [BYTE, DATE, TIMESTAMP, FLOAT, DOUBLE, GEOBYTE, GEOSHORT, GEOINT, GEOLONG, BINARY] - CharSequence createTableDDL = "create table comprehensive as (" + - " select" + - " rnd_byte(2,50) byte, " + - " rnd_date(to_date('2020', 'yyyy'), to_date('2021', 'yyyy'), 2) date, " + - " rnd_timestamp(to_timestamp('2020', 'yyyy'), to_timestamp('2021', 'yyyy'), 2) timestamp, " + - " rnd_float(2) float, " + - " rnd_double(2) double, " + - " rnd_geohash(5) gbyte, " + - " rnd_geohash(15) gshort, " + - " rnd_geohash(30) gint, " + - " rnd_geohash(60) glong, " + - " rnd_bin(10, 20, 2) binary, " + - " timestamp_sequence(0, 1000000000) ts" + - " from long_sequence(10)" + - ") timestamp(ts) partition by DAY"; - CharSequence expectedTail = "invalid type, only [BOOLEAN, SHORT, INT, LONG, LONG256, CHAR, STRING, SYMBOL] are supported in LATEST BY"; - assertFailure( - "comprehensive latest by byte", - createTableDDL, - 24, - "byte (BYTE): " + expectedTail); - for (String[] nameType : new String[][]{ - {"date", "DATE"}, - {"timestamp", "TIMESTAMP"}, - {"float", "FLOAT"}, - {"double", "DOUBLE"}, - {"gbyte", "GEOHASH(1c)"}, - {"gshort", "GEOHASH(3c)"}, - {"gint", "GEOHASH(6c)"}, - {"glong", "GEOHASH(12c)"}, - {"binary", "BINARY"}, - {"ts", "TIMESTAMP"}}) { - assertFailure( - "comprehensive latest by " + nameType[0], - null, - 24, - String.format("%s (%s): %s", nameType[0], nameType[1], expectedTail)); - } - } - - @Test - public void testLatestBySupportedColumnTypes0() throws Exception { - testLatestBySupportedColumnTypes( - "create table tab (" + - " boolean boolean, " + - " short short, " + - " int int, " + - " long long, " + - " long256 long256, " + - " char char, " + - " string string, " + - " symbol symbol, " + - " ts timestamp" + - ")", - null); - } - - @Test - public void testLatestBySupportedColumnTypes1() throws Exception { - testLatestBySupportedColumnTypes( - "create table tab (" + - " boolean boolean, " + - " short short, " + - " int int, " + - " long long, " + - " long256 long256, " + - " char char, " + - " string string, " + - " symbol symbol, " + - " ts timestamp" + - ") timestamp(ts) partition by DAY", - "ts"); - } - - @Test - public void testLatestBySupportedColumnTypes2() throws Exception { - testLatestBySupportedColumnTypes( - "create table tab (" + - " boolean boolean, " + - " short short, " + - " int int, " + - " long long, " + - " long256 long256, " + - " char char, " + - " string string, " + - " symbol symbol index, " + - " ts timestamp" + - ")", - null); + private void createGeoHashTable(int chars) throws SqlException { + compiler.compile( + String.format("create table pos(time timestamp, uuid symbol, hash geohash(%dc))", chars) + + ", index(uuid) timestamp(time) partition by DAY", + sqlExecutionContext + ); + executeInsert("insert into pos values('2021-05-10T23:59:59.150000Z','XXX','f91t48s7')"); + executeInsert("insert into pos values('2021-05-10T23:59:59.322000Z','ddd','bbqyzfp6')"); + executeInsert("insert into pos values('2021-05-10T23:59:59.351000Z','bbb','9egcyrxq')"); + executeInsert("insert into pos values('2021-05-10T23:59:59.439000Z','bbb','ewef1vk8')"); + executeInsert("insert into pos values('2021-05-10T00:00:00.016000Z','aaa','vb2wg49h')"); + executeInsert("insert into pos values('2021-05-10T00:00:00.042000Z','ccc','bft3gn89')"); + executeInsert("insert into pos values('2021-05-10T00:00:00.055000Z','aaa','z6cf5j85')"); + executeInsert("insert into pos values('2021-05-11T00:00:00.066000Z','ddd','vcunv6j7')"); + executeInsert("insert into pos values('2021-05-11T00:00:00.072000Z','ccc','edez0n5y')"); + executeInsert("insert into pos values('2021-05-11T00:00:00.074000Z','aaa','fds32zgc')"); + executeInsert("insert into pos values('2021-05-11T00:00:00.083000Z','YYY','z31wzd5w')"); + executeInsert("insert into pos values('2021-05-11T00:00:00.092000Z','ddd','v9nwc4ny')"); + executeInsert("insert into pos values('2021-05-11T00:00:00.107000Z','ccc','f6yb1yx9')"); + executeInsert("insert into pos values('2021-05-11T00:00:00.111000Z','ddd','bcnktpnw')"); + executeInsert("insert into pos values('2021-05-11T00:00:00.123000Z','aaa','z3t2we5z')"); + executeInsert("insert into pos values('2021-05-11T00:00:00.127000Z','aaa','bgn1yt4y')"); + executeInsert("insert into pos values('2021-05-11T00:00:00.144000Z','aaa','fuetk3k6')"); + executeInsert("insert into pos values('2021-05-12T00:00:00.167000Z','ccc','bchx5x14')"); + executeInsert("insert into pos values('2021-05-12T00:00:00.167000Z','ZZZ','bbxwb5jj')"); + executeInsert("insert into pos values('2021-05-12T00:00:00.186000Z','ZZZ','vepe7h62')"); + executeInsert("insert into pos values('2021-05-12T00:00:00.241000Z','bbb','bchxpmmg')"); + executeInsert("insert into pos values('2021-05-12T00:00:00.245000Z','ddd','f90z3bs5')"); + executeInsert("insert into pos values('2021-05-12T00:00:00.247000Z','bbb','bftqreuh')"); + executeInsert("insert into pos values('2021-05-12T00:00:00.295000Z','ddd','u2rqgy9s')"); + executeInsert("insert into pos values('2021-05-12T00:00:00.304000Z','aaa','w23bhjd2')"); } - @Test - public void testLatestBySupportedColumnTypes3() throws Exception { - testLatestBySupportedColumnTypes( - "create table tab (" + - " boolean boolean, " + - " short short, " + - " int int, " + - " long long, " + - " long256 long256, " + - " char char, " + - " string string, " + - " symbol symbol index, " + - " ts timestamp" + - ") timestamp(ts) partition by DAY", - "ts"); + private void createRndGeoHashBitsTable() throws SqlException { + compiler.compile( + "create table x as (" + + "select" + + " cast(x as int) i," + + " rnd_symbol(113, 4, 4, 2) s," + + " timestamp_sequence(500000000000L,100000000L) ts," + + " rnd_geohash(3) bits3," + + " rnd_geohash(7) bits7," + + " rnd_geohash(9) bits9" + + " from long_sequence(10000)" + + "), index(s) timestamp (ts) partition by DAY", + sqlExecutionContext + ); } - private void testLatestBySupportedColumnTypes(CharSequence ddl, CharSequence ts) throws Exception { - assertMemoryLeak(() -> { - // supported: [BOOLEAN, CHAR, INT, LONG, LONG256, STRING, SYMBOL] - compiler.compile(ddl, sqlExecutionContext); - executeInsert("insert into tab values (false, cast(24814 as short), 24814, 8260188555232587029, 0x7ee65ec7b6e3bc3a422a8855e9d7bfd29199af5c2aa91ba39c022fa261bdede7, 'J', 'ORANGE', '123', '1970-01-01T00:00:01.000000Z')"); - executeInsert("insert into tab values (true, cast(14817 as short), 14817, 8260188555232587029, 0x4e1c798ce76392e690c6042566c5a1cda5b9a155686af43ac109ac68336ea0c9, 'A', 'COCO', 'XoXoX', '1970-01-01T00:00:02.000000Z')"); - executeInsert("insert into tab values (true, cast(14817 as short), 14817, 3614738589890112276, 0x386129f34be87b5e3990fb6012dac1d3495a30aaa8bf53224e89d27e7ee5104e, 'Q', null, 'XoXoX', '1970-01-01T00:00:03.000000Z')"); - executeInsert("insert into tab values (true, cast(24814 as short), 24814, 3614738589890112276, 0x7ee65ec7b6e3bc3a422a8855e9d7bfd29199af5c2aa91ba39c022fa261bdede7, 'J', null, 'XoXoX', '1970-01-01T00:00:04.000000Z')"); - executeInsert("insert into tab values (true, cast(24814 as short), 24814, 8260188555232587029, 0x4e1c798ce76392e690c6042566c5a1cda5b9a155686af43ac109ac68336ea0c9, 'Q', 'BANANA', '_(*y*)_', '1970-01-01T00:00:05.000000Z')"); - executeInsert("insert into tab values (false, cast(14817 as short), 14817, 6404066507400987550, 0x386129f34be87b5e3990fb6012dac1d3495a30aaa8bf53224e89d27e7ee5104e, 'M', null, '123', '1970-01-01T00:00:06.000000Z')"); - executeInsert("insert into tab values (false, cast(14333 as short), 14333, 8260188555232587029, 0x7ee65ec7b6e3bc3a422a8855e9d7bfd29199af5c2aa91ba39c022fa261bdede7, 'J', 'COCO', '123', '1970-01-01T00:00:07.000000Z')"); - executeInsert("insert into tab values (false, cast(14817 as short), 14817, 8260188555232587029, 0x4e1c798ce76392e690c6042566c5a1cda5b9a155686af43ac109ac68336ea0c9, 'Z', 'BANANA', '_(*y*)_', '1970-01-01T00:00:08.000000Z')"); - executeInsert("insert into tab values (true, cast(24814 as short), 24814, 7759636733976435003, 0x386129f34be87b5e3990fb6012dac1d3495a30aaa8bf53224e89d27e7ee5104e, 'J', 'ORANGE', '123', '1970-01-01T00:00:09.000000Z')"); - executeInsert("insert into tab values (false, cast(24814 as short), 24814, 6404066507400987550, 0x8b04de5aad1f110fdda84f010e21add4b83e6733ca158dd091627fc790e28086, 'W', 'BANANA', '123', '1970-01-01T00:00:10.000000Z')"); - executeInsert("insert into tab values (false, cast(24814 as short), 24814, 6404066507400987550, 0x8b04de5aad1f110fdda84f010e21add4b83e6733ca158dd091627fc790e28086, 'W', 'BANANA', '123', '1970-01-02T00:00:01.000000Z')"); - expectSqlResult( - "boolean\tshort\tint\tlong\tlong256\tchar\tstring\tsymbol\tts\n" + - "true\t24814\t24814\t7759636733976435003\t0x386129f34be87b5e3990fb6012dac1d3495a30aaa8bf53224e89d27e7ee5104e\tJ\tORANGE\t123\t1970-01-01T00:00:09.000000Z\n" + - "false\t24814\t24814\t6404066507400987550\t0x8b04de5aad1f110fdda84f010e21add4b83e6733ca158dd091627fc790e28086\tW\tBANANA\t123\t1970-01-02T00:00:01.000000Z\n", - "tab latest by boolean", ts); - - expectSqlResult( - "boolean\tshort\tint\tlong\tlong256\tchar\tstring\tsymbol\tts\n" + - "false\t14333\t14333\t8260188555232587029\t0x7ee65ec7b6e3bc3a422a8855e9d7bfd29199af5c2aa91ba39c022fa261bdede7\tJ\tCOCO\t123\t1970-01-01T00:00:07.000000Z\n" + - "false\t14817\t14817\t8260188555232587029\t0x4e1c798ce76392e690c6042566c5a1cda5b9a155686af43ac109ac68336ea0c9\tZ\tBANANA\t_(*y*)_\t1970-01-01T00:00:08.000000Z\n" + - "false\t24814\t24814\t6404066507400987550\t0x8b04de5aad1f110fdda84f010e21add4b83e6733ca158dd091627fc790e28086\tW\tBANANA\t123\t1970-01-02T00:00:01.000000Z\n", - "tab timestamp(ts) latest by short", "ts"); - - expectSqlResult( - "boolean\tshort\tint\tlong\tlong256\tchar\tstring\tsymbol\tts\n" + - "false\t14333\t14333\t8260188555232587029\t0x7ee65ec7b6e3bc3a422a8855e9d7bfd29199af5c2aa91ba39c022fa261bdede7\tJ\tCOCO\t123\t1970-01-01T00:00:07.000000Z\n" + - "false\t14817\t14817\t8260188555232587029\t0x4e1c798ce76392e690c6042566c5a1cda5b9a155686af43ac109ac68336ea0c9\tZ\tBANANA\t_(*y*)_\t1970-01-01T00:00:08.000000Z\n" + - "false\t24814\t24814\t6404066507400987550\t0x8b04de5aad1f110fdda84f010e21add4b83e6733ca158dd091627fc790e28086\tW\tBANANA\t123\t1970-01-02T00:00:01.000000Z\n", - "tab latest by int", ts); - - expectSqlResult( - "boolean\tshort\tint\tlong\tlong256\tchar\tstring\tsymbol\tts\n" + - "true\t24814\t24814\t3614738589890112276\t0x7ee65ec7b6e3bc3a422a8855e9d7bfd29199af5c2aa91ba39c022fa261bdede7\tJ\t\tXoXoX\t1970-01-01T00:00:04.000000Z\n" + - "false\t14817\t14817\t8260188555232587029\t0x4e1c798ce76392e690c6042566c5a1cda5b9a155686af43ac109ac68336ea0c9\tZ\tBANANA\t_(*y*)_\t1970-01-01T00:00:08.000000Z\n" + - "true\t24814\t24814\t7759636733976435003\t0x386129f34be87b5e3990fb6012dac1d3495a30aaa8bf53224e89d27e7ee5104e\tJ\tORANGE\t123\t1970-01-01T00:00:09.000000Z\n" + - "false\t24814\t24814\t6404066507400987550\t0x8b04de5aad1f110fdda84f010e21add4b83e6733ca158dd091627fc790e28086\tW\tBANANA\t123\t1970-01-02T00:00:01.000000Z\n", - "tab latest by long", ts); - - expectSqlResult( - "boolean\tshort\tint\tlong\tlong256\tchar\tstring\tsymbol\tts\n" + - "false\t14333\t14333\t8260188555232587029\t0x7ee65ec7b6e3bc3a422a8855e9d7bfd29199af5c2aa91ba39c022fa261bdede7\tJ\tCOCO\t123\t1970-01-01T00:00:07.000000Z\n" + - "false\t14817\t14817\t8260188555232587029\t0x4e1c798ce76392e690c6042566c5a1cda5b9a155686af43ac109ac68336ea0c9\tZ\tBANANA\t_(*y*)_\t1970-01-01T00:00:08.000000Z\n" + - "true\t24814\t24814\t7759636733976435003\t0x386129f34be87b5e3990fb6012dac1d3495a30aaa8bf53224e89d27e7ee5104e\tJ\tORANGE\t123\t1970-01-01T00:00:09.000000Z\n" + - "false\t24814\t24814\t6404066507400987550\t0x8b04de5aad1f110fdda84f010e21add4b83e6733ca158dd091627fc790e28086\tW\tBANANA\t123\t1970-01-02T00:00:01.000000Z\n", - "tab latest by long256", ts); - - expectSqlResult( - "boolean\tshort\tint\tlong\tlong256\tchar\tstring\tsymbol\tts\n" + - "true\t14817\t14817\t8260188555232587029\t0x4e1c798ce76392e690c6042566c5a1cda5b9a155686af43ac109ac68336ea0c9\tA\tCOCO\tXoXoX\t1970-01-01T00:00:02.000000Z\n" + - "true\t24814\t24814\t8260188555232587029\t0x4e1c798ce76392e690c6042566c5a1cda5b9a155686af43ac109ac68336ea0c9\tQ\tBANANA\t_(*y*)_\t1970-01-01T00:00:05.000000Z\n" + - "false\t14817\t14817\t6404066507400987550\t0x386129f34be87b5e3990fb6012dac1d3495a30aaa8bf53224e89d27e7ee5104e\tM\t\t123\t1970-01-01T00:00:06.000000Z\n" + - "false\t14817\t14817\t8260188555232587029\t0x4e1c798ce76392e690c6042566c5a1cda5b9a155686af43ac109ac68336ea0c9\tZ\tBANANA\t_(*y*)_\t1970-01-01T00:00:08.000000Z\n" + - "true\t24814\t24814\t7759636733976435003\t0x386129f34be87b5e3990fb6012dac1d3495a30aaa8bf53224e89d27e7ee5104e\tJ\tORANGE\t123\t1970-01-01T00:00:09.000000Z\n" + - "false\t24814\t24814\t6404066507400987550\t0x8b04de5aad1f110fdda84f010e21add4b83e6733ca158dd091627fc790e28086\tW\tBANANA\t123\t1970-01-02T00:00:01.000000Z\n", - "tab latest by char", ts); - - expectSqlResult( - "boolean\tshort\tint\tlong\tlong256\tchar\tstring\tsymbol\tts\n" + - "false\t14817\t14817\t6404066507400987550\t0x386129f34be87b5e3990fb6012dac1d3495a30aaa8bf53224e89d27e7ee5104e\tM\t\t123\t1970-01-01T00:00:06.000000Z\n" + - "false\t14333\t14333\t8260188555232587029\t0x7ee65ec7b6e3bc3a422a8855e9d7bfd29199af5c2aa91ba39c022fa261bdede7\tJ\tCOCO\t123\t1970-01-01T00:00:07.000000Z\n" + - "true\t24814\t24814\t7759636733976435003\t0x386129f34be87b5e3990fb6012dac1d3495a30aaa8bf53224e89d27e7ee5104e\tJ\tORANGE\t123\t1970-01-01T00:00:09.000000Z\n" + - "false\t24814\t24814\t6404066507400987550\t0x8b04de5aad1f110fdda84f010e21add4b83e6733ca158dd091627fc790e28086\tW\tBANANA\t123\t1970-01-02T00:00:01.000000Z\n", - "tab latest by string", ts); - - expectSqlResult( - "boolean\tshort\tint\tlong\tlong256\tchar\tstring\tsymbol\tts\n" + - "true\t24814\t24814\t3614738589890112276\t0x7ee65ec7b6e3bc3a422a8855e9d7bfd29199af5c2aa91ba39c022fa261bdede7\tJ\t\tXoXoX\t1970-01-01T00:00:04.000000Z\n" + - "false\t14817\t14817\t8260188555232587029\t0x4e1c798ce76392e690c6042566c5a1cda5b9a155686af43ac109ac68336ea0c9\tZ\tBANANA\t_(*y*)_\t1970-01-01T00:00:08.000000Z\n" + - "false\t24814\t24814\t6404066507400987550\t0x8b04de5aad1f110fdda84f010e21add4b83e6733ca158dd091627fc790e28086\tW\tBANANA\t123\t1970-01-02T00:00:01.000000Z\n", - "tab latest by symbol", ts); - }); + private void createRndGeoHashTable() throws SqlException { + compiler.compile( + "create table x as (" + + "select" + + " cast(x as int) i," + + " rnd_symbol(113, 4, 4, 2) s," + + " timestamp_sequence(500000000000L,100000000L) ts," + + " rnd_geohash(5) geo1," + + " rnd_geohash(15) geo2," + + " rnd_geohash(25) geo4," + + " rnd_geohash(40) geo8" + + " from long_sequence(10000)" + + "), index(s) timestamp (ts) partition by DAY", + sqlExecutionContext + ); } - @Test - public void testLatestByTsIsPickedAtRuntimeNoDesignated() throws Exception { - assertMemoryLeak(() -> { - compiler.compile("create table tab(" + - " id symbol, " + - " name symbol, " + - " value double, " + - " other_ts timestamp, " + - " ts timestamp" + - ")", sqlExecutionContext); - executeInsert("insert into tab values ('d1', 'c1', 101.1, '2021-10-15T11:31:35.878Z', '2021-10-05T13:31:35.878Z')"); - executeInsert("insert into tab values ('d1', 'c1', 101.2, '2021-10-15T12:31:35.878Z', '2021-10-05T12:31:35.878Z')"); - executeInsert("insert into tab values ('d1', 'c1', 101.3, '2021-10-15T13:31:35.878Z', '2021-10-05T17:31:35.878Z')"); - executeInsert("insert into tab values ('d1', 'c1', 101.4, '2021-10-15T14:31:35.878Z', '2021-10-05T14:31:35.878Z')"); - executeInsert("insert into tab values ('d1', 'c2', 102.1, '2021-10-15T15:31:35.878Z', '2021-10-04T11:31:35.878Z')"); - executeInsert("insert into tab values ('d1', 'c2', 102.2, '2021-10-15T16:31:35.878Z', '2021-10-03T12:31:35.878Z')"); - executeInsert("insert into tab values ('d1', 'c2', 102.3, '2021-10-15T17:31:35.878Z', '2021-10-02T13:31:35.878Z')"); - executeInsert("insert into tab values ('d1', 'c2', 102.4, '2021-10-15T11:31:35.878Z', '2021-09-05T14:31:35.878Z')"); - executeInsert("insert into tab values ('d1', 'c2', 102.5, '2021-10-15T12:31:35.878Z', '2021-01-05T15:31:35.878Z')"); - executeInsert("insert into tab values ('d2', 'c1', 201.1, '2021-10-15T13:31:35.878Z', '2021-10-05T11:31:35.878Z')"); - executeInsert("insert into tab values ('d2', 'c1', 201.2, '2021-10-15T14:31:35.878Z', '2021-10-05T12:31:35.878Z')"); - executeInsert("insert into tab values ('d2', 'c1', 201.3, '2021-10-15T15:31:35.878Z', '2021-10-25T13:31:35.878Z')"); - executeInsert("insert into tab values ('d2', 'c1', 201.4, '2021-10-15T16:31:35.878Z', '2021-10-05T14:31:35.878Z')"); - executeInsert("insert into tab values ('d2', 'c2', 401.1, '2021-10-16T17:31:35.878Z', '2021-10-26T11:31:35.878Z')"); - executeInsert("insert into tab values ('d2', 'c1', 401.2, '2021-10-16T11:31:35.878Z', '2021-10-06T12:31:35.878Z')"); - executeInsert("insert into tab values ('d2', 'c1', 111.7, '2021-10-16T17:31:35.878Z', '2021-10-26T15:31:35.878Z')"); - assertSql( - "(tab latest by id where name in ('c2')) timestamp(other_ts)", - "id\tname\tvalue\tother_ts\tts\n" + - "d1\tc2\t102.5\t2021-10-15T12:31:35.878000Z\t2021-01-05T15:31:35.878000Z\n" + - "d2\tc2\t401.1\t2021-10-16T17:31:35.878000Z\t2021-10-26T11:31:35.878000Z\n"); - }); + private void executeInsertStatement(double d) throws SqlException { + String ddl = "insert into x (ds) values (" + d + ")"; + executeInsert(ddl); } - @Ignore() - // TODO: if the table has a designated timestamp, it becomes sticky - // on latest by, and we cannot change it explicitly. - // The query: (tab latest by id where name in ('c2')) timestamp(ts) - // - only interested in rows with name == 'c2' - // - for these, we want the latest row based on the ts column, - // using id as unique key for the whole row - @Test - public void testLatestByTsIsPickedAtRuntimeOtherThanDesignated() throws Exception { - assertMemoryLeak(() -> { - compiler.compile("create table tab(" + - " id symbol, " + - " name symbol, " + - " value double, " + - " other_ts timestamp, " + - " ts timestamp" + - ") timestamp(other_ts) partition by day", sqlExecutionContext); - executeInsert("insert into tab values ('d1', 'c1', 101.1, '2021-10-15T11:31:35.878Z', '2021-10-05T13:31:35.878Z')"); - executeInsert("insert into tab values ('d1', 'c1', 101.2, '2021-10-15T12:31:35.878Z', '2021-10-05T12:31:35.878Z')"); - executeInsert("insert into tab values ('d1', 'c1', 101.3, '2021-10-15T13:31:35.878Z', '2021-10-05T17:31:35.878Z')"); - executeInsert("insert into tab values ('d1', 'c1', 101.4, '2021-10-15T14:31:35.878Z', '2021-10-05T14:31:35.878Z')"); - executeInsert("insert into tab values ('d1', 'c2', 102.1, '2021-10-15T15:31:35.878Z', '2021-10-04T11:31:35.878Z')"); - executeInsert("insert into tab values ('d1', 'c2', 102.2, '2021-10-15T16:31:35.878Z', '2021-10-03T12:31:35.878Z')"); - executeInsert("insert into tab values ('d1', 'c2', 102.3, '2021-10-15T17:31:35.878Z', '2021-10-02T13:31:35.878Z')"); - executeInsert("insert into tab values ('d1', 'c2', 102.4, '2021-10-15T11:31:35.878Z', '2021-09-05T14:31:35.878Z')"); - executeInsert("insert into tab values ('d1', 'c2', 102.5, '2021-10-15T12:31:35.878Z', '2021-01-05T15:31:35.878Z')"); - executeInsert("insert into tab values ('d2', 'c1', 201.1, '2021-10-15T13:31:35.878Z', '2021-10-05T11:31:35.878Z')"); - executeInsert("insert into tab values ('d2', 'c1', 201.2, '2021-10-15T14:31:35.878Z', '2021-10-05T12:31:35.878Z')"); - executeInsert("insert into tab values ('d2', 'c1', 201.3, '2021-10-15T15:31:35.878Z', '2021-10-25T13:31:35.878Z')"); - executeInsert("insert into tab values ('d2', 'c1', 201.4, '2021-10-15T16:31:35.878Z', '2021-10-05T14:31:35.878Z')"); - executeInsert("insert into tab values ('d2', 'c2', 401.1, '2021-10-16T17:31:35.878Z', '2021-10-26T11:31:35.878Z')"); - executeInsert("insert into tab values ('d2', 'c1', 401.2, '2021-10-16T11:31:35.878Z', '2021-10-06T12:31:35.878Z')"); - executeInsert("insert into tab values ('d2', 'c1', 111.7, '2021-10-16T17:31:35.878Z', '2021-10-26T15:31:35.878Z')"); - assertSql( - "(tab latest by id where name in ('c2')) timestamp(ts)", - "id\tname\tvalue\tother_ts\tts\n" + - "d1\tc2\t102.10000000000001\t2021-10-15T17:31:35.878000Z\t2021-10-04T11:31:35.878000Z\n" + - "d2\tc2\t401.1\t2021-10-16T17:31:35.878000Z\t2021-10-26T11:31:35.878000Z\n"); - }); + private void expectSqlResult(CharSequence expected, CharSequence query, CharSequence ts) throws SqlException { + printSqlResult(expected, query, ts, + null, + null, + true, + true, + true, + false, + null); } - @Ignore - // TODO: fix, where is applied after latest by, the optimized I suspect - @Test - public void testLatestByIsApplicableToSubQueriesNoDesignatedTimestamp() throws Exception { + private void testBindVariableWithLike0(String keyword) throws Exception { assertMemoryLeak(() -> { - compiler.compile("create table tab(" + - " id symbol, " + - " name symbol, " + - " value double, " + - " other_ts timestamp, " + - " ts timestamp" + - ")", sqlExecutionContext); - executeInsert("insert into tab values ('d1', 'c1', 101.1, '2021-10-15T11:31:35.878Z', '2021-10-05T11:31:35.878Z')"); - executeInsert("insert into tab values ('d2', 'c1', 111.7, '2021-10-16T17:31:35.878Z', '2021-10-06T15:31:35.878Z')"); - assertSql( - "(tab latest by id where name in (select distinct name from tab where name != 'c2')) timestamp(other_ts)", - "id\tname\tvalue\tother_ts\tts\n" + - "d1\tc1\t101.4\t2021-10-15 14:31:35.878\t2021-10-05 14:31:35.878\n" + - "d2\tc1\t111.7\t2021-10-16 17:31:35.878\t2021-10-06 15:31:35.878\n"); - }); - } + final CairoConfiguration configuration = new DefaultCairoConfiguration(root); + try ( + CairoEngine engine = new CairoEngine(configuration); + SqlCompiler compiler = new SqlCompiler(engine) + ) { + compiler.compile("create table xy as (select rnd_str() v from long_sequence(100))", sqlExecutionContext); + bindVariableService.clear(); + try (RecordCursorFactory factory = compiler.compile("xy where v " + keyword + " $1", sqlExecutionContext).getRecordCursorFactory()) { - @Test - public void testLatestByMultiColumnPlusFilter0() throws Exception { - testLatestByMultiColumnPlusFilter("create table tab(" + - " id symbol, " + - " name symbol, " + - " value double, " + - " ts timestamp" + - ")"); - } + bindVariableService.setStr(0, "MBE%"); + assertCursor("v\n" + + "MBEZGHW\n", + factory, + true, + true, + false + ); - @Test - public void testLatestByMultiColumnPlusFilter1() throws Exception { - testLatestByMultiColumnPlusFilter("create table tab(" + - " id symbol, " + - " name symbol, " + - " value double, " + - " ts timestamp" + - ") timestamp(ts) partition by DAY"); - } + bindVariableService.setStr(0, "Z%"); + assertCursor("v\n" + + "ZSQLDGLOG\n" + + "ZLUOG\n" + + "ZLCBDMIG\n" + + "ZJYYFLSVI\n" + + "ZWEVQTQO\n" + + "ZSFXUNYQ\n", + factory, + true, + true, + false + ); - @Ignore("LatestByAllIndexedFilteredRecordCursorFactory applies filter after latest by is executed") - @Test - public void testLatestByMultiColumnPlusFilter2() throws Exception { - testLatestByMultiColumnPlusFilter("create table tab(" + - " id symbol index, " + - " name symbol, " + - " value double, " + - " ts timestamp" + - ") timestamp(ts) partition by DAY"); - } + assertCursor("v\n" + + "ZSQLDGLOG\n" + + "ZLUOG\n" + + "ZLCBDMIG\n" + + "ZJYYFLSVI\n" + + "ZWEVQTQO\n" + + "ZSFXUNYQ\n", + factory, + true, + true, + false + ); - @Ignore("LatestByAllIndexedFilteredRecordCursorFactory applies filter after latest by is executed") - @Test - public void testLatestByMultiColumnPlusFilter3() throws Exception { - testLatestByMultiColumnPlusFilter("create table tab(" + - " id symbol, " + - " name symbol index, " + - " value double, " + - " ts timestamp" + - ") timestamp(ts) partition by DAY"); - } - @Ignore("LatestByAllIndexedFilteredRecordCursorFactory applies filter after latest by is executed") - @Test - public void testLatestByMultiColumnPlusFilter4() throws Exception { - testLatestByMultiColumnPlusFilter("create table tab(" + - " id symbol index, " + - " name symbol index, " + - " value double, " + - " ts timestamp" + - ") timestamp(ts) partition by DAY"); + bindVariableService.setStr(0, null); + assertCursor("v\n", + factory, + true, + true, + false + ); + } + } + }); } - @Ignore("LatestByAllIndexedFilteredRecordCursorFactory applies filter after latest by is executed") - @Test - public void testLatestByMultiColumnPlusFilter5() throws Exception { - testLatestByMultiColumnPlusFilter("create table tab(" + - " id symbol index, " + - " name symbol index, " + - " value double, " + - " ts timestamp" + - ")"); - } + private void testLatestByFilteredBySymbolIn(String ddl) throws Exception { + assertMemoryLeak(() -> { + compiler.compile(ddl, sqlExecutionContext); - @Ignore("LatestByAllIndexedFilteredRecordCursorFactory applies filter after latest by is executed") - @Test - public void testLatestByMultiColumnPlusFilter6() throws Exception { - testLatestByMultiColumnPlusFilter("create table tab(" + - " id symbol, " + - " name symbol index, " + - " value double, " + - " ts timestamp" + - ")"); - } + executeInsert("insert into x values ('2021-11-17T17:35:01.000000Z', 'node1', 'cpu', 1)"); + executeInsert("insert into x values ('2021-11-17T17:35:02.000000Z', 'node1', 'cpu', 10)"); + executeInsert("insert into x values ('2021-11-17T17:35:03.000000Z', 'node1', 'cpu', 100)"); + executeInsert("insert into x values ('2021-11-17T17:35:01.000000Z', 'node2', 'cpu', 7)"); + executeInsert("insert into x values ('2021-11-17T17:35:02.000000Z', 'node2', 'cpu', 15)"); + executeInsert("insert into x values ('2021-11-17T17:35:03.000000Z', 'node2', 'cpu', 75)"); + executeInsert("insert into x values ('2021-11-17T17:35:01.000000Z', 'node3', 'cpu', 5)"); + executeInsert("insert into x values ('2021-11-17T17:35:02.000000Z', 'node3', 'cpu', 20)"); + executeInsert("insert into x values ('2021-11-17T17:35:03.000000Z', 'node3', 'cpu', 25)"); + executeInsert("insert into x values ('2021-11-17T17:35:01.000000Z', 'node1', 'memory', 20)"); + executeInsert("insert into x values ('2021-11-17T17:35:02.000000Z', 'node1', 'memory', 200)"); + executeInsert("insert into x values ('2021-11-17T17:35:03.000000Z', 'node1', 'memory', 2000)"); + executeInsert("insert into x values ('2021-11-17T17:35:01.000000Z', 'node2', 'memory', 30)"); + executeInsert("insert into x values ('2021-11-17T17:35:02.000000Z', 'node2', 'memory', 300)"); + executeInsert("insert into x values ('2021-11-17T17:35:03.000000Z', 'node2', 'memory', 3000)"); + executeInsert("insert into x values ('2021-11-17T17:35:01.000000Z', 'node3', 'memory', 40)"); + executeInsert("insert into x values ('2021-11-17T17:35:02.000000Z', 'node3', 'memory', 400)"); + executeInsert("insert into x values ('2021-11-17T17:35:03.000000Z', 'node3', 'memory', 4000)"); - @Ignore("LatestByAllIndexedFilteredRecordCursorFactory applies filter after latest by is executed") - @Test - public void testLatestByMultiColumnPlusFilter7() throws Exception { - testLatestByMultiColumnPlusFilter("create table tab(" + - " id symbol index, " + - " name symbol, " + - " value double, " + - " ts timestamp" + - ")"); + TestUtils.assertSql( + compiler, + sqlExecutionContext, + "select metric, sum(value) from x latest by node \n" + + "where node in ('node1', 'node2') and metric in ('cpu')", + sink, + "metric\tsum\n" + + "cpu\t175\n" + ); + }); } private void testLatestByMultiColumnPlusFilter(CharSequence ddl) throws Exception { @@ -6994,164 +7067,117 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest { }); } - @Ignore("result order is currently dependent on stability of sorting method") - // TODO: this is broken, the expected result order for "select * from tab" in the presence - // of repeated timestamps needs to be predefined and consistent, one of two alternatives: - // 1.- most recent insert for a given timestamp first: - // "d2\tc1\t201.10000000000002\t2021-10-05T11:31:35.878000Z\n" + - // "d1\tc2\t102.10000000000001\t2021-10-05T11:31:35.878000Z\n" + - // "d1\tc1\t101.10000000000001\t2021-10-05T11:31:35.878000Z\n" + - // "d2\tc1\t201.20000000000002\t2021-10-05T12:31:35.878000Z\n" + - // "d1\tc2\t102.2\t2021-10-05T12:31:35.878000Z\n" + - // "d1\tc1\t101.2\t2021-10-05T12:31:35.878000Z\n" + - // "d2\tc1\t201.3\t2021-10-05T13:31:35.878000Z\n" + - // "d1\tc2\t102.30000000000001\t2021-10-05T13:31:35.878000Z\n" + - // "d1\tc1\t101.30000000000001\t2021-10-05T13:31:35.878000Z\n" + - // "d2\tc1\t201.4\t2021-10-05T14:31:35.878000Z\n" + - // "d1\tc2\t102.4\t2021-10-05T14:31:35.878000Z\n" + - // "d1\tc1\t101.4\t2021-10-05T14:31:35.878000Z\n" + - // "d1\tc2\t102.5\t2021-10-05T15:31:35.878000Z\n" + - // "d2\tc2\t401.1\t2021-10-06T11:31:35.878000Z\n" + - // "d2\tc1\t401.20000000000005\t2021-10-06T12:31:35.878000Z\n" + - // "d2\tc1\t111.7\t2021-10-06T15:31:35.878000Z\n" - // 2.- least recent insert for a given timestamp first: - // "d1\tc1\t101.10000000000001\t2021-10-05T11:31:35.878000Z\n" + - // "d1\tc2\t102.10000000000001\t2021-10-05T11:31:35.878000Z\n" + - // "d2\tc1\t201.10000000000002\t2021-10-05T11:31:35.878000Z\n" + - // "d1\tc1\t101.2\t2021-10-05T12:31:35.878000Z\n" + - // "d1\tc2\t102.2\t2021-10-05T12:31:35.878000Z\n" + - // "d2\tc1\t201.20000000000002\t2021-10-05T12:31:35.878000Z\n" + - // "d1\tc1\t101.30000000000001\t2021-10-05T13:31:35.878000Z\n" + - // "d1\tc2\t102.30000000000001\t2021-10-05T13:31:35.878000Z\n" + - // "d2\tc1\t201.3\t2021-10-05T13:31:35.878000Z\n" + - // "d1\tc1\t101.4\t2021-10-05T14:31:35.878000Z\n" + - // "d1\tc2\t102.4\t2021-10-05T14:31:35.878000Z\n" + - // "d2\tc1\t201.4\t2021-10-05T14:31:35.878000Z\n" + - // "d1\tc2\t102.5\t2021-10-05T15:31:35.878000Z\n" + - // "d2\tc2\t401.1\t2021-10-06T11:31:35.878000Z\n" + - // "d2\tc1\t401.20000000000005\t2021-10-06T12:31:35.878000Z\n" + - // "d2\tc1\t111.7\t2021-10-06T15:31:35.878000Z\n" - // in the assertSql that follows, option #2 has been taken. - @Test - public void testSelectExpectedOrder() throws Exception { + private void testLatestBySelectAllFilteredBySymbolIn(String ddl) throws Exception { assertMemoryLeak(() -> { - compiler.compile("create table tab(" + - " id symbol index, " + - " name symbol index, " + - " value double, " + - " ts timestamp" + - ") timestamp(ts) partition by DAY", sqlExecutionContext); - executeInsert("insert into tab values ('d1', 'c1', 101.1, '2021-10-05T11:31:35.878Z')"); - executeInsert("insert into tab values ('d1', 'c1', 101.2, '2021-10-05T12:31:35.878Z')"); - executeInsert("insert into tab values ('d1', 'c1', 101.3, '2021-10-05T13:31:35.878Z')"); - executeInsert("insert into tab values ('d1', 'c1', 101.4, '2021-10-05T14:31:35.878Z')"); - executeInsert("insert into tab values ('d1', 'c2', 102.1, '2021-10-05T11:31:35.878Z')"); - executeInsert("insert into tab values ('d1', 'c2', 102.2, '2021-10-05T12:31:35.878Z')"); - executeInsert("insert into tab values ('d1', 'c2', 102.3, '2021-10-05T13:31:35.878Z')"); - executeInsert("insert into tab values ('d1', 'c2', 102.4, '2021-10-05T14:31:35.878Z')"); - executeInsert("insert into tab values ('d1', 'c2', 102.5, '2021-10-05T15:31:35.878Z')"); - executeInsert("insert into tab values ('d2', 'c1', 201.1, '2021-10-05T11:31:35.878Z')"); - executeInsert("insert into tab values ('d2', 'c1', 201.2, '2021-10-05T12:31:35.878Z')"); - executeInsert("insert into tab values ('d2', 'c1', 201.3, '2021-10-05T13:31:35.878Z')"); - executeInsert("insert into tab values ('d2', 'c1', 201.4, '2021-10-05T14:31:35.878Z')"); - executeInsert("insert into tab values ('d2', 'c2', 401.1, '2021-10-06T11:31:35.878Z')"); - executeInsert("insert into tab values ('d2', 'c1', 401.2, '2021-10-06T12:31:35.878Z')"); - executeInsert("insert into tab values ('d2', 'c1', 111.7, '2021-10-06T15:31:35.878Z')"); - assertSql( - "tab", - "id\tname\tvalue\tts\n" + - "d1\tc1\t101.10000000000001\t2021-10-05T11:31:35.878000Z\n" + - "d1\tc2\t102.10000000000001\t2021-10-05T11:31:35.878000Z\n" + - "d2\tc1\t201.10000000000002\t2021-10-05T11:31:35.878000Z\n" + - "d1\tc1\t101.2\t2021-10-05T12:31:35.878000Z\n" + - "d1\tc2\t102.2\t2021-10-05T12:31:35.878000Z\n" + - "d2\tc1\t201.20000000000002\t2021-10-05T12:31:35.878000Z\n" + - "d1\tc1\t101.30000000000001\t2021-10-05T13:31:35.878000Z\n" + - "d1\tc2\t102.30000000000001\t2021-10-05T13:31:35.878000Z\n" + - "d2\tc1\t201.3\t2021-10-05T13:31:35.878000Z\n" + - "d1\tc1\t101.4\t2021-10-05T14:31:35.878000Z\n" + - "d1\tc2\t102.4\t2021-10-05T14:31:35.878000Z\n" + - "d2\tc1\t201.4\t2021-10-05T14:31:35.878000Z\n" + - "d1\tc2\t102.5\t2021-10-05T15:31:35.878000Z\n" + - "d2\tc2\t401.1\t2021-10-06T11:31:35.878000Z\n" + - "d2\tc1\t401.20000000000005\t2021-10-06T12:31:35.878000Z\n" + - "d2\tc1\t111.7\t2021-10-06T15:31:35.878000Z\n"); - }); - } + compiler.compile(ddl, sqlExecutionContext); - private void expectSqlResult(CharSequence expected, CharSequence query, CharSequence ts) throws SqlException { - printSqlResult(expected, query, ts, - null, - null, - true, - true, - true, - false, - null); - } + executeInsert("insert into x values ('2021-11-17T17:35:01.000000Z', 'node1', 'cpu', 1)"); + executeInsert("insert into x values ('2021-11-17T17:35:02.000000Z', 'node1', 'cpu', 10)"); + executeInsert("insert into x values ('2021-11-17T17:35:03.000000Z', 'node1', 'cpu', 100)"); + executeInsert("insert into x values ('2021-11-17T17:35:01.000000Z', 'node2', 'cpu', 7)"); + executeInsert("insert into x values ('2021-11-17T17:35:02.000000Z', 'node2', 'cpu', 15)"); + executeInsert("insert into x values ('2021-11-17T17:35:03.000000Z', 'node2', 'cpu', 75)"); + executeInsert("insert into x values ('2021-11-17T17:35:01.000000Z', 'node3', 'cpu', 5)"); + executeInsert("insert into x values ('2021-11-17T17:35:02.000000Z', 'node3', 'cpu', 20)"); + executeInsert("insert into x values ('2021-11-17T17:35:03.000000Z', 'node3', 'cpu', 25)"); + executeInsert("insert into x values ('2021-11-17T17:35:01.000000Z', 'node1', 'memory', 20)"); + executeInsert("insert into x values ('2021-11-17T17:35:02.000000Z', 'node1', 'memory', 200)"); + executeInsert("insert into x values ('2021-11-17T17:35:03.000000Z', 'node1', 'memory', 2000)"); + executeInsert("insert into x values ('2021-11-17T17:35:01.000000Z', 'node2', 'memory', 30)"); + executeInsert("insert into x values ('2021-11-17T17:35:02.000000Z', 'node2', 'memory', 300)"); + executeInsert("insert into x values ('2021-11-17T17:35:03.000000Z', 'node2', 'memory', 3000)"); + executeInsert("insert into x values ('2021-11-17T17:35:01.000000Z', 'node3', 'memory', 40)"); + executeInsert("insert into x values ('2021-11-17T17:35:02.000000Z', 'node3', 'memory', 400)"); + executeInsert("insert into x values ('2021-11-17T17:35:03.000000Z', 'node3', 'memory', 4000)"); - private void executeInsertStatement(double d) throws SqlException { - String ddl = "insert into x (ds) values (" + d + ")"; - executeInsert(ddl); + TestUtils.assertSql( + compiler, + sqlExecutionContext, + "select * from x latest by node, metric \n" + + "where node in ('node2') and metric in ('cpu', 'memory')", + sink, + "ts\tnode\tmetric\tvalue\n" + + "2021-11-17T17:35:03.000000Z\tnode2\tcpu\t75\n" + + "2021-11-17T17:35:03.000000Z\tnode2\tmemory\t3000\n" + ); + }); } - private void testBindVariableWithLike0(String keyword) throws Exception { + private void testLatestBySupportedColumnTypes(CharSequence ddl, CharSequence ts) throws Exception { assertMemoryLeak(() -> { - final CairoConfiguration configuration = new DefaultCairoConfiguration(root); - try ( - CairoEngine engine = new CairoEngine(configuration); - SqlCompiler compiler = new SqlCompiler(engine) - ) { - compiler.compile("create table xy as (select rnd_str() v from long_sequence(100))", sqlExecutionContext); - bindVariableService.clear(); - try (RecordCursorFactory factory = compiler.compile("xy where v " + keyword + " $1", sqlExecutionContext).getRecordCursorFactory()) { + // supported: [BOOLEAN, CHAR, INT, LONG, LONG256, STRING, SYMBOL] + compiler.compile(ddl, sqlExecutionContext); + executeInsert("insert into tab values (false, cast(24814 as short), 24814, 8260188555232587029, 0x7ee65ec7b6e3bc3a422a8855e9d7bfd29199af5c2aa91ba39c022fa261bdede7, 'J', 'ORANGE', '123', '1970-01-01T00:00:01.000000Z')"); + executeInsert("insert into tab values (true, cast(14817 as short), 14817, 8260188555232587029, 0x4e1c798ce76392e690c6042566c5a1cda5b9a155686af43ac109ac68336ea0c9, 'A', 'COCO', 'XoXoX', '1970-01-01T00:00:02.000000Z')"); + executeInsert("insert into tab values (true, cast(14817 as short), 14817, 3614738589890112276, 0x386129f34be87b5e3990fb6012dac1d3495a30aaa8bf53224e89d27e7ee5104e, 'Q', null, 'XoXoX', '1970-01-01T00:00:03.000000Z')"); + executeInsert("insert into tab values (true, cast(24814 as short), 24814, 3614738589890112276, 0x7ee65ec7b6e3bc3a422a8855e9d7bfd29199af5c2aa91ba39c022fa261bdede7, 'J', null, 'XoXoX', '1970-01-01T00:00:04.000000Z')"); + executeInsert("insert into tab values (true, cast(24814 as short), 24814, 8260188555232587029, 0x4e1c798ce76392e690c6042566c5a1cda5b9a155686af43ac109ac68336ea0c9, 'Q', 'BANANA', '_(*y*)_', '1970-01-01T00:00:05.000000Z')"); + executeInsert("insert into tab values (false, cast(14817 as short), 14817, 6404066507400987550, 0x386129f34be87b5e3990fb6012dac1d3495a30aaa8bf53224e89d27e7ee5104e, 'M', null, '123', '1970-01-01T00:00:06.000000Z')"); + executeInsert("insert into tab values (false, cast(14333 as short), 14333, 8260188555232587029, 0x7ee65ec7b6e3bc3a422a8855e9d7bfd29199af5c2aa91ba39c022fa261bdede7, 'J', 'COCO', '123', '1970-01-01T00:00:07.000000Z')"); + executeInsert("insert into tab values (false, cast(14817 as short), 14817, 8260188555232587029, 0x4e1c798ce76392e690c6042566c5a1cda5b9a155686af43ac109ac68336ea0c9, 'Z', 'BANANA', '_(*y*)_', '1970-01-01T00:00:08.000000Z')"); + executeInsert("insert into tab values (true, cast(24814 as short), 24814, 7759636733976435003, 0x386129f34be87b5e3990fb6012dac1d3495a30aaa8bf53224e89d27e7ee5104e, 'J', 'ORANGE', '123', '1970-01-01T00:00:09.000000Z')"); + executeInsert("insert into tab values (false, cast(24814 as short), 24814, 6404066507400987550, 0x8b04de5aad1f110fdda84f010e21add4b83e6733ca158dd091627fc790e28086, 'W', 'BANANA', '123', '1970-01-01T00:00:10.000000Z')"); + executeInsert("insert into tab values (false, cast(24814 as short), 24814, 6404066507400987550, 0x8b04de5aad1f110fdda84f010e21add4b83e6733ca158dd091627fc790e28086, 'W', 'BANANA', '123', '1970-01-02T00:00:01.000000Z')"); + expectSqlResult( + "boolean\tshort\tint\tlong\tlong256\tchar\tstring\tsymbol\tts\n" + + "true\t24814\t24814\t7759636733976435003\t0x386129f34be87b5e3990fb6012dac1d3495a30aaa8bf53224e89d27e7ee5104e\tJ\tORANGE\t123\t1970-01-01T00:00:09.000000Z\n" + + "false\t24814\t24814\t6404066507400987550\t0x8b04de5aad1f110fdda84f010e21add4b83e6733ca158dd091627fc790e28086\tW\tBANANA\t123\t1970-01-02T00:00:01.000000Z\n", + "tab latest by boolean", ts); - bindVariableService.setStr(0, "MBE%"); - assertCursor("v\n" + - "MBEZGHW\n", - factory, - true, - true, - false - ); + expectSqlResult( + "boolean\tshort\tint\tlong\tlong256\tchar\tstring\tsymbol\tts\n" + + "false\t14333\t14333\t8260188555232587029\t0x7ee65ec7b6e3bc3a422a8855e9d7bfd29199af5c2aa91ba39c022fa261bdede7\tJ\tCOCO\t123\t1970-01-01T00:00:07.000000Z\n" + + "false\t14817\t14817\t8260188555232587029\t0x4e1c798ce76392e690c6042566c5a1cda5b9a155686af43ac109ac68336ea0c9\tZ\tBANANA\t_(*y*)_\t1970-01-01T00:00:08.000000Z\n" + + "false\t24814\t24814\t6404066507400987550\t0x8b04de5aad1f110fdda84f010e21add4b83e6733ca158dd091627fc790e28086\tW\tBANANA\t123\t1970-01-02T00:00:01.000000Z\n", + "tab timestamp(ts) latest by short", "ts"); - bindVariableService.setStr(0, "Z%"); - assertCursor("v\n" + - "ZSQLDGLOG\n" + - "ZLUOG\n" + - "ZLCBDMIG\n" + - "ZJYYFLSVI\n" + - "ZWEVQTQO\n" + - "ZSFXUNYQ\n", - factory, - true, - true, - false - ); + expectSqlResult( + "boolean\tshort\tint\tlong\tlong256\tchar\tstring\tsymbol\tts\n" + + "false\t14333\t14333\t8260188555232587029\t0x7ee65ec7b6e3bc3a422a8855e9d7bfd29199af5c2aa91ba39c022fa261bdede7\tJ\tCOCO\t123\t1970-01-01T00:00:07.000000Z\n" + + "false\t14817\t14817\t8260188555232587029\t0x4e1c798ce76392e690c6042566c5a1cda5b9a155686af43ac109ac68336ea0c9\tZ\tBANANA\t_(*y*)_\t1970-01-01T00:00:08.000000Z\n" + + "false\t24814\t24814\t6404066507400987550\t0x8b04de5aad1f110fdda84f010e21add4b83e6733ca158dd091627fc790e28086\tW\tBANANA\t123\t1970-01-02T00:00:01.000000Z\n", + "tab latest by int", ts); - assertCursor("v\n" + - "ZSQLDGLOG\n" + - "ZLUOG\n" + - "ZLCBDMIG\n" + - "ZJYYFLSVI\n" + - "ZWEVQTQO\n" + - "ZSFXUNYQ\n", - factory, - true, - true, - false - ); + expectSqlResult( + "boolean\tshort\tint\tlong\tlong256\tchar\tstring\tsymbol\tts\n" + + "true\t24814\t24814\t3614738589890112276\t0x7ee65ec7b6e3bc3a422a8855e9d7bfd29199af5c2aa91ba39c022fa261bdede7\tJ\t\tXoXoX\t1970-01-01T00:00:04.000000Z\n" + + "false\t14817\t14817\t8260188555232587029\t0x4e1c798ce76392e690c6042566c5a1cda5b9a155686af43ac109ac68336ea0c9\tZ\tBANANA\t_(*y*)_\t1970-01-01T00:00:08.000000Z\n" + + "true\t24814\t24814\t7759636733976435003\t0x386129f34be87b5e3990fb6012dac1d3495a30aaa8bf53224e89d27e7ee5104e\tJ\tORANGE\t123\t1970-01-01T00:00:09.000000Z\n" + + "false\t24814\t24814\t6404066507400987550\t0x8b04de5aad1f110fdda84f010e21add4b83e6733ca158dd091627fc790e28086\tW\tBANANA\t123\t1970-01-02T00:00:01.000000Z\n", + "tab latest by long", ts); + expectSqlResult( + "boolean\tshort\tint\tlong\tlong256\tchar\tstring\tsymbol\tts\n" + + "false\t14333\t14333\t8260188555232587029\t0x7ee65ec7b6e3bc3a422a8855e9d7bfd29199af5c2aa91ba39c022fa261bdede7\tJ\tCOCO\t123\t1970-01-01T00:00:07.000000Z\n" + + "false\t14817\t14817\t8260188555232587029\t0x4e1c798ce76392e690c6042566c5a1cda5b9a155686af43ac109ac68336ea0c9\tZ\tBANANA\t_(*y*)_\t1970-01-01T00:00:08.000000Z\n" + + "true\t24814\t24814\t7759636733976435003\t0x386129f34be87b5e3990fb6012dac1d3495a30aaa8bf53224e89d27e7ee5104e\tJ\tORANGE\t123\t1970-01-01T00:00:09.000000Z\n" + + "false\t24814\t24814\t6404066507400987550\t0x8b04de5aad1f110fdda84f010e21add4b83e6733ca158dd091627fc790e28086\tW\tBANANA\t123\t1970-01-02T00:00:01.000000Z\n", + "tab latest by long256", ts); - bindVariableService.setStr(0, null); - assertCursor("v\n", - factory, - true, - true, - false - ); - } - } + expectSqlResult( + "boolean\tshort\tint\tlong\tlong256\tchar\tstring\tsymbol\tts\n" + + "true\t14817\t14817\t8260188555232587029\t0x4e1c798ce76392e690c6042566c5a1cda5b9a155686af43ac109ac68336ea0c9\tA\tCOCO\tXoXoX\t1970-01-01T00:00:02.000000Z\n" + + "true\t24814\t24814\t8260188555232587029\t0x4e1c798ce76392e690c6042566c5a1cda5b9a155686af43ac109ac68336ea0c9\tQ\tBANANA\t_(*y*)_\t1970-01-01T00:00:05.000000Z\n" + + "false\t14817\t14817\t6404066507400987550\t0x386129f34be87b5e3990fb6012dac1d3495a30aaa8bf53224e89d27e7ee5104e\tM\t\t123\t1970-01-01T00:00:06.000000Z\n" + + "false\t14817\t14817\t8260188555232587029\t0x4e1c798ce76392e690c6042566c5a1cda5b9a155686af43ac109ac68336ea0c9\tZ\tBANANA\t_(*y*)_\t1970-01-01T00:00:08.000000Z\n" + + "true\t24814\t24814\t7759636733976435003\t0x386129f34be87b5e3990fb6012dac1d3495a30aaa8bf53224e89d27e7ee5104e\tJ\tORANGE\t123\t1970-01-01T00:00:09.000000Z\n" + + "false\t24814\t24814\t6404066507400987550\t0x8b04de5aad1f110fdda84f010e21add4b83e6733ca158dd091627fc790e28086\tW\tBANANA\t123\t1970-01-02T00:00:01.000000Z\n", + "tab latest by char", ts); + + expectSqlResult( + "boolean\tshort\tint\tlong\tlong256\tchar\tstring\tsymbol\tts\n" + + "false\t14817\t14817\t6404066507400987550\t0x386129f34be87b5e3990fb6012dac1d3495a30aaa8bf53224e89d27e7ee5104e\tM\t\t123\t1970-01-01T00:00:06.000000Z\n" + + "false\t14333\t14333\t8260188555232587029\t0x7ee65ec7b6e3bc3a422a8855e9d7bfd29199af5c2aa91ba39c022fa261bdede7\tJ\tCOCO\t123\t1970-01-01T00:00:07.000000Z\n" + + "true\t24814\t24814\t7759636733976435003\t0x386129f34be87b5e3990fb6012dac1d3495a30aaa8bf53224e89d27e7ee5104e\tJ\tORANGE\t123\t1970-01-01T00:00:09.000000Z\n" + + "false\t24814\t24814\t6404066507400987550\t0x8b04de5aad1f110fdda84f010e21add4b83e6733ca158dd091627fc790e28086\tW\tBANANA\t123\t1970-01-02T00:00:01.000000Z\n", + "tab latest by string", ts); + + expectSqlResult( + "boolean\tshort\tint\tlong\tlong256\tchar\tstring\tsymbol\tts\n" + + "true\t24814\t24814\t3614738589890112276\t0x7ee65ec7b6e3bc3a422a8855e9d7bfd29199af5c2aa91ba39c022fa261bdede7\tJ\t\tXoXoX\t1970-01-01T00:00:04.000000Z\n" + + "false\t14817\t14817\t8260188555232587029\t0x4e1c798ce76392e690c6042566c5a1cda5b9a155686af43ac109ac68336ea0c9\tZ\tBANANA\t_(*y*)_\t1970-01-01T00:00:08.000000Z\n" + + "false\t24814\t24814\t6404066507400987550\t0x8b04de5aad1f110fdda84f010e21add4b83e6733ca158dd091627fc790e28086\tW\tBANANA\t123\t1970-01-02T00:00:01.000000Z\n", + "tab latest by symbol", ts); }); } } diff --git a/core/src/test/java/io/questdb/log/LogFactoryTest.java b/core/src/test/java/io/questdb/log/LogFactoryTest.java index 3ed69f3450ab7ca5342c9f1ca96521355b207be8..f355eade3da3d66a18fdeab23b03e2537d247339 100644 --- a/core/src/test/java/io/questdb/log/LogFactoryTest.java +++ b/core/src/test/java/io/questdb/log/LogFactoryTest.java @@ -31,8 +31,8 @@ import io.questdb.mp.SPSequence; import io.questdb.std.*; import io.questdb.std.datetime.microtime.MicrosecondClock; import io.questdb.std.datetime.microtime.TimestampFormatUtils; -import io.questdb.std.str.NativeLPSZ; import io.questdb.std.str.Path; +import io.questdb.std.str.StringSink; import io.questdb.test.tools.TestUtils; import org.junit.Assert; import org.junit.Rule; @@ -544,20 +544,21 @@ public class LogFactoryTest { int fileCount = 0; try (Path path = new Path()) { - NativeLPSZ lpsz = new NativeLPSZ(); + StringSink fileNameSink = new StringSink(); path.of(base).$(); long pFind = Files.findFirst(path); try { Assert.assertTrue(pFind != 0); do { - lpsz.of(Files.findName(pFind)); - if (Files.isDots(lpsz)) { + fileNameSink.clear(); + Chars.utf8DecodeZ(Files.findName(pFind), fileNameSink); + if (Files.isDots(fileNameSink)) { continue; } // don't hardcode hour, it is liable to vary // because of different default timezones - TestUtils.assertContains(lpsz, mustContain); - Assert.assertFalse(Chars.contains(lpsz, ".1")); + TestUtils.assertContains(fileNameSink, mustContain); + Assert.assertFalse(Chars.contains(fileNameSink, ".1")); fileCount++; } while (Files.findNext(pFind) > 0); diff --git a/core/src/test/java/io/questdb/network/NetTest.java b/core/src/test/java/io/questdb/network/NetTest.java index a1265f78d6087219bc582cf1d30855e3e2e749fd..0f02e6e5be26c06e8e59e525aca07d4363c953d8 100644 --- a/core/src/test/java/io/questdb/network/NetTest.java +++ b/core/src/test/java/io/questdb/network/NetTest.java @@ -24,11 +24,11 @@ package io.questdb.network; +import io.questdb.std.Chars; import io.questdb.std.MemoryTag; import io.questdb.std.Os; import io.questdb.std.Unsafe; import io.questdb.std.str.CharSequenceZ; -import io.questdb.std.str.NativeLPSZ; import io.questdb.std.str.StringSink; import io.questdb.test.tools.TestUtils; import org.junit.Assert; @@ -211,8 +211,8 @@ public class NetTest { @Test public void testSeek() { int port = 9993; - NativeLPSZ lpsz = new NativeLPSZ(); String msg = "Test ABC"; + StringSink sink = new StringSink(); CharSequenceZ charSink = new CharSequenceZ(msg); int msgLen = charSink.length() + 1; @@ -231,11 +231,12 @@ public class NetTest { long serverFd = Net.accept(acceptFd); long serverBuf = Unsafe.malloc(msgLen, MemoryTag.NATIVE_DEFAULT); Assert.assertEquals(msgLen, Net.peek(serverFd, serverBuf, msgLen)); - lpsz.of(serverBuf); - Assert.assertEquals(msg, lpsz.toString()); + Chars.utf8DecodeZ(serverBuf, sink); + TestUtils.assertEquals(msg, sink); Assert.assertEquals(msgLen, Net.recv(serverFd, serverBuf, msgLen)); - lpsz.of(serverBuf); - Assert.assertEquals(msg, lpsz.toString()); + sink.clear(); + Chars.utf8DecodeZ(serverBuf, sink); + TestUtils.assertEquals(msg, sink); Unsafe.free(serverBuf, msgLen, MemoryTag.NATIVE_DEFAULT); Net.close(serverFd);