From 762a0d77dbe6efbeb4fe76170496542498bfc5d6 Mon Sep 17 00:00:00 2001 From: Andrey Pechkurov <37772591+puzpuzpuz@users.noreply.github.com> Date: Thu, 21 Oct 2021 21:04:33 +0300 Subject: [PATCH] fix(ilp): boolean to number conversion for TCP (#1418) Boolean to number conversion was only supported for long fields in ILP over TCP. This commit extends the conversion to support other numeric types: byte, short, int, float, double. Also fixes Boolean format (single byte) being used when writing converted Boolean values on disk. Important note: this is a potentially breaking change. --- .../java/io/questdb/cairo/ColumnType.java | 4 -- .../line/tcp/LineTcpMeasurementScheduler.java | 41 +++++++++--- .../line/tcp/LineTcpInsertOtherTypesTest.java | 62 +++++++++++++++---- 3 files changed, 85 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/io/questdb/cairo/ColumnType.java b/core/src/main/java/io/questdb/cairo/ColumnType.java index 653643346..a93810a9d 100644 --- a/core/src/main/java/io/questdb/cairo/ColumnType.java +++ b/core/src/main/java/io/questdb/cairo/ColumnType.java @@ -162,10 +162,6 @@ public final class ColumnType { return columnType == ColumnType.INT; } - public static boolean isLong(int columnType) { - return columnType == ColumnType.LONG; - } - public static boolean isNull(int columnType) { return columnType == NULL; } diff --git a/core/src/main/java/io/questdb/cutlass/line/tcp/LineTcpMeasurementScheduler.java b/core/src/main/java/io/questdb/cutlass/line/tcp/LineTcpMeasurementScheduler.java index 6c9f266ba..26c95d053 100644 --- a/core/src/main/java/io/questdb/cutlass/line/tcp/LineTcpMeasurementScheduler.java +++ b/core/src/main/java/io/questdb/cutlass/line/tcp/LineTcpMeasurementScheduler.java @@ -823,13 +823,40 @@ class LineTcpMeasurementScheduler implements Closeable { byte b = Unsafe.getUnsafe().getByte(bufPos); bufPos += Byte.BYTES; final int colType = writer.getMetadata().getColumnType(colIndex); - if (ColumnType.isBoolean(colType) || ColumnType.isLong(colType)) { - row.putBool(colIndex, b == 1); - } else { - throw CairoException.instance(0) - .put("cast error for line protocol boolean [columnIndex=").put(colIndex) - .put(", columnType=").put(ColumnType.nameOf(colType)) - .put(']'); + switch (ColumnType.tagOf(colType)) { + case ColumnType.BOOLEAN: + row.putBool(colIndex, b == 1); + break; + + case ColumnType.BYTE: + row.putByte(colIndex, b); + break; + + case ColumnType.SHORT: + row.putShort(colIndex, b); + break; + + case ColumnType.INT: + row.putInt(colIndex, b); + break; + + case ColumnType.LONG: + row.putLong(colIndex, b); + break; + + case ColumnType.FLOAT: + row.putFloat(colIndex, b); + break; + + case ColumnType.DOUBLE: + row.putDouble(colIndex, b); + break; + + default: + throw CairoException.instance(0) + .put("cast error for line protocol boolean [columnIndex=").put(colIndex) + .put(", columnType=").put(ColumnType.nameOf(colType)) + .put(']'); } break; } diff --git a/core/src/test/java/io/questdb/cutlass/line/tcp/LineTcpInsertOtherTypesTest.java b/core/src/test/java/io/questdb/cutlass/line/tcp/LineTcpInsertOtherTypesTest.java index 28420d3b8..60759beee 100644 --- a/core/src/test/java/io/questdb/cutlass/line/tcp/LineTcpInsertOtherTypesTest.java +++ b/core/src/test/java/io/questdb/cutlass/line/tcp/LineTcpInsertOtherTypesTest.java @@ -186,7 +186,9 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest { "2147483647\t1970-01-01T00:00:07.000000Z\n" + "-2147483647\t1970-01-01T00:00:08.000000Z\n" + "NaN\t1970-01-01T00:00:11.000000Z\n" + - "NaN\t1970-01-01T00:00:19.000000Z\n", + "1\t1970-01-01T00:00:14.000000Z\n" + + "0\t1970-01-01T00:00:15.000000Z\n" + + "NaN\t1970-01-01T00:00:21.000000Z\n", new CharSequence[]{ "0i", // valid "100i", // valid @@ -198,9 +200,11 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest { "-2147483647i", // valid "0", // discarded bad type double "100", // discarded bad type double - "-2147483648i", // discarded out of range + "-2147483648i", // valid NaN same as null "-2147483648", // discarded out of range "null", // discarded bad type symbol + "true", // valid, true casts down to 1 + "false", // valid, true casts down to 0 "-0", // discarded bad type double "-100", // discarded bad type double "2147483647", // discarded bad type double @@ -223,7 +227,9 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest { "2147483647\t1970-01-01T00:00:07.000000Z\n" + "-2147483647\t1970-01-01T00:00:08.000000Z\n" + "-2147483648\t1970-01-01T00:00:11.000000Z\n" + - "NaN\t1970-01-01T00:00:19.000000Z\n", + "NaN\t1970-01-01T00:00:19.000000Z\n" + + "1\t1970-01-01T00:00:21.000000Z\n" + + "0\t1970-01-01T00:00:22.000000Z\n", new CharSequence[]{ "0i", // valid "100i", // valid @@ -245,6 +251,8 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest { "NaN", // discarded bad type symbol "", // valid null "0t", // discarded bad type timestamp + "true", // valid, true casts down to 1 + "false", // valid, true casts down to 0 }); } @@ -259,7 +267,9 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest { "32767\t1970-01-01T00:00:05.000000Z\n" + "-32767\t1970-01-01T00:00:06.000000Z\n" + "0\t1970-01-01T00:00:08.000000Z\n" + - "0\t1970-01-01T00:00:19.000000Z\n", + "0\t1970-01-01T00:00:19.000000Z\n" + + "1\t1970-01-01T00:00:21.000000Z\n" + + "0\t1970-01-01T00:00:22.000000Z\n", new CharSequence[]{ "0i", // valid "100i", // valid @@ -281,6 +291,8 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest { "NaN", // discarded bad type symbol "", // valid null "0t", // discarded bad type timestamp + "true", // valid, true casts down to 1 + "false", // valid, true casts down to 0 }); } @@ -295,7 +307,9 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest { "-32767\t1970-01-01T00:00:06.000000Z\n" + "-2147483648\t1970-01-01T00:00:08.000000Z\n" + "2147483648\t1970-01-01T00:00:09.000000Z\n" + - "NaN\t1970-01-01T00:00:15.000000Z\n", + "NaN\t1970-01-01T00:00:15.000000Z\n" + + "1\t1970-01-01T00:00:17.000000Z\n" + + "0\t1970-01-01T00:00:18.000000Z\n", new CharSequence[]{ "0i", // valid, taken as long, no way to make a short "100i", // valid @@ -313,6 +327,8 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest { "NaN", // discarded bad type symbol "", // valid null "0t", // discarded bad type timestamp + "true", // valid, true casts down to 1 + "false", // valid, true casts down to 0 }); } @@ -326,7 +342,9 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest { "-100\t1970-01-01T00:00:04.000000Z\n" + "127\t1970-01-01T00:00:05.000000Z\n" + "-128\t1970-01-01T00:00:06.000000Z\n" + - "0\t1970-01-01T00:00:14.000000Z\n", + "0\t1970-01-01T00:00:14.000000Z\n" + + "1\t1970-01-01T00:00:16.000000Z\n" + + "0\t1970-01-01T00:00:17.000000Z\n", new CharSequence[]{ "0i", // valid "100i", // valid @@ -343,6 +361,8 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest { "NaN", // discarded bad type symbol "", // valid null "0t", // discarded bad type timestamp + "true", // valid, true casts down to 1 + "false", // valid, true casts down to 0 }); } @@ -356,7 +376,9 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest { "127\t1970-01-01T00:00:05.000000Z\n" + "-2147483648\t1970-01-01T00:00:06.000000Z\n" + "-127\t1970-01-01T00:00:07.000000Z\n" + - "NaN\t1970-01-01T00:00:13.000000Z\n", + "NaN\t1970-01-01T00:00:13.000000Z\n" + + "1\t1970-01-01T00:00:15.000000Z\n" + + "0\t1970-01-01T00:00:16.000000Z\n", new CharSequence[]{ "0i", // valid, taken as long, no way to make a short "100i", // valid @@ -372,6 +394,8 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest { "NaN", // discarded bad type symbol "", // valid null "0t", // discarded bad type timestamp + "true", // valid, true casts down to 1 + "false", // valid, true casts down to 0 }); } @@ -641,7 +665,9 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest { "123.0\t1970-01-01T00:00:16.000000Z\n" + "-123.0\t1970-01-01T00:00:17.000000Z\n" + "NaN\t1970-01-01T00:00:18.000000Z\n" + - "NaN\t1970-01-01T00:00:19.000000Z\n", + "NaN\t1970-01-01T00:00:19.000000Z\n" + + "1.0\t1970-01-01T00:00:21.000000Z\n" + + "0.0\t1970-01-01T00:00:22.000000Z\n", new CharSequence[]{ "1.6x", // discarded bad type symbol "1.7976931348623157E308", // valid @@ -663,6 +689,8 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest { "NaN", // valid null "", // valid null "0t", // discarded bad type timestamp + "true", // valid, true casts down to 1.0 + "false", // valid, true casts down to 0.0 }); } @@ -685,7 +713,9 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest { "123.0\t1970-01-01T00:00:16.000000Z\n" + "-123.0\t1970-01-01T00:00:17.000000Z\n" + "NaN\t1970-01-01T00:00:18.000000Z\n" + - "NaN\t1970-01-01T00:00:19.000000Z\n", + "NaN\t1970-01-01T00:00:19.000000Z\n" + + "1.0\t1970-01-01T00:00:21.000000Z\n" + + "0.0\t1970-01-01T00:00:22.000000Z\n", new CharSequence[]{ "1.7976931348623156E308", // valid "0.425667788123", // valid @@ -707,6 +737,8 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest { "NaN", // valid null "", // valid null "0t", // discarded bad type timestamp + "true", // valid, true casts down to 1.0 + "false", // valid, true casts down to 0.0 }); } @@ -729,7 +761,9 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest { "-123.0000\t1970-01-01T00:00:14.000000Z\n" + "NaN\t1970-01-01T00:00:15.000000Z\n" + "NaN\t1970-01-01T00:00:16.000000Z\n" + - "NaN\t1970-01-01T00:00:17.000000Z\n", + "NaN\t1970-01-01T00:00:17.000000Z\n" + + "1.0000\t1970-01-01T00:00:20.000000Z\n" + + "0.0000\t1970-01-01T00:00:21.000000Z\n", new CharSequence[]{ "0.425667788123", // valid "3.14159265358979323846", // valid @@ -750,6 +784,8 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest { "NaN", // valid null "1.6x", // discarded bad type symbol "0t", // discarded bad type timestamp + "true", // valid, true casts down to 1.0 + "false", // valid, true casts down to 0.0 }); } @@ -770,7 +806,9 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest { "123.0\t1970-01-01T00:00:13.000000Z\n" + "-123.0\t1970-01-01T00:00:14.000000Z\n" + "NaN\t1970-01-01T00:00:15.000000Z\n" + - "NaN\t1970-01-01T00:00:16.000000Z\n", + "NaN\t1970-01-01T00:00:16.000000Z\n" + + "1.0\t1970-01-01T00:00:19.000000Z\n" + + "0.0\t1970-01-01T00:00:20.000000Z\n", new CharSequence[]{ "0.425667788123", // valid, but interpreted as double, cannot make float columns "3.14159265358979323846", // valid @@ -790,6 +828,8 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest { "", // valid null "1.6x", // discarded bad type symbol "0t", // discarded bad type timestamp + "true", // valid, true casts down to 1.0 + "false", // valid, true casts down to 0.0 }); } -- GitLab