未验证 提交 16fb2356 编写于 作者: A Andrey Pechkurov 提交者: GitHub

feat(ilp): support timestamp fields (#1358)

上级 57a80ef6
......@@ -162,6 +162,10 @@ public final class ColumnType {
return columnType == ColumnType.INT;
}
public static boolean isLong(int columnType) {
return columnType == ColumnType.LONG;
}
public static boolean isNull(int columnType) {
return columnType == NULL;
}
......
......@@ -186,6 +186,7 @@ public class CairoLineProtoParserSupport {
// and then it will be parsed accordingly by 'putValue'.
int valueLen = value.length();
if (valueLen > 0) {
char first = value.charAt(0);
char last = value.charAt(valueLen - 1); // see LineProtoSender.field methods
switch (last) {
case 'i':
......@@ -193,14 +194,18 @@ public class CairoLineProtoParserSupport {
return ColumnType.LONG256;
}
return valueLen == 1 ? ColumnType.SYMBOL : ColumnType.LONG;
case 'e':
case 'E':
// tru(e)
// fals(e)
case 't':
if (valueLen > 1 && ((first >= '0' && first <= '9') || first == '-')) {
return ColumnType.TIMESTAMP;
}
// fall through
case 'T':
// t
// T
case 'e':
case 'E':
// tru(e)
// fals(e)
case 'f':
case 'F':
// f
......@@ -217,7 +222,6 @@ public class CairoLineProtoParserSupport {
}
return ColumnType.STRING;
default:
char first = value.charAt(0);
if (last >= '0' && last <= '9' && ((first >= '0' && first <= '9') || first == '-' || first == '.')) {
return ColumnType.DOUBLE;
}
......
......@@ -603,6 +603,13 @@ class LineTcpMeasurementScheduler implements Closeable {
bufPos += Byte.BYTES;
break;
}
case NewLineProtoParser.ENTITY_TYPE_TIMESTAMP: {
Unsafe.getUnsafe().putByte(bufPos, entity.getType());
bufPos += Byte.BYTES;
Unsafe.getUnsafe().putLong(bufPos, entity.getTimestampValue());
bufPos += Long.BYTES;
break;
}
default:
// unsupported types are ignored
break;
......@@ -790,7 +797,15 @@ class LineTcpMeasurementScheduler implements Closeable {
case NewLineProtoParser.ENTITY_TYPE_BOOLEAN: {
byte b = Unsafe.getUnsafe().getByte(bufPos);
bufPos += Byte.BYTES;
row.putBool(colIndex, b == 1);
final int colType = writer.getMetadata().getColumnType(colIndex);
if (ColumnType.isBoolean(colType) || ColumnType.isLong(colType)) {
row.putBool(colIndex, b == 1);
} else {
throw CairoException.instance(0)
.put("cast error for line protocol boolean [columnIndex=").put(colIndex)
.put(", columnType=").put(ColumnType.nameOf(colType))
.put(']');
}
break;
}
......@@ -878,6 +893,21 @@ class LineTcpMeasurementScheduler implements Closeable {
break;
}
case NewLineProtoParser.ENTITY_TYPE_TIMESTAMP: {
long ts = Unsafe.getUnsafe().getLong(bufPos);
bufPos += Long.BYTES;
final int colType = writer.getMetadata().getColumnType(colIndex);
if (ColumnType.isTimestamp(colType)) {
row.putTimestamp(colIndex, ts);
} else {
throw CairoException.instance(0)
.put("cast error for line protocol timestamp [columnIndex=").put(colIndex)
.put(", columnType=").put(ColumnType.nameOf(colType))
.put(']');
}
break;
}
case NewLineProtoParser.ENTITY_TYPE_NULL: {
// ignored, default nulls is used
break;
......@@ -1555,5 +1585,6 @@ class LineTcpMeasurementScheduler implements Closeable {
DEFAULT_COLUMN_TYPES[NewLineProtoParser.ENTITY_TYPE_GEOSHORT] = ColumnType.getGeoHashTypeWithBits(16);
DEFAULT_COLUMN_TYPES[NewLineProtoParser.ENTITY_TYPE_GEOINT] = ColumnType.getGeoHashTypeWithBits(32);
DEFAULT_COLUMN_TYPES[NewLineProtoParser.ENTITY_TYPE_GEOLONG] = ColumnType.getGeoHashTypeWithBits(60);
DEFAULT_COLUMN_TYPES[NewLineProtoParser.ENTITY_TYPE_TIMESTAMP] = ColumnType.TIMESTAMP;
}
}
......@@ -48,7 +48,8 @@ public class NewLineProtoParser implements Closeable {
public static final byte ENTITY_TYPE_GEOSHORT = 10;
public static final byte ENTITY_TYPE_GEOINT = 11;
public static final byte ENTITY_TYPE_GEOLONG = 12;
public static final int N_ENTITY_TYPES = ENTITY_TYPE_GEOLONG + 1;
public static final byte ENTITY_TYPE_TIMESTAMP = 13;
public static final int N_ENTITY_TYPES = ENTITY_TYPE_TIMESTAMP + 1;
static final byte ENTITY_TYPE_NONE = (byte) 0xff; // visible for testing
private final DirectByteCharSequence measurementName = new DirectByteCharSequence();
private final DirectByteCharSequence charSeq = new DirectByteCharSequence();
......@@ -386,6 +387,7 @@ public class NewLineProtoParser implements Closeable {
private long integerValue;
private boolean booleanValue;
private double floatValue;
private long timestampValue;
public boolean getBooleanValue() {
return booleanValue;
......@@ -399,6 +401,10 @@ public class NewLineProtoParser implements Closeable {
return integerValue;
}
public long getTimestampValue() {
return timestampValue;
}
public DirectByteCharSequence getName() {
return name;
}
......@@ -441,18 +447,30 @@ public class NewLineProtoParser implements Closeable {
}
type = ENTITY_TYPE_SYMBOL;
return true;
case 'e':
case 'E':
// tru(e)
// fals(e)
case 't':
if (valueLen > 1) {
try {
charSeq.of(value.getLo(), value.getHi() - 1);
timestampValue = Numbers.parseLong(charSeq);
value.decHi(); // remove 't'
type = ENTITY_TYPE_TIMESTAMP;
} catch (NumericException notANumber) {
type = ENTITY_TYPE_SYMBOL;
}
return true;
}
// fall through
case 'T':
case 'f':
case 'F':
// f
// F
case 'e':
case 'E':
// t
// T
// f
// F
// tru(e)
// fals(e)
if (valueLen == 1) {
if (last != 'e') {
booleanValue = (last | 32) == 't';
......
......@@ -385,6 +385,9 @@ public class CairoLineProtoParser implements LineProtoParser, Closeable {
case ColumnType.LONG256:
valid = columnTypeTag == ColumnType.LONG256;
break;
case ColumnType.TIMESTAMP:
valid = columnTypeTag == ColumnType.TIMESTAMP;
break;
default:
valid = false;
}
......
......@@ -233,6 +233,7 @@ public class CairoLineProtoParserSupportTest extends LineUdpInsertTest {
Assert.assertEquals(ColumnType.DOUBLE, CairoLineProtoParserSupport.getValueType("1e-13"));
Assert.assertEquals(ColumnType.DOUBLE, CairoLineProtoParserSupport.getValueType("1.0"));
Assert.assertEquals(ColumnType.DOUBLE, CairoLineProtoParserSupport.getValueType("1"));
Assert.assertEquals(ColumnType.TIMESTAMP, CairoLineProtoParserSupport.getValueType("123t"));
Assert.assertEquals(ColumnType.UNDEFINED, CairoLineProtoParserSupport.getValueType("aaa\""));
Assert.assertEquals(ColumnType.UNDEFINED, CairoLineProtoParserSupport.getValueType("\"aaa"));
......
......@@ -35,30 +35,36 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest {
@Test
public void testInsertTimestampTableExists() throws Exception {
// no literal representation for timestamp, only longs can be inserted
assertType(ColumnType.TIMESTAMP,
"value\ttimestamp\n" +
"1970-01-19T21:02:13.921000Z\t1970-01-01T00:00:01.000000Z\n" +
"1970-01-01T00:00:00.000000Z\t1970-01-01T00:00:07.000000Z\n" +
"\t1970-01-01T00:00:08.000000Z\n" +
"\t1970-01-01T00:00:09.000000Z\n" +
"1970-01-01T00:00:00.000000Z\t1970-01-01T00:00:10.000000Z\n" +
"294247-01-10T04:00:54.775807Z\t1970-01-01T00:00:11.000000Z\n",
"1970-01-19T21:02:13.921000Z\t1970-01-01T00:00:02.000000Z\n" +
"1970-01-01T00:00:00.000000Z\t1970-01-01T00:00:08.000000Z\n" +
"1970-01-01T00:00:00.000000Z\t1970-01-01T00:00:09.000000Z\n" +
"\t1970-01-01T00:00:10.000000Z\n" +
"\t1970-01-01T00:00:11.000000Z\n" +
"1970-01-01T00:00:00.000000Z\t1970-01-01T00:00:12.000000Z\n" +
"1970-01-01T00:00:00.000000Z\t1970-01-01T00:00:13.000000Z\n" +
"294247-01-10T04:00:54.775807Z\t1970-01-01T00:00:14.000000Z\n",
new CharSequence[]{
"1630933921000i", // valid
"1630933921000t", // valid
"1630933921000", // discarded bad type double
"\"1970-01-01T00:00:05.000000Z\"", // discarded bad type string
"1970-01-01T00:\"00:05.00\"0000Z", // discarded bad type symbol
"\"1970-01-01T00:00:05.000000Z", // discarded bad string value
"1970-01-01T00:00:05.000000Z\"", // discarded bad string value
"0i", // valid
"0t", // valid
"-9223372036854775808i", // valid NaN, same as null
"", // valid null
"-0i", // valid
"-0t", // valid
"9223372036854775807i", // valid
"NaN", // discarded bad type symbol
"null", // discarded bad type symbol
"1970-01-01T00:00:05.000000Z" // discarded bad type symbol
"1970-01-01T00:00:05.000000Z", // discarded bad type symbol
"t", // discarded bad type boolean
});
}
......@@ -87,7 +93,8 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest {
"9223372036854775807i", // valid
"NaN", // discarded bad type symbol
"null", // discarded bad type symbol
"1970-01-01T00:00:05.000000Z" // discarded bad type symbol
"1970-01-01T00:00:05.000000Z", // discarded bad type symbol
"0t", // discarded bad type timestamp
});
}
......@@ -125,6 +132,7 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest {
"-100", // discarded bad type double
"null", // discarded bad type symbol
"", // valid null
"0t", // discarded bad type timestamp
});
}
......@@ -162,6 +170,7 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest {
"-100", // discarded bad type double
"null", // discarded bad type symbol
"", // valid null
"0t", // discarded bad type timestamp
});
}
......@@ -197,7 +206,8 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest {
"2147483647", // discarded bad type double
"-2147483647", // discarded bad type double
"NaN", // discarded bad type symbol
"" // valid null
"", // valid null
"0t", // discarded bad type timestamp
});
}
......@@ -233,7 +243,8 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest {
"2147483647", // discarded bad type double
"-2147483647", // discarded bad type double
"NaN", // discarded bad type symbol
"" // valid null
"", // valid null
"0t", // discarded bad type timestamp
});
}
......@@ -268,7 +279,8 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest {
"100", // discarded bad type double
"-0", // discarded bad type double
"NaN", // discarded bad type symbol
"" // valid null
"", // valid null
"0t", // discarded bad type timestamp
});
}
......@@ -299,7 +311,8 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest {
"100", // discarded bad type double
"-0", // discarded bad type double
"NaN", // discarded bad type symbol
"" // valid null
"", // valid null
"0t", // discarded bad type timestamp
});
}
......@@ -328,7 +341,8 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest {
"100", // discarded bad type double
"-0", // discarded bad type double
"NaN", // discarded bad type symbol
"" // valid null
"", // valid null
"0t", // discarded bad type timestamp
});
}
......@@ -356,7 +370,8 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest {
"100", // discarded bad type double
"-0", // discarded bad type double
"NaN", // discarded bad type symbol
"" // valid null
"", // valid null
"0t", // discarded bad type timestamp
});
}
......@@ -378,6 +393,7 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest {
"null", // discarded bad type symbol
"\"N\"", // valid
"0", // discarded bad type double
"0t", // discarded bad type timestamp
"1970-01-01T00:00:05.000000Z" // discarded bad type symbol
});
}
......@@ -397,7 +413,8 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest {
"0x1234i", // actual long256
"0x1234", // discarded bad type double
"0x00", // discarded bad type double
"" // valid null
"", // valid null
"0t", // discarded bad type timestamp
});
}
......@@ -416,6 +433,7 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest {
"\"null\"", // discarded bad type string
"120i", // discarded bad type long
"0x1234", // discarded bad type double
"0t", // discarded bad type timestamp
});
}
......@@ -448,6 +466,7 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest {
"F", // valid
"", // valid null, equals false
"e", // valid
"0t", // discarded bad type timestamp
});
}
......@@ -479,6 +498,7 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest {
"F", // valid
"", // valid null, equals false
"e", // discarded bad type symbol
"0t", // discarded bad type timestamp
});
}
......@@ -640,7 +660,8 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest {
"123", // valid
"-123", // valid
"NaN", // valid null
"" // valid null
"", // valid null
"0t", // discarded bad type timestamp
});
}
......@@ -683,7 +704,8 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest {
"123", // valid
"-123", // valid
"NaN", // valid null
"" // valid null
"", // valid null
"0t", // discarded bad type timestamp
});
}
......@@ -725,7 +747,8 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest {
"NaN", // valid null
"", // valid null
"NaN", // valid null
"1.6x" // discarded bad type symbol
"1.6x", // discarded bad type symbol
"0t", // discarded bad type timestamp
});
}
......@@ -764,7 +787,8 @@ public class LineTcpInsertOtherTypesTest extends BaseLineTcpContextTest {
"-123", // valid
"NaN", // valid null
"", // valid null
"1.6x" // discarded bad type symbol
"1.6x", // discarded bad type symbol
"0t", // discarded bad type timestamp
});
}
......
......@@ -674,7 +674,7 @@ public class LineTcpServerTest extends AbstractCairoTest {
}
@Test
public void testWrterCommitFails() throws Exception {
public void testWriterCommitFails() throws Exception {
try (TableModel m = new TableModel(configuration, "table_a", PartitionBy.DAY)) {
m.timestamp("ReceiveTime")
.col("SequenceNumber", ColumnType.SYMBOL).indexed(true, 256)
......
......@@ -35,30 +35,36 @@ public class LineUdpInsertOtherTypesTest extends LineUdpInsertTest {
@Test
public void testInsertTimestampTableExists() throws Exception {
// no literal representation for timestamp, only longs can be inserted
assertType(ColumnType.TIMESTAMP,
"value\ttimestamp\n" +
"1970-01-19T21:02:13.921000Z\t1970-01-01T00:00:01.000000Z\n" +
"1970-01-01T00:00:00.000000Z\t1970-01-01T00:00:07.000000Z\n" +
"\t1970-01-01T00:00:08.000000Z\n" +
"\t1970-01-01T00:00:09.000000Z\n" +
"1970-01-01T00:00:00.000000Z\t1970-01-01T00:00:10.000000Z\n" +
"294247-01-10T04:00:54.775807Z\t1970-01-01T00:00:11.000000Z\n",
"1970-01-19T21:02:13.921000Z\t1970-01-01T00:00:02.000000Z\n" +
"1970-01-01T00:00:00.000000Z\t1970-01-01T00:00:08.000000Z\n" +
"1970-01-01T00:00:00.000000Z\t1970-01-01T00:00:09.000000Z\n" +
"\t1970-01-01T00:00:10.000000Z\n" +
"\t1970-01-01T00:00:11.000000Z\n" +
"1970-01-01T00:00:00.000000Z\t1970-01-01T00:00:12.000000Z\n" +
"1970-01-01T00:00:00.000000Z\t1970-01-01T00:00:13.000000Z\n" +
"294247-01-10T04:00:54.775807Z\t1970-01-01T00:00:14.000000Z\n",
new String[]{
"1630933921000i", // valid
"1630933921000t", // valid
"1630933921000", // discarded bad type double
"\"1970-01-01T00:00:05.000000Z\"", // discarded bad type string
"1970-01-01T00:\"00:05.00\"0000Z", // discarded bad type symbol
"\"1970-01-01T00:00:05.000000Z", // discarded bad string value
"1970-01-01T00:00:05.000000Z\"", // discarded bad string value
"0i", // valid
"0t", // valid
"-9223372036854775808i", // valid NaN, same as null
"", // valid null
"-0i", // valid
"-0t", // valid
"9223372036854775807i", // valid
"NaN", // discarded bad type symbol
"null", // discarded bad type symbol
"1970-01-01T00:00:05.000000Z" // discarded bad type symbol
"1970-01-01T00:00:05.000000Z", // discarded bad type symbol
"t", // discarded bad type boolean
});
}
......@@ -87,7 +93,8 @@ public class LineUdpInsertOtherTypesTest extends LineUdpInsertTest {
"9223372036854775807i", // valid
"NaN", // discarded bad type symbol
"null", // discarded bad type symbol
"1970-01-01T00:00:05.000000Z" // discarded bad type symbol
"1970-01-01T00:00:05.000000Z", // discarded bad type symbol
"0t", // discarded bad type timestamp
});
}
......@@ -123,6 +130,7 @@ public class LineUdpInsertOtherTypesTest extends LineUdpInsertTest {
"-100", // discarded bad type double
"null", // discarded bad type symbol
"", // valid null
"0t", // discarded bad type timestamp
});
}
......@@ -158,6 +166,7 @@ public class LineUdpInsertOtherTypesTest extends LineUdpInsertTest {
"-100", // discarded bad type double
"null", // discarded bad type symbol
"", // valid null
"0t", // discarded bad type timestamp
});
}
......@@ -194,7 +203,8 @@ public class LineUdpInsertOtherTypesTest extends LineUdpInsertTest {
"2147483647", // discarded bad type double
"-2147483647", // discarded bad type double
"NaN", // discarded bad type symbol
"" // valid null
"", // valid null
"0t", // discarded bad type timestamp
});
}
......@@ -230,7 +240,8 @@ public class LineUdpInsertOtherTypesTest extends LineUdpInsertTest {
"2147483647", // discarded bad type double
"-2147483647", // discarded bad type double
"NaN", // discarded bad type symbol
"" // valid null
"", // valid null
"0t", // discarded bad type timestamp
});
}
......@@ -270,7 +281,8 @@ public class LineUdpInsertOtherTypesTest extends LineUdpInsertTest {
"100", // discarded bad type double
"-0", // discarded bad type double
"NaN", // discarded bad type symbol
"" // valid null
"", // valid null
"0t", // discarded bad type timestamp
});
}
......@@ -301,7 +313,8 @@ public class LineUdpInsertOtherTypesTest extends LineUdpInsertTest {
"100", // discarded bad type double
"-0", // discarded bad type double
"NaN", // discarded bad type symbol
"" // valid null
"", // valid null
"0t", // discarded bad type timestamp
});
}
......@@ -330,7 +343,8 @@ public class LineUdpInsertOtherTypesTest extends LineUdpInsertTest {
"100", // discarded bad type double
"-0", // discarded bad type double
"NaN", // discarded bad type symbol
"" // valid null
"", // valid null
"0t", // discarded bad type timestamp
});
}
......@@ -358,7 +372,8 @@ public class LineUdpInsertOtherTypesTest extends LineUdpInsertTest {
"100", // discarded bad type double
"-0", // discarded bad type double
"NaN", // discarded bad type symbol
"" // valid null
"", // valid null
"0t", // discarded bad type timestamp
});
}
......@@ -399,7 +414,8 @@ public class LineUdpInsertOtherTypesTest extends LineUdpInsertTest {
"0x1234i", // actual long256
"0x1234", // discarded bad type double
"0x00", // discarded bad type double
"" // valid null
"", // valid null
"0t", // discarded bad type timestamp
});
}
......@@ -418,6 +434,7 @@ public class LineUdpInsertOtherTypesTest extends LineUdpInsertTest {
"\"null\"", // discarded bad type string
"120i", // discarded bad type long
"0x1234", // discarded bad type double
"0t", // discarded bad type timestamp
});
}
......@@ -450,6 +467,7 @@ public class LineUdpInsertOtherTypesTest extends LineUdpInsertTest {
"F", // valid
"", // valid null, equals false
"e", // valid
"0t", // discarded bad type timestamp
});
}
......@@ -481,6 +499,7 @@ public class LineUdpInsertOtherTypesTest extends LineUdpInsertTest {
"F", // valid
"", // valid null, equals false
"e", // discarded bad type symbol
"0t", // discarded bad type timestamp
});
}
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册