未验证 提交 d4b9e9c7 编写于 作者: V Vlad Ilyushchenko 提交者: GitHub

fix(core): fixed incorrect _txn content to be occasionally written to the file (#1805)

上级 060ea1db
......@@ -359,10 +359,12 @@ public final class TxWriter extends TxReader implements Closeable, Mutable, Symb
}
private void saveAttachedPartitionsToTx(int symbolColumnCount) {
final int size = attachedPartitions.size();
final long partitionTableOffset = getPartitionTableSizeOffset(symbolColumnCount);
txMem.putInt(partitionTableOffset, size * Long.BYTES);
// change partition count only when we have something to save to the
// partition table
if (maxTimestamp != Long.MIN_VALUE) {
final int size = attachedPartitions.size();
final long partitionTableOffset = getPartitionTableSizeOffset(symbolColumnCount);
txMem.putInt(partitionTableOffset, size * Long.BYTES);
for (int i = attachedPositionDirtyIndex; i < size; i++) {
txMem.putLong(getPartitionTableIndexOffset(partitionTableOffset, i), attachedPartitions.getQuick(i));
}
......
......@@ -110,7 +110,131 @@ public class TableReaderTest extends AbstractCairoTest {
Assert.assertNull(r.getSym(7));
}
};
private static final RecordAssert BATCH2_ASSERTER = (r, rnd, ts, blob) -> {
BATCH1_ASSERTER.assertRecord(r, rnd, ts, blob);
if ((rnd.nextPositiveInt() & 3) == 0) {
assertStrColumn(rnd.nextChars(15), r, 11);
}
};
private static final RecordAssert BATCH3_ASSERTER = (r, rnd, ts, blob) -> {
BATCH2_ASSERTER.assertRecord(r, rnd, ts, blob);
if ((rnd.nextPositiveInt() & 3) == 0) {
Assert.assertEquals(rnd.nextInt(), r.getInt(12));
}
};
private static final RecordAssert BATCH4_ASSERTER = (r, rnd, ts, blob) -> {
BATCH3_ASSERTER.assertRecord(r, rnd, ts, blob);
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextShort(), r.getShort(13));
} else {
Assert.assertEquals(0, r.getShort(13));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextBoolean(), r.getBool(14));
} else {
Assert.assertFalse(r.getBool(14));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextByte(), r.getByte(15));
} else {
Assert.assertEquals(0, r.getByte(15));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextFloat(), r.getFloat(16), 0.00000001f);
} else {
Assert.assertTrue(Float.isNaN(r.getFloat(16)));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextDouble(), r.getDouble(17), 0.0000001d);
} else {
Assert.assertTrue(Double.isNaN(r.getDouble(17)));
}
if (rnd.nextBoolean()) {
TestUtils.assertEquals(rnd.nextChars(10), r.getSym(18));
} else {
Assert.assertNull(r.getSym(18));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextLong(), r.getLong(19));
} else {
Assert.assertEquals(Numbers.LONG_NaN, r.getLong(19));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextLong(), r.getDate(20));
} else {
Assert.assertEquals(Numbers.LONG_NaN, r.getDate(20));
}
assertBin(r, rnd, blob, 21);
};
private static final RecordAssert BATCH6_ASSERTER = (r, rnd, ts, blob) -> {
BATCH3_ASSERTER.assertRecord(r, rnd, ts, blob);
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextShort(), r.getShort(13));
} else {
Assert.assertEquals(0, r.getShort(13));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextBoolean(), r.getBool(14));
} else {
Assert.assertFalse(r.getBool(14));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextByte(), r.getByte(15));
} else {
Assert.assertEquals(0, r.getByte(15));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextFloat(), r.getFloat(16), 0.00000001f);
} else {
Assert.assertTrue(Float.isNaN(r.getFloat(16)));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextDouble(), r.getDouble(17), 0.0000001d);
} else {
Assert.assertTrue(Double.isNaN(r.getDouble(17)));
}
if (rnd.nextBoolean()) {
TestUtils.assertEquals(rnd.nextChars(10), r.getSym(18));
} else {
Assert.assertNull(r.getSym(18));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextLong(), r.getLong(19));
} else {
Assert.assertEquals(Numbers.LONG_NaN, r.getLong(19));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextLong(), r.getDate(20));
} else {
Assert.assertEquals(Numbers.LONG_NaN, r.getDate(20));
}
};
private static final RecordAssert BATCH5_ASSERTER = (r, rnd, ts, blob) -> {
BATCH6_ASSERTER.assertRecord(r, rnd, ts, blob);
// generate blob to roll forward random generator, don't assert blob value
if (rnd.nextBoolean()) {
rnd.nextChars(blob, blobLen / 2);
}
};
private static final RecordAssert BATCH1_ASSERTER_NULL_BIN = (r, exp, ts, blob) -> {
// same as BATCH1_ASSERTER + special treatment of "bin" column
......@@ -372,117 +496,12 @@ public class TableReaderTest extends AbstractCairoTest {
Assert.assertEquals(Numbers.INT_NaN, r.getInt(20));
};
private static final RecordAssert BATCH1_9_ASSERTER = (r, exp, ts, blob) -> {
if (exp.nextBoolean()) {
Assert.assertEquals(exp.nextByte(), r.getByte(1));
} else {
Assert.assertEquals(0, r.getByte(1));
}
if (exp.nextBoolean()) {
Assert.assertEquals(exp.nextBoolean(), r.getBool(6));
} else {
Assert.assertFalse(r.getBool(6));
}
if (exp.nextBoolean()) {
Assert.assertEquals(exp.nextShort(), r.getShort(0));
} else {
Assert.assertEquals(0, r.getShort(0));
}
if (exp.nextBoolean()) {
exp.nextInt();
}
if (exp.nextBoolean()) {
Assert.assertEquals(exp.nextDouble(), r.getDouble(2), 0.00000001);
} else {
Assert.assertTrue(Double.isNaN(r.getDouble(2)));
}
if (exp.nextBoolean()) {
Assert.assertEquals(exp.nextFloat(), r.getFloat(3), 0.000001f);
} else {
Assert.assertTrue(Float.isNaN(r.getFloat(3)));
}
if (exp.nextBoolean()) {
Assert.assertEquals(exp.nextLong(), r.getLong(4));
} else {
Assert.assertEquals(Numbers.LONG_NaN, r.getLong(4));
}
if (exp.nextBoolean()) {
Assert.assertEquals(ts, r.getDate(8));
} else {
Assert.assertEquals(Numbers.LONG_NaN, r.getDate(8));
}
assertBin(r, exp, blob, 7);
if (exp.nextBoolean()) {
assertStrColumn(exp.nextChars(10), r, 5);
} else {
assertNullStr(r, 5);
}
// exercise random generator for column we removed
if (exp.nextBoolean()) {
exp.nextChars(7);
}
Assert.assertEquals(Numbers.INT_NaN, r.getInt(19));
};
private static final RecordAssert BATCH_2_7_BEFORE_ASSERTER = (r, rnd, ts, blob) -> assertNullStr(r, 10);
private static final RecordAssert BATCH_2_9_BEFORE_ASSERTER = (r, rnd, ts, blob) -> assertNullStr(r, 9);
private static final RecordAssert BATCH_3_7_BEFORE_ASSERTER = (r, rnd, ts, blob) -> Assert.assertEquals(Numbers.INT_NaN, r.getInt(11));
private static final RecordAssert BATCH_3_9_BEFORE_ASSERTER = (r, rnd, ts, blob) -> Assert.assertEquals(Numbers.INT_NaN, r.getInt(10));
private static final RecordAssert BATCH_4_7_BEFORE_ASSERTER = (r, rnd, ts, blob) -> {
Assert.assertEquals(0, r.getShort(12));
Assert.assertFalse(r.getBool(13));
Assert.assertEquals(0, r.getByte(14));
Assert.assertTrue(Float.isNaN(r.getFloat(15)));
Assert.assertTrue(Double.isNaN(r.getDouble(16)));
Assert.assertNull(r.getSym(17));
Assert.assertEquals(Numbers.LONG_NaN, r.getLong(18));
Assert.assertEquals(Numbers.LONG_NaN, r.getDate(19));
};
private static final RecordAssert BATCH_4_9_BEFORE_ASSERTER = (r, rnd, ts, blob) -> {
Assert.assertEquals(0, r.getShort(11));
Assert.assertFalse(r.getBool(12));
Assert.assertEquals(0, r.getByte(13));
Assert.assertTrue(Float.isNaN(r.getFloat(14)));
Assert.assertTrue(Double.isNaN(r.getDouble(15)));
Assert.assertNull(r.getSym(16));
Assert.assertEquals(Numbers.LONG_NaN, r.getLong(17));
Assert.assertEquals(Numbers.LONG_NaN, r.getDate(18));
};
private static final RecordAssert BATCH2_ASSERTER = (r, rnd, ts, blob) -> {
BATCH1_ASSERTER.assertRecord(r, rnd, ts, blob);
if ((rnd.nextPositiveInt() & 3) == 0) {
assertStrColumn(rnd.nextChars(15), r, 11);
}
};
private static final RecordAssert BATCH2_7_ASSERTER = (r, rnd, ts, blob) -> {
BATCH1_7_ASSERTER.assertRecord(r, rnd, ts, blob);
if ((rnd.nextPositiveInt() & 3) == 0) {
assertStrColumn(rnd.nextChars(15), r, 10);
}
};
private static final RecordAssert BATCH2_9_ASSERTER = (r, rnd, ts, blob) -> {
BATCH1_9_ASSERTER.assertRecord(r, rnd, ts, blob);
if ((rnd.nextPositiveInt() & 3) == 0) {
assertStrColumn(rnd.nextChars(15), r, 9);
}
};
private static final RecordAssert BATCH3_BEFORE_ASSERTER = (r, rnd, ts, blob) -> Assert.assertEquals(Numbers.INT_NaN, r.getInt(12));
private static final RecordAssert BATCH3_ASSERTER = (r, rnd, ts, blob) -> {
BATCH2_ASSERTER.assertRecord(r, rnd, ts, blob);
if ((rnd.nextPositiveInt() & 3) == 0) {
Assert.assertEquals(rnd.nextInt(), r.getInt(12));
}
};
private static final RecordAssert BATCH3_7_ASSERTER = (r, rnd, ts, blob) -> {
BATCH2_7_ASSERTER.assertRecord(r, rnd, ts, blob);
......@@ -490,188 +509,137 @@ public class TableReaderTest extends AbstractCairoTest {
Assert.assertEquals(rnd.nextInt(), r.getInt(11));
}
};
private static final RecordAssert BATCH3_9_ASSERTER = (r, rnd, ts, blob) -> {
BATCH2_9_ASSERTER.assertRecord(r, rnd, ts, blob);
if ((rnd.nextPositiveInt() & 3) == 0) {
Assert.assertEquals(rnd.nextInt(), r.getInt(10));
}
};
private static final RecordAssert BATCH4_BEFORE_ASSERTER = (r, rnd, ts, blob) -> {
Assert.assertEquals(0, r.getShort(13));
Assert.assertFalse(r.getBool(14));
Assert.assertEquals(0, r.getByte(15));
Assert.assertTrue(Float.isNaN(r.getFloat(16)));
Assert.assertTrue(Double.isNaN(r.getDouble(17)));
Assert.assertNull(r.getSym(18));
Assert.assertEquals(Numbers.LONG_NaN, r.getLong(19));
Assert.assertEquals(Numbers.LONG_NaN, r.getDate(20));
Assert.assertNull(r.getBin(21));
Assert.assertEquals(TableUtils.NULL_LEN, r.getBinLen(21));
};
private static final RecordAssert BATCH5_BEFORE_ASSERTER = (r, rnd, ts, blob) -> {
Assert.assertEquals(0, r.getShort(13));
Assert.assertFalse(r.getBool(14));
Assert.assertEquals(0, r.getByte(15));
Assert.assertTrue(Float.isNaN(r.getFloat(16)));
Assert.assertTrue(Double.isNaN(r.getDouble(17)));
Assert.assertNull(r.getSym(18));
Assert.assertEquals(Numbers.LONG_NaN, r.getLong(19));
Assert.assertEquals(Numbers.LONG_NaN, r.getDate(20));
};
private static final RecordAssert BATCH4_ASSERTER = (r, rnd, ts, blob) -> {
BATCH3_ASSERTER.assertRecord(r, rnd, ts, blob);
private static final RecordAssert BATCH6_7_ASSERTER = (r, rnd, ts, blob) -> {
BATCH3_7_ASSERTER.assertRecord(r, rnd, ts, blob);
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextShort(), r.getShort(13));
Assert.assertEquals(rnd.nextShort(), r.getShort(12));
} else {
Assert.assertEquals(0, r.getShort(13));
Assert.assertEquals(0, r.getShort(12));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextBoolean(), r.getBool(14));
Assert.assertEquals(rnd.nextBoolean(), r.getBool(13));
} else {
Assert.assertFalse(r.getBool(14));
Assert.assertFalse(r.getBool(13));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextByte(), r.getByte(15));
Assert.assertEquals(rnd.nextByte(), r.getByte(14));
} else {
Assert.assertEquals(0, r.getByte(15));
Assert.assertEquals(0, r.getByte(14));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextFloat(), r.getFloat(16), 0.00000001f);
Assert.assertEquals(rnd.nextFloat(), r.getFloat(15), 0.00000001f);
} else {
Assert.assertTrue(Float.isNaN(r.getFloat(16)));
Assert.assertTrue(Float.isNaN(r.getFloat(15)));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextDouble(), r.getDouble(17), 0.0000001d);
Assert.assertEquals(rnd.nextDouble(), r.getDouble(16), 0.0000001d);
} else {
Assert.assertTrue(Double.isNaN(r.getDouble(17)));
Assert.assertTrue(Double.isNaN(r.getDouble(16)));
}
if (rnd.nextBoolean()) {
TestUtils.assertEquals(rnd.nextChars(10), r.getSym(18));
TestUtils.assertEquals(rnd.nextChars(10), r.getSym(17));
} else {
Assert.assertNull(r.getSym(18));
Assert.assertNull(r.getSym(17));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextLong(), r.getLong(19));
Assert.assertEquals(rnd.nextLong(), r.getLong(18));
} else {
Assert.assertEquals(Numbers.LONG_NaN, r.getLong(19));
Assert.assertEquals(Numbers.LONG_NaN, r.getLong(18));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextLong(), r.getDate(20));
Assert.assertEquals(rnd.nextLong(), r.getDate(19));
} else {
Assert.assertEquals(Numbers.LONG_NaN, r.getDate(20));
Assert.assertEquals(Numbers.LONG_NaN, r.getDate(19));
}
assertBin(r, rnd, blob, 21);
};
private static final RecordAssert BATCH6_ASSERTER = (r, rnd, ts, blob) -> {
BATCH3_ASSERTER.assertRecord(r, rnd, ts, blob);
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextShort(), r.getShort(13));
} else {
Assert.assertEquals(0, r.getShort(13));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextBoolean(), r.getBool(14));
} else {
Assert.assertFalse(r.getBool(14));
}
private static final RecordAssert BATCH5_7_ASSERTER = (r, rnd, ts, blob) -> {
BATCH6_7_ASSERTER.assertRecord(r, rnd, ts, blob);
// generate blob to roll forward random generator, don't assert blob value
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextByte(), r.getByte(15));
} else {
Assert.assertEquals(0, r.getByte(15));
rnd.nextChars(blob, blobLen / 2);
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextFloat(), r.getFloat(16), 0.00000001f);
};
private static final RecordAssert BATCH1_9_ASSERTER = (r, exp, ts, blob) -> {
if (exp.nextBoolean()) {
Assert.assertEquals(exp.nextByte(), r.getByte(1));
} else {
Assert.assertTrue(Float.isNaN(r.getFloat(16)));
Assert.assertEquals(0, r.getByte(1));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextDouble(), r.getDouble(17), 0.0000001d);
if (exp.nextBoolean()) {
Assert.assertEquals(exp.nextBoolean(), r.getBool(6));
} else {
Assert.assertTrue(Double.isNaN(r.getDouble(17)));
Assert.assertFalse(r.getBool(6));
}
if (rnd.nextBoolean()) {
TestUtils.assertEquals(rnd.nextChars(10), r.getSym(18));
if (exp.nextBoolean()) {
Assert.assertEquals(exp.nextShort(), r.getShort(0));
} else {
Assert.assertNull(r.getSym(18));
Assert.assertEquals(0, r.getShort(0));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextLong(), r.getLong(19));
} else {
Assert.assertEquals(Numbers.LONG_NaN, r.getLong(19));
if (exp.nextBoolean()) {
exp.nextInt();
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextLong(), r.getDate(20));
if (exp.nextBoolean()) {
Assert.assertEquals(exp.nextDouble(), r.getDouble(2), 0.00000001);
} else {
Assert.assertEquals(Numbers.LONG_NaN, r.getDate(20));
Assert.assertTrue(Double.isNaN(r.getDouble(2)));
}
};
private static final RecordAssert BATCH6_7_ASSERTER = (r, rnd, ts, blob) -> {
BATCH3_7_ASSERTER.assertRecord(r, rnd, ts, blob);
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextShort(), r.getShort(12));
if (exp.nextBoolean()) {
Assert.assertEquals(exp.nextFloat(), r.getFloat(3), 0.000001f);
} else {
Assert.assertEquals(0, r.getShort(12));
Assert.assertTrue(Float.isNaN(r.getFloat(3)));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextBoolean(), r.getBool(13));
if (exp.nextBoolean()) {
Assert.assertEquals(exp.nextLong(), r.getLong(4));
} else {
Assert.assertFalse(r.getBool(13));
Assert.assertEquals(Numbers.LONG_NaN, r.getLong(4));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextByte(), r.getByte(14));
if (exp.nextBoolean()) {
Assert.assertEquals(ts, r.getDate(8));
} else {
Assert.assertEquals(0, r.getByte(14));
Assert.assertEquals(Numbers.LONG_NaN, r.getDate(8));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextFloat(), r.getFloat(15), 0.00000001f);
} else {
Assert.assertTrue(Float.isNaN(r.getFloat(15)));
}
assertBin(r, exp, blob, 7);
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextDouble(), r.getDouble(16), 0.0000001d);
if (exp.nextBoolean()) {
assertStrColumn(exp.nextChars(10), r, 5);
} else {
Assert.assertTrue(Double.isNaN(r.getDouble(16)));
assertNullStr(r, 5);
}
if (rnd.nextBoolean()) {
TestUtils.assertEquals(rnd.nextChars(10), r.getSym(17));
} else {
Assert.assertNull(r.getSym(17));
// exercise random generator for column we removed
if (exp.nextBoolean()) {
exp.nextChars(7);
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextLong(), r.getLong(18));
} else {
Assert.assertEquals(Numbers.LONG_NaN, r.getLong(18));
Assert.assertEquals(Numbers.INT_NaN, r.getInt(19));
};
private static final RecordAssert BATCH2_9_ASSERTER = (r, rnd, ts, blob) -> {
BATCH1_9_ASSERTER.assertRecord(r, rnd, ts, blob);
if ((rnd.nextPositiveInt() & 3) == 0) {
assertStrColumn(rnd.nextChars(15), r, 9);
}
};
private static final RecordAssert BATCH3_9_ASSERTER = (r, rnd, ts, blob) -> {
BATCH2_9_ASSERTER.assertRecord(r, rnd, ts, blob);
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextLong(), r.getDate(19));
} else {
Assert.assertEquals(Numbers.LONG_NaN, r.getDate(19));
if ((rnd.nextPositiveInt() & 3) == 0) {
Assert.assertEquals(rnd.nextInt(), r.getInt(10));
}
};
private static final RecordAssert BATCH6_9_ASSERTER = (r, rnd, ts, blob) -> {
......@@ -725,14 +693,6 @@ public class TableReaderTest extends AbstractCairoTest {
Assert.assertEquals(Numbers.LONG_NaN, r.getDate(18));
}
};
private static final RecordAssert BATCH5_7_ASSERTER = (r, rnd, ts, blob) -> {
BATCH6_7_ASSERTER.assertRecord(r, rnd, ts, blob);
// generate blob to roll forward random generator, don't assert blob value
if (rnd.nextBoolean()) {
rnd.nextChars(blob, blobLen / 2);
}
};
private static final RecordAssert BATCH5_9_ASSERTER = (r, rnd, ts, blob) -> {
BATCH6_9_ASSERTER.assertRecord(r, rnd, ts, blob);
......@@ -741,13 +701,52 @@ public class TableReaderTest extends AbstractCairoTest {
rnd.nextChars(blob, blobLen / 2);
}
};
private static final RecordAssert BATCH5_ASSERTER = (r, rnd, ts, blob) -> {
BATCH6_ASSERTER.assertRecord(r, rnd, ts, blob);
// generate blob to roll forward random generator, don't assert blob value
if (rnd.nextBoolean()) {
rnd.nextChars(blob, blobLen / 2);
}
private static final RecordAssert BATCH_2_7_BEFORE_ASSERTER = (r, rnd, ts, blob) -> assertNullStr(r, 10);
private static final RecordAssert BATCH_2_9_BEFORE_ASSERTER = (r, rnd, ts, blob) -> assertNullStr(r, 9);
private static final RecordAssert BATCH_3_7_BEFORE_ASSERTER = (r, rnd, ts, blob) -> Assert.assertEquals(Numbers.INT_NaN, r.getInt(11));
private static final RecordAssert BATCH_3_9_BEFORE_ASSERTER = (r, rnd, ts, blob) -> Assert.assertEquals(Numbers.INT_NaN, r.getInt(10));
private static final RecordAssert BATCH_4_7_BEFORE_ASSERTER = (r, rnd, ts, blob) -> {
Assert.assertEquals(0, r.getShort(12));
Assert.assertFalse(r.getBool(13));
Assert.assertEquals(0, r.getByte(14));
Assert.assertTrue(Float.isNaN(r.getFloat(15)));
Assert.assertTrue(Double.isNaN(r.getDouble(16)));
Assert.assertNull(r.getSym(17));
Assert.assertEquals(Numbers.LONG_NaN, r.getLong(18));
Assert.assertEquals(Numbers.LONG_NaN, r.getDate(19));
};
private static final RecordAssert BATCH_4_9_BEFORE_ASSERTER = (r, rnd, ts, blob) -> {
Assert.assertEquals(0, r.getShort(11));
Assert.assertFalse(r.getBool(12));
Assert.assertEquals(0, r.getByte(13));
Assert.assertTrue(Float.isNaN(r.getFloat(14)));
Assert.assertTrue(Double.isNaN(r.getDouble(15)));
Assert.assertNull(r.getSym(16));
Assert.assertEquals(Numbers.LONG_NaN, r.getLong(17));
Assert.assertEquals(Numbers.LONG_NaN, r.getDate(18));
};
private static final RecordAssert BATCH3_BEFORE_ASSERTER = (r, rnd, ts, blob) -> Assert.assertEquals(Numbers.INT_NaN, r.getInt(12));
private static final RecordAssert BATCH4_BEFORE_ASSERTER = (r, rnd, ts, blob) -> {
Assert.assertEquals(0, r.getShort(13));
Assert.assertFalse(r.getBool(14));
Assert.assertEquals(0, r.getByte(15));
Assert.assertTrue(Float.isNaN(r.getFloat(16)));
Assert.assertTrue(Double.isNaN(r.getDouble(17)));
Assert.assertNull(r.getSym(18));
Assert.assertEquals(Numbers.LONG_NaN, r.getLong(19));
Assert.assertEquals(Numbers.LONG_NaN, r.getDate(20));
Assert.assertNull(r.getBin(21));
Assert.assertEquals(TableUtils.NULL_LEN, r.getBinLen(21));
};
private static final RecordAssert BATCH5_BEFORE_ASSERTER = (r, rnd, ts, blob) -> {
Assert.assertEquals(0, r.getShort(13));
Assert.assertFalse(r.getBool(14));
Assert.assertEquals(0, r.getByte(15));
Assert.assertTrue(Float.isNaN(r.getFloat(16)));
Assert.assertTrue(Double.isNaN(r.getDouble(17)));
Assert.assertNull(r.getSym(18));
Assert.assertEquals(Numbers.LONG_NaN, r.getLong(19));
Assert.assertEquals(Numbers.LONG_NaN, r.getDate(20));
};
private static final FieldGenerator BATCH1_GENERATOR = (r1, rnd1, ts1, blob1) -> {
if (rnd1.nextBoolean()) {
......@@ -795,6 +794,95 @@ public class TableReaderTest extends AbstractCairoTest {
r1.putSym(7, rnd1.nextChars(7));
}
};
private static final FieldGenerator BATCH2_GENERATOR = (r1, rnd1, ts1, blob1) -> {
BATCH1_GENERATOR.generate(r1, rnd1, ts1, blob1);
if ((rnd1.nextPositiveInt() & 3) == 0) {
r1.putStr(11, rnd1.nextChars(15));
}
};
private static final FieldGenerator BATCH3_GENERATOR = (r1, rnd1, ts1, blob1) -> {
BATCH2_GENERATOR.generate(r1, rnd1, ts1, blob1);
if ((rnd1.nextPositiveInt() & 3) == 0) {
r1.putInt(12, rnd1.nextInt());
}
};
private static final FieldGenerator BATCH4_GENERATOR = (r, rnd, ts, blob) -> {
BATCH3_GENERATOR.generate(r, rnd, ts, blob);
if (rnd.nextBoolean()) {
r.putShort(13, rnd.nextShort());
}
if (rnd.nextBoolean()) {
r.putBool(14, rnd.nextBoolean());
}
if (rnd.nextBoolean()) {
r.putByte(15, rnd.nextByte());
}
if (rnd.nextBoolean()) {
r.putFloat(16, rnd.nextFloat());
}
if (rnd.nextBoolean()) {
r.putDouble(17, rnd.nextDouble());
}
if (rnd.nextBoolean()) {
r.putSym(18, rnd.nextChars(10));
}
if (rnd.nextBoolean()) {
r.putLong(19, rnd.nextLong());
}
if (rnd.nextBoolean()) {
r.putDate(20, rnd.nextLong());
}
if (rnd.nextBoolean()) {
rnd.nextChars(blob, blobLen / 2);
r.putBin(21, blob, blobLen);
}
};
private static final FieldGenerator BATCH6_GENERATOR = (r, rnd, ts, blob) -> {
BATCH3_GENERATOR.generate(r, rnd, ts, blob);
if (rnd.nextBoolean()) {
r.putShort(13, rnd.nextShort());
}
if (rnd.nextBoolean()) {
r.putBool(14, rnd.nextBoolean());
}
if (rnd.nextBoolean()) {
r.putByte(15, rnd.nextByte());
}
if (rnd.nextBoolean()) {
r.putFloat(16, rnd.nextFloat());
}
if (rnd.nextBoolean()) {
r.putDouble(17, rnd.nextDouble());
}
if (rnd.nextBoolean()) {
r.putSym(18, rnd.nextChars(10));
}
if (rnd.nextBoolean()) {
r.putLong(19, rnd.nextLong());
}
if (rnd.nextBoolean()) {
r.putDate(20, rnd.nextLong());
}
};
private static final RecordAssert BATCH8_ASSERTER = (r, rnd, ts, blob) -> {
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextByte(), r.getByte(1));
......@@ -1105,152 +1193,63 @@ public class TableReaderTest extends AbstractCairoTest {
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextShort(), r.getShort(11));
} else {
Assert.assertEquals(0, r.getShort(11));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextBoolean(), r.getBool(12));
} else {
Assert.assertFalse(r.getBool(12));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextByte(), r.getByte(13));
} else {
Assert.assertEquals(0, r.getByte(13));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextFloat(), r.getFloat(14), 0.000001f);
} else {
Assert.assertTrue(Float.isNaN(r.getFloat(14)));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextDouble(), r.getDouble(15), 0.0000001d);
} else {
Assert.assertTrue(Double.isNaN(r.getDouble(15)));
}
if (rnd.nextBoolean()) {
TestUtils.assertEquals(rnd.nextChars(10), r.getSym(16));
} else {
Assert.assertNull(r.getSym(16));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextLong(), r.getLong(17));
} else {
Assert.assertEquals(Numbers.LONG_NaN, r.getLong(17));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextLong(), r.getDate(18));
} else {
Assert.assertEquals(Numbers.LONG_NaN, r.getDate(18));
}
if (rnd.nextBoolean()) {
Assert.assertEquals(rnd.nextInt(), r.getInt(19));
} else {
Assert.assertEquals(Numbers.INT_NaN, r.getInt(19));
}
if (rnd.nextBoolean()) {
TestUtils.assertEquals(rnd.nextChars(8), r.getSym(20));
} else {
Assert.assertNull(r.getSym(20));
}
};
private static final FieldGenerator BATCH2_GENERATOR = (r1, rnd1, ts1, blob1) -> {
BATCH1_GENERATOR.generate(r1, rnd1, ts1, blob1);
if ((rnd1.nextPositiveInt() & 3) == 0) {
r1.putStr(11, rnd1.nextChars(15));
}
};
private static final FieldGenerator BATCH3_GENERATOR = (r1, rnd1, ts1, blob1) -> {
BATCH2_GENERATOR.generate(r1, rnd1, ts1, blob1);
if ((rnd1.nextPositiveInt() & 3) == 0) {
r1.putInt(12, rnd1.nextInt());
}
};
private static final FieldGenerator BATCH4_GENERATOR = (r, rnd, ts, blob) -> {
BATCH3_GENERATOR.generate(r, rnd, ts, blob);
if (rnd.nextBoolean()) {
r.putShort(13, rnd.nextShort());
}
if (rnd.nextBoolean()) {
r.putBool(14, rnd.nextBoolean());
}
if (rnd.nextBoolean()) {
r.putByte(15, rnd.nextByte());
}
if (rnd.nextBoolean()) {
r.putFloat(16, rnd.nextFloat());
}
if (rnd.nextBoolean()) {
r.putDouble(17, rnd.nextDouble());
}
if (rnd.nextBoolean()) {
r.putSym(18, rnd.nextChars(10));
}
if (rnd.nextBoolean()) {
r.putLong(19, rnd.nextLong());
}
if (rnd.nextBoolean()) {
r.putDate(20, rnd.nextLong());
Assert.assertEquals(rnd.nextShort(), r.getShort(11));
} else {
Assert.assertEquals(0, r.getShort(11));
}
if (rnd.nextBoolean()) {
rnd.nextChars(blob, blobLen / 2);
r.putBin(21, blob, blobLen);
Assert.assertEquals(rnd.nextBoolean(), r.getBool(12));
} else {
Assert.assertFalse(r.getBool(12));
}
};
private static final FieldGenerator BATCH6_GENERATOR = (r, rnd, ts, blob) -> {
BATCH3_GENERATOR.generate(r, rnd, ts, blob);
if (rnd.nextBoolean()) {
r.putShort(13, rnd.nextShort());
Assert.assertEquals(rnd.nextByte(), r.getByte(13));
} else {
Assert.assertEquals(0, r.getByte(13));
}
if (rnd.nextBoolean()) {
r.putBool(14, rnd.nextBoolean());
Assert.assertEquals(rnd.nextFloat(), r.getFloat(14), 0.000001f);
} else {
Assert.assertTrue(Float.isNaN(r.getFloat(14)));
}
if (rnd.nextBoolean()) {
r.putByte(15, rnd.nextByte());
Assert.assertEquals(rnd.nextDouble(), r.getDouble(15), 0.0000001d);
} else {
Assert.assertTrue(Double.isNaN(r.getDouble(15)));
}
if (rnd.nextBoolean()) {
r.putFloat(16, rnd.nextFloat());
TestUtils.assertEquals(rnd.nextChars(10), r.getSym(16));
} else {
Assert.assertNull(r.getSym(16));
}
if (rnd.nextBoolean()) {
r.putDouble(17, rnd.nextDouble());
Assert.assertEquals(rnd.nextLong(), r.getLong(17));
} else {
Assert.assertEquals(Numbers.LONG_NaN, r.getLong(17));
}
if (rnd.nextBoolean()) {
r.putSym(18, rnd.nextChars(10));
Assert.assertEquals(rnd.nextLong(), r.getDate(18));
} else {
Assert.assertEquals(Numbers.LONG_NaN, r.getDate(18));
}
if (rnd.nextBoolean()) {
r.putLong(19, rnd.nextLong());
Assert.assertEquals(rnd.nextInt(), r.getInt(19));
} else {
Assert.assertEquals(Numbers.INT_NaN, r.getInt(19));
}
if (rnd.nextBoolean()) {
r.putDate(20, rnd.nextLong());
TestUtils.assertEquals(rnd.nextChars(8), r.getSym(20));
} else {
Assert.assertNull(r.getSym(20));
}
};
private static final FieldGenerator BATCH8_GENERATOR = (r, rnd, ts, blob) -> {
......@@ -1542,107 +1541,6 @@ public class TableReaderTest extends AbstractCairoTest {
testConcurrentReloadMultiplePartitions(PartitionBy.MONTH, 12 * 3000000);
}
static boolean isSamePartition(long timestampA, long timestampB, int partitionBy) {
switch (partitionBy) {
case PartitionBy.NONE:
return true;
case PartitionBy.DAY:
return Timestamps.floorDD(timestampA) == Timestamps.floorDD(timestampB);
case PartitionBy.MONTH:
return Timestamps.floorMM(timestampA) == Timestamps.floorMM(timestampB);
case PartitionBy.YEAR:
return Timestamps.floorYYYY(timestampA) == Timestamps.floorYYYY(timestampB);
case PartitionBy.HOUR:
return Timestamps.floorHH(timestampA) == Timestamps.floorHH(timestampB);
default:
throw CairoException.instance(0).put("Cannot compare timestamps for unsupported partition type: [").put(partitionBy).put(']');
}
}
private void testConcurrentReloadMultiplePartitions(int partitionBy, long stride) throws Exception {
TestUtils.assertMemoryLeak(() -> {
final int N = 1024_0000;
// model table
try (TableModel model = new TableModel(configuration, "w", partitionBy).col("l", ColumnType.LONG).timestamp()) {
CairoTestUtils.create(model);
}
final int threads = 2;
final CyclicBarrier startBarrier = new CyclicBarrier(threads);
final CountDownLatch stopLatch = new CountDownLatch(threads);
final AtomicInteger errors = new AtomicInteger(0);
// start writer
new Thread(() -> {
try {
startBarrier.await();
long timestampUs = TimestampFormatUtils.parseTimestamp("2017-12-11T00:00:00.000Z");
try (TableWriter writer = new TableWriter(configuration, "w")) {
for (int i = 0; i < N; i++) {
TableWriter.Row row = writer.newRow(timestampUs);
row.putLong(0, i);
row.append();
writer.commit();
timestampUs += stride;
}
}
} catch (Exception e) {
e.printStackTrace();
errors.incrementAndGet();
} finally {
stopLatch.countDown();
}
}).start();
// start reader
new Thread(() -> {
try {
startBarrier.await();
try (TableReader reader = new TableReader(configuration, "w")) {
RecordCursor cursor = reader.getCursor();
final Record record = cursor.getRecord();
sink.clear();
((Sinkable)record).toSink(sink);
TestUtils.assertEquals("TableReaderRecord [columnBase=0, recordIndex=-1]", sink);
do {
// we deliberately ignore result of reload()
// to create more race conditions
reader.reload();
cursor.toTop();
int count = 0;
while (cursor.hasNext()) {
if (count++ != record.getLong(0)) {
sink.clear();
sink.put("Test [count=").put(count--).put(", rec=").put(record.getLong(0)).put(']').put(',');
((Sinkable)record).toSink(sink);
Assert.fail(sink.toString());
}
}
if (count == N) {
break;
}
} while (true);
}
} catch (Throwable e) {
e.printStackTrace();
errors.incrementAndGet();
} finally {
stopLatch.countDown();
}
}).start();
Assert.assertTrue(stopLatch.await(120, TimeUnit.SECONDS));
Assert.assertEquals(0, errors.get());
// check that we had multiple partitions created during the test
try (TableReader reader = new TableReader(configuration, "w")) {
Assert.assertTrue(reader.getPartitionCount() > 10);
}
});
}
@Test
public void testConcurrentReloadNonPartitioned() throws Exception {
testConcurrentReloadSinglePartition(PartitionBy.NONE);
......@@ -2112,6 +2010,46 @@ public class TableReaderTest extends AbstractCairoTest {
});
}
@Test
public void testReaderReloadWhenColumnAddedBeforeTheData() throws Exception {
assertMemoryLeak(
() -> {
// model table
try (TableModel model = new TableModel(configuration, "w", PartitionBy.HOUR).col("l", ColumnType.LONG).timestamp()) {
CairoTestUtils.create(model);
}
try (
TableWriter w = new TableWriter(configuration, "w");
TableReader r = new TableReader(configuration, "w")
) {
// Create and cancel row to ensure partition entry and NULL max timestamp
// this used to trigger a problem with very last reload of the reader.
w.newRow(TimestampFormatUtils.parseTimestamp("2016-03-02T10:00:00.000000Z")).cancel();
// before adding any data add column
w.addColumn("xyz", ColumnType.SYMBOL);
Assert.assertTrue(r.reload());
TableWriter.Row row = w.newRow(TimestampFormatUtils.parseTimestamp("2016-03-02T10:00:00.000000Z"));
row.append();
w.commit();
Assert.assertTrue(r.reload());
sink.clear();
TestUtils.printer.print(r.getCursor(), r.getMetadata(), true, sink);
TestUtils.assertEquals(
"l\ttimestamp\txyz\n" +
"NaN\t2016-03-02T10:00:00.000000Z\t\n",
sink
);
}
}
);
}
@Test
public void testReloadByDaySwitch() throws Exception {
testReload(PartitionBy.DAY, 150, 6 * 60000L, MUST_SWITCH);
......@@ -3255,6 +3193,23 @@ public class TableReaderTest extends AbstractCairoTest {
});
}
static boolean isSamePartition(long timestampA, long timestampB, int partitionBy) {
switch (partitionBy) {
case PartitionBy.NONE:
return true;
case PartitionBy.DAY:
return Timestamps.floorDD(timestampA) == Timestamps.floorDD(timestampB);
case PartitionBy.MONTH:
return Timestamps.floorMM(timestampA) == Timestamps.floorMM(timestampB);
case PartitionBy.YEAR:
return Timestamps.floorYYYY(timestampA) == Timestamps.floorYYYY(timestampB);
case PartitionBy.HOUR:
return Timestamps.floorHH(timestampA) == Timestamps.floorHH(timestampB);
default:
throw CairoException.instance(0).put("Cannot compare timestamps for unsupported partition type: [").put(partitionBy).put(']');
}
}
private static long allocBlob() {
return Unsafe.malloc(blobLen, MemoryTag.NATIVE_DEFAULT);
}
......@@ -3605,6 +3560,90 @@ public class TableReaderTest extends AbstractCairoTest {
}
}
private void testConcurrentReloadMultiplePartitions(int partitionBy, long stride) throws Exception {
TestUtils.assertMemoryLeak(() -> {
final int N = 1024_0000;
// model table
try (TableModel model = new TableModel(configuration, "w", partitionBy).col("l", ColumnType.LONG).timestamp()) {
CairoTestUtils.create(model);
}
final int threads = 2;
final CyclicBarrier startBarrier = new CyclicBarrier(threads);
final CountDownLatch stopLatch = new CountDownLatch(threads);
final AtomicInteger errors = new AtomicInteger(0);
// start writer
new Thread(() -> {
try {
startBarrier.await();
long timestampUs = TimestampFormatUtils.parseTimestamp("2017-12-11T00:00:00.000Z");
try (TableWriter writer = new TableWriter(configuration, "w")) {
for (int i = 0; i < N; i++) {
TableWriter.Row row = writer.newRow(timestampUs);
row.putLong(0, i);
row.append();
writer.commit();
timestampUs += stride;
}
}
} catch (Exception e) {
e.printStackTrace();
errors.incrementAndGet();
} finally {
stopLatch.countDown();
}
}).start();
// start reader
new Thread(() -> {
try {
startBarrier.await();
try (TableReader reader = new TableReader(configuration, "w")) {
RecordCursor cursor = reader.getCursor();
final Record record = cursor.getRecord();
sink.clear();
((Sinkable) record).toSink(sink);
TestUtils.assertEquals("TableReaderRecord [columnBase=0, recordIndex=-1]", sink);
do {
// we deliberately ignore result of reload()
// to create more race conditions
reader.reload();
cursor.toTop();
int count = 0;
while (cursor.hasNext()) {
if (count++ != record.getLong(0)) {
sink.clear();
sink.put("Test [count=").put(count--).put(", rec=").put(record.getLong(0)).put(']').put(',');
((Sinkable) record).toSink(sink);
Assert.fail(sink.toString());
}
}
if (count == N) {
break;
}
} while (true);
}
} catch (Throwable e) {
e.printStackTrace();
errors.incrementAndGet();
} finally {
stopLatch.countDown();
}
}).start();
Assert.assertTrue(stopLatch.await(120, TimeUnit.SECONDS));
Assert.assertEquals(0, errors.get());
// check that we had multiple partitions created during the test
try (TableReader reader = new TableReader(configuration, "w")) {
Assert.assertTrue(reader.getPartitionCount() > 10);
}
});
}
private void testReload(int partitionBy, int count, long inct, final int testPartitionSwitch) throws Exception {
final long increment = inct * 1000;
......
......@@ -27,6 +27,9 @@ package io.questdb.cutlass.line.tcp;
import org.junit.Ignore;
import org.junit.Test;
@Ignore
// we have known issues with handling metadata reload in TableReader. Alex is working on a solution. When
// this is ready we will make this test part of the CI
public class LineTcpReceiverFuzzTest extends AbstractLineTcpReceiverFuzzTest {
// there seem to be an issue with the transactionality of adding new columns
......
......@@ -88,6 +88,6 @@ public class OsTest {
TestUtils.await(barrier);
t.interrupt();
Assert.assertTrue(doneLatch.await(10_000_000_000L));
Assert.assertTrue(System.currentTimeMillis() - time > 1000);
Assert.assertTrue(System.currentTimeMillis() - time >= 1000);
}
}
\ No newline at end of file
......@@ -24,9 +24,9 @@
package io.questdb.test.tools;
import io.questdb.cairo.sql.Record;
import io.questdb.cairo.*;
import io.questdb.cairo.sql.*;
import io.questdb.cairo.sql.Record;
import io.questdb.griffin.CompiledQuery;
import io.questdb.griffin.SqlCompiler;
import io.questdb.griffin.SqlException;
......@@ -46,7 +46,6 @@ import org.jetbrains.annotations.NotNull;
import org.junit.Assert;
import java.io.*;
import java.util.concurrent.BrokenBarrierException;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.atomic.AtomicInteger;
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册