newModification = compactionModificationFile.getModifications();
compactionModificationFile.close();
- sourceFile.resetModFile();
// write the new modifications to its old modification file
try (ModificationFile oldModificationFile = sourceFile.getModFile()) {
for (Modification modification : newModification) {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index ffbd9fcf89..630df4ffff 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -97,6 +97,8 @@ public class TsFileResource {
private ModificationFile modFile;
+ private ModificationFile compactionModFile;
+
protected volatile boolean closed = false;
private volatile boolean deleted = false;
volatile boolean isMerging = false;
@@ -357,14 +359,14 @@ public class TsFileResource {
}
public ModificationFile getCompactionModFile() {
- if (modFile == null) {
+ if (compactionModFile == null) {
synchronized (this) {
- if (modFile == null) {
- modFile = ModificationFile.getCompactionMods(this);
+ if (compactionModFile == null) {
+ compactionModFile = ModificationFile.getCompactionMods(this);
}
}
}
- return modFile;
+ return compactionModFile;
}
public void resetModFile() {
@@ -442,6 +444,10 @@ public class TsFileResource {
modFile.close();
modFile = null;
}
+ if (compactionModFile != null) {
+ compactionModFile.close();
+ compactionModFile = null;
+ }
processor = null;
pathToChunkMetadataListMap = null;
pathToReadOnlyMemChunkMap = null;
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/VirtualStorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/VirtualStorageGroupProcessor.java
index 5c7f7caae0..b25a09c420 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/VirtualStorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/VirtualStorageGroupProcessor.java
@@ -2006,10 +2006,13 @@ public class VirtualStorageGroupProcessor {
// we have to set modification offset to MAX_VALUE, as the offset of source chunk may
// change after compaction
deletion.setFileOffset(Long.MAX_VALUE);
- // write deletion into modification file
+ // write deletion into compaction modification file
tsFileResource.getCompactionModFile().write(deletion);
+ // write deletion into modification file to enable query during compaction
+ tsFileResource.getModFile().write(deletion);
// remember to close mod file
tsFileResource.getCompactionModFile().close();
+ tsFileResource.getModFile().close();
} else {
deletion.setFileOffset(tsFileResource.getTsFileSize());
// write deletion into modification file
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java
index 2067f437f1..7e7f1b8d63 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java
@@ -191,15 +191,16 @@ public class AbstractCompactionTest {
TsFileResource resource = new TsFileResource(file);
int deviceStartindex = 0;
if (isAlign) {
+ deviceStartindex = TsFileGeneratorUtils.getAlignDeviceOffset();
+ for (int i = deviceStartindex; i < deviceStartindex + deviceNum; i++) {
+ resource.updateStartTime(COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i, startTime);
+ resource.updateEndTime(COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i, endTime);
+ }
+ } else {
for (int i = deviceStartindex; i < deviceStartindex + deviceNum; i++) {
resource.updateStartTime(COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i, startTime);
resource.updateEndTime(COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i, endTime);
}
- deviceStartindex = TsFileGeneratorUtils.getAlignDeviceOffset();
- }
- for (int i = deviceStartindex; i < deviceStartindex + deviceNum; i++) {
- resource.updateStartTime(COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i, startTime);
- resource.updateEndTime(COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i, endTime);
}
resource.updatePlanIndexes(fileVersion);
resource.setClosed(true);
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java
index 3d3916c04f..aaa8d82fef 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java
@@ -22,12 +22,16 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.compaction.AbstractCompactionTest;
import org.apache.iotdb.db.engine.compaction.cross.rewrite.task.RewriteCrossSpaceCompactionTask;
import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils;
+import org.apache.iotdb.db.engine.flush.TsFileFlushPolicy;
import org.apache.iotdb.db.engine.storagegroup.TsFileManager;
+import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.engine.storagegroup.VirtualStorageGroupProcessor;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.exception.metadata.IllegalPathException;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.metadata.path.AlignedPath;
+import org.apache.iotdb.db.metadata.path.PartialPath;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.query.reader.series.SeriesRawDataBatchReader;
import org.apache.iotdb.db.utils.EnvironmentUtils;
@@ -316,6 +320,477 @@ public class RewriteCrossSpaceCompactionTest extends AbstractCompactionTest {
}
}
+ /**
+ * Total 4 seq files and 5 unseq files, each file has different aligned timeseries.
+ *
+ * Seq files
+ * first and second file has d0 ~ d1 and s0 ~ s2, time range is 0 ~ 299 and 350 ~ 649, value range
+ * is 0 ~ 299 and 350 ~ 649.
+ * third and forth file has d0 ~ d3 and s0 ~ S4,time range is 700 ~ 999 and 1050 ~ 1349, value
+ * range is 700 ~ 999 and 1050 ~ 1349.
+ *
+ *
UnSeq files
+ * first, second and third file has d0 ~ d2 and s0 ~ s3, time range is 20 ~ 219, 250 ~ 449 and 480
+ * ~ 679, value range is 10020 ~ 10219, 10250 ~ 10449 and 10480 ~ 10679.
+ * forth and fifth file has d0 and s0 ~ s4, time range is 450 ~ 549 and 550 ~ 649, value range is
+ * 20450 ~ 20549 and 20550 ~ 20649.
+ *
+ *
The data of d0, d1 and d2 is deleted in each file. The first target file is empty.
+ */
+ @Test
+ public void testAlignedCrossSpaceCompactionWithAllDataDeletedInOneTargetFile() throws Exception {
+ TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(30);
+ registerTimeseriesInMManger(4, 5, true);
+ createFiles(2, 2, 3, 300, 0, 0, 50, 50, true, true);
+ createFiles(2, 4, 5, 300, 700, 700, 50, 50, true, true);
+ createFiles(3, 3, 4, 200, 20, 10020, 30, 30, true, false);
+ createFiles(2, 1, 5, 100, 450, 20450, 0, 0, true, false);
+
+ // generate mods file
+ List seriesPaths = new ArrayList<>();
+ for (int i = 0; i < 5; i++) {
+ seriesPaths.add(
+ COMPACTION_TEST_SG
+ + PATH_SEPARATOR
+ + "d"
+ + TsFileGeneratorUtils.getAlignDeviceOffset()
+ + PATH_SEPARATOR
+ + "s"
+ + i);
+ seriesPaths.add(
+ COMPACTION_TEST_SG
+ + PATH_SEPARATOR
+ + "d"
+ + (TsFileGeneratorUtils.getAlignDeviceOffset() + 1)
+ + PATH_SEPARATOR
+ + "s"
+ + i);
+ seriesPaths.add(
+ COMPACTION_TEST_SG
+ + PATH_SEPARATOR
+ + "d"
+ + (TsFileGeneratorUtils.getAlignDeviceOffset() + 2)
+ + PATH_SEPARATOR
+ + "s"
+ + i);
+ seriesPaths.add(COMPACTION_TEST_SG + PATH_SEPARATOR + "d0" + PATH_SEPARATOR + "s" + i);
+ seriesPaths.add(COMPACTION_TEST_SG + PATH_SEPARATOR + "d1" + PATH_SEPARATOR + "s" + i);
+ seriesPaths.add(COMPACTION_TEST_SG + PATH_SEPARATOR + "d2" + PATH_SEPARATOR + "s" + i);
+ }
+ generateModsFile(seriesPaths, seqResources, Long.MIN_VALUE, Long.MAX_VALUE, false);
+ generateModsFile(seriesPaths, unseqResources, Long.MIN_VALUE, Long.MAX_VALUE, false);
+ generateModsFile(seriesPaths, seqResources, Long.MIN_VALUE, Long.MAX_VALUE, true);
+ generateModsFile(seriesPaths, unseqResources, Long.MIN_VALUE, Long.MAX_VALUE, true);
+
+ for (int i = TsFileGeneratorUtils.getAlignDeviceOffset();
+ i < TsFileGeneratorUtils.getAlignDeviceOffset() + 4;
+ i++) {
+ for (int j = 0; j < 5; j++) {
+ List schemas = new ArrayList<>();
+ schemas.add(new MeasurementSchema("s" + j, TSDataType.INT64));
+ AlignedPath path =
+ new AlignedPath(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i,
+ Collections.singletonList("s" + j),
+ schemas);
+ IBatchReader tsFilesReader =
+ new SeriesRawDataBatchReader(
+ path,
+ TSDataType.VECTOR,
+ EnvironmentUtils.TEST_QUERY_CONTEXT,
+ seqResources,
+ unseqResources,
+ null,
+ null,
+ true);
+ int count = 0;
+ while (tsFilesReader.hasNextBatch()) {
+ BatchData batchData = tsFilesReader.nextBatch();
+ while (batchData.hasCurrent()) {
+ if (i == TsFileGeneratorUtils.getAlignDeviceOffset()
+ && ((450 <= batchData.currentTime() && batchData.currentTime() < 550)
+ || (550 <= batchData.currentTime() && batchData.currentTime() < 650))) {
+ assertEquals(
+ batchData.currentTime() + 20000,
+ ((TsPrimitiveType[]) (batchData.currentValue()))[0].getValue());
+ } else if ((i < TsFileGeneratorUtils.getAlignDeviceOffset() + 3 && j < 4)
+ && ((20 <= batchData.currentTime() && batchData.currentTime() < 220)
+ || (250 <= batchData.currentTime() && batchData.currentTime() < 450)
+ || (480 <= batchData.currentTime() && batchData.currentTime() < 680))) {
+ assertEquals(
+ batchData.currentTime() + 10000,
+ ((TsPrimitiveType[]) (batchData.currentValue()))[0].getValue());
+ } else {
+ assertEquals(
+ batchData.currentTime(),
+ ((TsPrimitiveType[]) (batchData.currentValue()))[0].getValue());
+ }
+ count++;
+ batchData.next();
+ }
+ }
+ tsFilesReader.close();
+ if (i == 0 || i == 1 || i == 2) {
+ assertEquals(0, count);
+ }
+ if ((i == TsFileGeneratorUtils.getAlignDeviceOffset())
+ || (i == TsFileGeneratorUtils.getAlignDeviceOffset() + 1)
+ || (i == TsFileGeneratorUtils.getAlignDeviceOffset() + 2)) {
+ assertEquals(0, count);
+ } else if (i < TsFileGeneratorUtils.getAlignDeviceOffset() + 2 && j < 3) {
+ assertEquals(1280, count);
+ } else if (i < TsFileGeneratorUtils.getAlignDeviceOffset() + 1 && j < 4) {
+ assertEquals(1230, count);
+ } else if ((i == TsFileGeneratorUtils.getAlignDeviceOffset() + 1 && j == 4)) {
+ assertEquals(600, count);
+ } else if (i < TsFileGeneratorUtils.getAlignDeviceOffset() + 3 && j < 4) {
+ assertEquals(1200, count);
+ } else {
+ assertEquals(600, count);
+ }
+ }
+ }
+
+ List targetResources =
+ CompactionFileGeneratorUtils.getCrossCompactionTargetTsFileResources(seqResources);
+ TsFileManager tsFileManager =
+ new TsFileManager(COMPACTION_TEST_SG, "0", STORAGE_GROUP_DIR.getPath());
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+ RewriteCrossSpaceCompactionTask rewriteCrossSpaceCompactionTask =
+ new RewriteCrossSpaceCompactionTask(
+ COMPACTION_TEST_SG,
+ "0",
+ 0,
+ STORAGE_GROUP_DIR.getPath(),
+ tsFileManager,
+ seqResources,
+ unseqResources,
+ new AtomicInteger(0));
+ rewriteCrossSpaceCompactionTask.call();
+
+ for (TsFileResource resource : seqResources) {
+ Assert.assertFalse(resource.getModFile().exists());
+ }
+ for (TsFileResource resource : unseqResources) {
+ Assert.assertFalse(resource.getModFile().exists());
+ }
+ for (TsFileResource resource : targetResources) {
+ resource.setFile(
+ new File(
+ resource
+ .getTsFilePath()
+ .replace(CROSS_COMPACTION_TMP_FILE_SUFFIX, TsFileConstant.TSFILE_SUFFIX)));
+ if (!resource.getTsFile().exists()) {
+ continue;
+ }
+ Assert.assertTrue(resource.getModFile().exists());
+ Assert.assertEquals(180, resource.getModFile().getModifications().size());
+ }
+ FileReaderManager.getInstance().closeAndRemoveAllOpenedReaders();
+
+ for (int i = TsFileGeneratorUtils.getAlignDeviceOffset();
+ i < TsFileGeneratorUtils.getAlignDeviceOffset() + 4;
+ i++) {
+ for (int j = 0; j < 5; j++) {
+ List schemas = new ArrayList<>();
+ schemas.add(new MeasurementSchema("s" + j, TSDataType.INT64));
+ AlignedPath path =
+ new AlignedPath(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i,
+ Collections.singletonList("s" + j),
+ schemas);
+ IBatchReader tsFilesReader =
+ new SeriesRawDataBatchReader(
+ path,
+ TSDataType.VECTOR,
+ EnvironmentUtils.TEST_QUERY_CONTEXT,
+ tsFileManager.getTsFileList(true),
+ new ArrayList<>(),
+ null,
+ null,
+ true);
+ int count = 0;
+ while (tsFilesReader.hasNextBatch()) {
+ BatchData batchData = tsFilesReader.nextBatch();
+ while (batchData.hasCurrent()) {
+ if (i == TsFileGeneratorUtils.getAlignDeviceOffset()
+ && ((450 <= batchData.currentTime() && batchData.currentTime() < 550)
+ || (550 <= batchData.currentTime() && batchData.currentTime() < 650))) {
+ assertEquals(
+ batchData.currentTime() + 20000,
+ ((TsPrimitiveType[]) (batchData.currentValue()))[0].getValue());
+ } else if ((i < TsFileGeneratorUtils.getAlignDeviceOffset() + 3 && j < 4)
+ && ((20 <= batchData.currentTime() && batchData.currentTime() < 220)
+ || (250 <= batchData.currentTime() && batchData.currentTime() < 450)
+ || (480 <= batchData.currentTime() && batchData.currentTime() < 680))) {
+ assertEquals(
+ batchData.currentTime() + 10000,
+ ((TsPrimitiveType[]) (batchData.currentValue()))[0].getValue());
+ } else {
+ assertEquals(
+ batchData.currentTime(),
+ ((TsPrimitiveType[]) (batchData.currentValue()))[0].getValue());
+ }
+ count++;
+ batchData.next();
+ }
+ }
+ tsFilesReader.close();
+ if (i == 0 || i == 1 || i == 2) {
+ assertEquals(0, count);
+ }
+ if ((i == TsFileGeneratorUtils.getAlignDeviceOffset())
+ || (i == TsFileGeneratorUtils.getAlignDeviceOffset() + 1)
+ || (i == TsFileGeneratorUtils.getAlignDeviceOffset() + 2)) {
+ assertEquals(0, count);
+ } else if (i < TsFileGeneratorUtils.getAlignDeviceOffset() + 2 && j < 3) {
+ assertEquals(1280, count);
+ } else if (i < TsFileGeneratorUtils.getAlignDeviceOffset() + 1 && j < 4) {
+ assertEquals(1230, count);
+ } else if ((i == TsFileGeneratorUtils.getAlignDeviceOffset() + 1 && j == 4)) {
+ assertEquals(600, count);
+ } else if (i < TsFileGeneratorUtils.getAlignDeviceOffset() + 3 && j < 4) {
+ assertEquals(1200, count);
+ } else {
+ assertEquals(600, count);
+ }
+ }
+ }
+ }
+
+ /**
+ * Total 4 seq files and 5 unseq files, each file has different aligned timeseries.
+ *
+ * Seq files
+ * first and second file has d0 ~ d1 and s0 ~ s2, time range is 0 ~ 299 and 350 ~ 649, value range
+ * is 0 ~ 299 and 350 ~ 649.
+ * third and forth file has d0 ~ d3 and s0 ~ S4,time range is 700 ~ 999 and 1050 ~ 1349, value
+ * range is 700 ~ 999 and 1050 ~ 1349.
+ *
+ *
UnSeq files
+ * first, second and third file has d0 ~ d2 and s0 ~ s3, time range is 20 ~ 219, 250 ~ 449 and 480
+ * ~ 679, value range is 10020 ~ 10219, 10250 ~ 10449 and 10480 ~ 10679.
+ * forth and fifth file has d0 and s0 ~ s4, time range is 450 ~ 549 and 550 ~ 649, value range is
+ * 20450 ~ 20549 and 20550 ~ 20649.
+ *
+ *
The data of d3.s0 is deleted in each file. Test when there is a deletion to the file before
+ * compaction, then comes to a deletion during compaction.
+ */
+ @Test
+ public void testOneDeletionDuringCompaction() throws Exception {
+ VirtualStorageGroupProcessor vsgp =
+ new VirtualStorageGroupProcessor(
+ STORAGE_GROUP_DIR.getPath(),
+ "0",
+ new TsFileFlushPolicy.DirectFlushPolicy(),
+ COMPACTION_TEST_SG);
+ registerTimeseriesInMManger(4, 5, true);
+ createFiles(2, 2, 3, 300, 0, 0, 50, 50, true, true);
+ createFiles(2, 4, 5, 300, 700, 700, 50, 50, true, true);
+ createFiles(3, 3, 4, 200, 20, 10020, 30, 30, true, false);
+ createFiles(2, 1, 5, 100, 450, 20450, 0, 0, true, false);
+ vsgp.getTsFileResourceManager().addAll(seqResources, true);
+ vsgp.getTsFileResourceManager().addAll(unseqResources, false);
+ vsgp.delete(
+ new PartialPath(
+ COMPACTION_TEST_SG
+ + PATH_SEPARATOR
+ + "d"
+ + (TsFileGeneratorUtils.getAlignDeviceOffset() + 3)
+ + PATH_SEPARATOR
+ + "s0"),
+ 0,
+ 1000,
+ 0,
+ null);
+
+ RewriteCrossSpaceCompactionTask rewriteCrossSpaceCompactionTask =
+ new RewriteCrossSpaceCompactionTask(
+ COMPACTION_TEST_SG,
+ "0",
+ 0,
+ STORAGE_GROUP_DIR.getPath(),
+ vsgp.getTsFileResourceManager(),
+ seqResources,
+ unseqResources,
+ new AtomicInteger(0));
+ rewriteCrossSpaceCompactionTask.checkValidAndSetMerging();
+ // delete data in source file during compaction
+ vsgp.delete(
+ new PartialPath(
+ COMPACTION_TEST_SG
+ + PATH_SEPARATOR
+ + "d"
+ + (TsFileGeneratorUtils.getAlignDeviceOffset() + 3)
+ + PATH_SEPARATOR
+ + "s0"),
+ 0,
+ 1200,
+ 0,
+ null);
+ for (int i = 0; i < seqResources.size(); i++) {
+ TsFileResource resource = seqResources.get(i);
+ resource.resetModFile();
+ Assert.assertTrue(resource.getCompactionModFile().exists());
+ Assert.assertEquals(1, resource.getCompactionModFile().getModifications().size());
+ Assert.assertTrue(resource.getModFile().exists());
+ if (i == 3) {
+ Assert.assertEquals(1, resource.getModFile().getModifications().size());
+ } else {
+ Assert.assertEquals(2, resource.getModFile().getModifications().size());
+ }
+ }
+ for (TsFileResource resource : unseqResources) {
+ resource.resetModFile();
+ Assert.assertTrue(resource.getCompactionModFile().exists());
+ Assert.assertEquals(1, resource.getCompactionModFile().getModifications().size());
+ Assert.assertTrue(resource.getModFile().exists());
+ Assert.assertEquals(2, resource.getModFile().getModifications().size());
+ }
+ rewriteCrossSpaceCompactionTask.call();
+ for (TsFileResource resource : seqResources) {
+ Assert.assertFalse(resource.getTsFile().exists());
+ Assert.assertFalse(resource.getModFile().exists());
+ Assert.assertFalse(resource.getCompactionModFile().exists());
+ }
+ for (TsFileResource resource : unseqResources) {
+ Assert.assertFalse(resource.getTsFile().exists());
+ Assert.assertFalse(resource.getModFile().exists());
+ Assert.assertFalse(resource.getCompactionModFile().exists());
+ }
+ for (TsFileResource seqResource : seqResources) {
+ TsFileResource resource =
+ new TsFileResource(
+ TsFileNameGenerator.increaseCrossCompactionCnt(seqResource.getTsFile()));
+ Assert.assertTrue(resource.getModFile().exists());
+ Assert.assertEquals(6, resource.getModFile().getModifications().size());
+ Assert.assertFalse(resource.getCompactionModFile().exists());
+ }
+ }
+
+ /**
+ * Total 4 seq files and 5 unseq files, each file has different aligned timeseries.
+ *
+ *
Seq files
+ * first and second file has d0 ~ d1 and s0 ~ s2, time range is 0 ~ 299 and 350 ~ 649, value range
+ * is 0 ~ 299 and 350 ~ 649.
+ * third and forth file has d0 ~ d3 and s0 ~ S4,time range is 700 ~ 999 and 1050 ~ 1349, value
+ * range is 700 ~ 999 and 1050 ~ 1349.
+ *
+ *
UnSeq files
+ * first, second and third file has d0 ~ d2 and s0 ~ s3, time range is 20 ~ 219, 250 ~ 449 and 480
+ * ~ 679, value range is 10020 ~ 10219, 10250 ~ 10449 and 10480 ~ 10679.
+ * forth and fifth file has d0 and s0 ~ s4, time range is 450 ~ 549 and 550 ~ 649, value range is
+ * 20450 ~ 20549 and 20550 ~ 20649.
+ *
+ *
The data of d3.s0 is deleted in each file. Test when there is a deletion to the file before
+ * compaction, then comes to serveral deletions during compaction.
+ */
+ @Test
+ public void testSeveralDeletionsDuringCompaction() throws Exception {
+ VirtualStorageGroupProcessor vsgp =
+ new VirtualStorageGroupProcessor(
+ STORAGE_GROUP_DIR.getPath(),
+ "0",
+ new TsFileFlushPolicy.DirectFlushPolicy(),
+ COMPACTION_TEST_SG);
+ registerTimeseriesInMManger(4, 5, true);
+ createFiles(2, 2, 3, 300, 0, 0, 50, 50, true, true);
+ createFiles(2, 4, 5, 300, 700, 700, 50, 50, true, true);
+ createFiles(3, 3, 4, 200, 20, 10020, 30, 30, true, false);
+ createFiles(2, 1, 5, 100, 450, 20450, 0, 0, true, false);
+ vsgp.getTsFileResourceManager().addAll(seqResources, true);
+ vsgp.getTsFileResourceManager().addAll(unseqResources, false);
+ vsgp.delete(
+ new PartialPath(
+ COMPACTION_TEST_SG
+ + PATH_SEPARATOR
+ + "d"
+ + (TsFileGeneratorUtils.getAlignDeviceOffset() + 3)
+ + PATH_SEPARATOR
+ + "s0"),
+ 0,
+ 1000,
+ 0,
+ null);
+
+ RewriteCrossSpaceCompactionTask rewriteCrossSpaceCompactionTask =
+ new RewriteCrossSpaceCompactionTask(
+ COMPACTION_TEST_SG,
+ "0",
+ 0,
+ STORAGE_GROUP_DIR.getPath(),
+ vsgp.getTsFileResourceManager(),
+ seqResources,
+ unseqResources,
+ new AtomicInteger(0));
+ rewriteCrossSpaceCompactionTask.checkValidAndSetMerging();
+ // delete data in source file during compaction
+ vsgp.delete(
+ new PartialPath(
+ COMPACTION_TEST_SG
+ + PATH_SEPARATOR
+ + "d"
+ + (TsFileGeneratorUtils.getAlignDeviceOffset() + 3)
+ + PATH_SEPARATOR
+ + "s0"),
+ 0,
+ 1200,
+ 0,
+ null);
+ vsgp.delete(
+ new PartialPath(
+ COMPACTION_TEST_SG
+ + PATH_SEPARATOR
+ + "d"
+ + (TsFileGeneratorUtils.getAlignDeviceOffset() + 3)
+ + PATH_SEPARATOR
+ + "s0"),
+ 0,
+ 1800,
+ 0,
+ null);
+ for (int i = 0; i < seqResources.size(); i++) {
+ TsFileResource resource = seqResources.get(i);
+ resource.resetModFile();
+ Assert.assertTrue(resource.getCompactionModFile().exists());
+ Assert.assertEquals(2, resource.getCompactionModFile().getModifications().size());
+ Assert.assertTrue(resource.getModFile().exists());
+ if (i == 3) {
+ Assert.assertEquals(2, resource.getModFile().getModifications().size());
+ } else {
+ Assert.assertEquals(3, resource.getModFile().getModifications().size());
+ }
+ }
+ for (TsFileResource resource : unseqResources) {
+ resource.resetModFile();
+ Assert.assertTrue(resource.getCompactionModFile().exists());
+ Assert.assertEquals(2, resource.getCompactionModFile().getModifications().size());
+ Assert.assertTrue(resource.getModFile().exists());
+ Assert.assertEquals(3, resource.getModFile().getModifications().size());
+ }
+ rewriteCrossSpaceCompactionTask.call();
+ for (TsFileResource resource : seqResources) {
+ Assert.assertFalse(resource.getTsFile().exists());
+ Assert.assertFalse(resource.getModFile().exists());
+ Assert.assertFalse(resource.getCompactionModFile().exists());
+ }
+ for (TsFileResource resource : unseqResources) {
+ Assert.assertFalse(resource.getTsFile().exists());
+ Assert.assertFalse(resource.getModFile().exists());
+ Assert.assertFalse(resource.getCompactionModFile().exists());
+ }
+ for (TsFileResource seqResource : seqResources) {
+ TsFileResource resource =
+ new TsFileResource(
+ TsFileNameGenerator.increaseCrossCompactionCnt(seqResource.getTsFile()));
+ Assert.assertTrue(resource.getModFile().exists());
+ Assert.assertEquals(12, resource.getModFile().getModifications().size());
+ Assert.assertFalse(resource.getCompactionModFile().exists());
+ }
+ }
+
private void generateModsFile(
List seriesPaths,
List resources,
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTask.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTask.java
index 5a9ffd9b21..fd32a78da8 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTask.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTask.java
@@ -58,7 +58,7 @@ public class FakedCrossSpaceCompactionTask extends RewriteCrossSpaceCompactionTa
((FakedTsFileResource) resource)
.setTsFileSize(resource.getTsFileSize() + avgSizeAddToSeqFile);
}
+ selectedSeqTsFileResourceList.clear();
selectedUnSeqTsFileResourceList.clear();
- // unSeqTsFileResourceList.clear();
}
}
diff --git a/server/src/test/resources/logback.xml b/server/src/test/resources/logback.xml
index 0ee9f5cc85..f7972b43a8 100644
--- a/server/src/test/resources/logback.xml
+++ b/server/src/test/resources/logback.xml
@@ -45,7 +45,7 @@
-
+
--
GitLab