From f9011617315ac9a7e2516597eb773b102f0d84d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E6=B2=9B=E8=BE=B0?= <45144903+choubenson@users.noreply.github.com> Date: Sat, 22 Jan 2022 16:10:52 +0800 Subject: [PATCH] [IOTDB-2468][Compaction]There is 'null' in logs (#4958) --- .../cross/CrossCompactionStrategy.java | 4 --- .../CrossSpaceCompactionTaskFactory.java | 5 ---- .../RewriteCrossSpaceCompactionSelector.java | 3 -- .../RewriteCrossCompactionRecoverTask.java | 29 +++++-------------- .../task/RewriteCrossSpaceCompactionTask.java | 22 +++++--------- .../task/CompactionRecoverTask.java | 1 - .../cross/CrossSpaceCompactionTest.java | 9 ------ ...ewriteCrossSpaceCompactionRecoverTest.java | 5 ---- .../RewriteCrossSpaceCompactionTest.java | 4 --- .../task/FakedCrossSpaceCompactionTask.java | 2 -- .../FakedCrossSpaceCompactionTaskFactory.java | 5 ---- 11 files changed, 15 insertions(+), 74 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossCompactionStrategy.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossCompactionStrategy.java index 9b1dd73c93..a957961445 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossCompactionStrategy.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossCompactionStrategy.java @@ -42,7 +42,6 @@ public enum CrossCompactionStrategy { String logicalStorageGroupName, String virtualStorageGroupName, long timePartitionId, - String storageGroupDir, TsFileManager tsFileManager, List selectedSeqTsFileResourceList, List selectedUnSeqTsFileResourceList) { @@ -53,7 +52,6 @@ public enum CrossCompactionStrategy { logicalStorageGroupName, virtualStorageGroupName, timePartitionId, - storageGroupDir, tsFileManager, selectedSeqTsFileResourceList, selectedUnSeqTsFileResourceList, @@ -65,7 +63,6 @@ public enum CrossCompactionStrategy { String logicalStorageGroupName, String virtualStorageGroupName, long timePartitionId, - String storageGroupDir, File logFile, TsFileManager tsFileManager) { switch (this) { @@ -75,7 +72,6 @@ public enum CrossCompactionStrategy { logicalStorageGroupName, virtualStorageGroupName, timePartitionId, - storageGroupDir, logFile, CompactionTaskManager.currentTaskNum, tsFileManager); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTaskFactory.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTaskFactory.java index d2617a9f0a..3c98410ef3 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTaskFactory.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTaskFactory.java @@ -23,7 +23,6 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; -import org.apache.iotdb.db.engine.storagegroup.TsFileResourceList; import java.util.List; @@ -33,10 +32,7 @@ public class CrossSpaceCompactionTaskFactory { String logicalStorageGroupName, String virtualStorageGroupName, long timePartitionId, - String storageGroupDir, TsFileManager tsFileManager, - TsFileResourceList seqTsFileResourceList, - TsFileResourceList unseqTsFileResourceList, List selectedSeqTsFileResourceList, List selectedUnSeqTsFileResourceList) { return IoTDBDescriptor.getInstance() @@ -46,7 +42,6 @@ public class CrossSpaceCompactionTaskFactory { logicalStorageGroupName, virtualStorageGroupName, timePartitionId, - storageGroupDir, tsFileManager, selectedSeqTsFileResourceList, selectedUnSeqTsFileResourceList); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java index 79666a0cc2..455f3e8e73 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java @@ -134,10 +134,7 @@ public class RewriteCrossSpaceCompactionSelector extends AbstractCrossSpaceCompa logicalStorageGroupName, virtualGroupId, timePartition, - storageGroupDir, tsFileManager, - sequenceFileList, - unsequenceFileList, mergeFiles[0], mergeFiles[1]); CompactionTaskManager.getInstance().addTaskToWaitingQueue(compactionTask); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossCompactionRecoverTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossCompactionRecoverTask.java index ffbc6a1492..2f69c676cd 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossCompactionRecoverTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossCompactionRecoverTask.java @@ -45,14 +45,11 @@ public class RewriteCrossCompactionRecoverTask extends RewriteCrossSpaceCompacti private static final Logger LOGGER = LoggerFactory.getLogger(RewriteCrossCompactionRecoverTask.class); private File compactionLogFile; - private String logicalStorageGroupName; - private String virtualStorageGroup; public RewriteCrossCompactionRecoverTask( String logicalStorageGroupName, String virtualStorageGroupName, long timePartitionId, - String storageGroupDir, File logFile, AtomicInteger currentTaskNum, TsFileManager tsFileManager) { @@ -60,14 +57,11 @@ public class RewriteCrossCompactionRecoverTask extends RewriteCrossSpaceCompacti logicalStorageGroupName, virtualStorageGroupName, timePartitionId, - storageGroupDir, tsFileManager, null, null, currentTaskNum); this.compactionLogFile = logFile; - this.logicalStorageGroupName = logicalStorageGroupName; - this.virtualStorageGroup = virtualStorageGroupName; } @Override @@ -80,9 +74,8 @@ public class RewriteCrossCompactionRecoverTask extends RewriteCrossSpaceCompacti try { if (compactionLogFile.exists()) { LOGGER.info( - "{}-{} [Compaction][Recover] cross space compaction log file {} exists, start to recover it", - logicalStorageGroupName, - virtualStorageGroup, + "{} [Compaction][Recover] cross space compaction log file {} exists, start to recover it", + fullStorageGroupName, compactionLogFile); RewriteCrossSpaceCompactionLogAnalyzer logAnalyzer = new RewriteCrossSpaceCompactionLogAnalyzer(compactionLogFile); @@ -93,9 +86,7 @@ public class RewriteCrossCompactionRecoverTask extends RewriteCrossSpaceCompacti // compaction log file is incomplete if (targetFileIdentifiers.isEmpty() || sourceFileIdentifiers.isEmpty()) { LOGGER.info( - "{}-{} [Compaction][Recover] incomplete log file, abort recover", - logicalStorageGroupName, - virtualStorageGroup); + "{} [Compaction][Recover] incomplete log file, abort recover", fullStorageGroupName); return; } @@ -216,17 +207,15 @@ public class RewriteCrossCompactionRecoverTask extends RewriteCrossSpaceCompacti getFileFromDataDirs(sourceFileIdentifier.getFilePath() + ModificationFile.FILE_SUFFIX); if (compactionModFile != null && !compactionModFile.delete()) { LOGGER.error( - "{}-{} [Compaction][Recover] fail to delete target file {}, this may cause data incorrectness", - logicalStorageGroupName, - virtualStorageGroup, + "{} [Compaction][Recover] fail to delete target file {}, this may cause data incorrectness", + fullStorageGroupName, compactionModFile); handleSuccess = false; } if (modFile != null && !modFile.delete()) { LOGGER.error( - "{}-{} [Compaction][Recover] fail to delete target file {}, this may cause data incorrectness", - logicalStorageGroupName, - virtualStorageGroup, + "{} [Compaction][Recover] fail to delete target file {}, this may cause data incorrectness", + fullStorageGroupName, modFile); handleSuccess = false; } @@ -235,9 +224,7 @@ public class RewriteCrossCompactionRecoverTask extends RewriteCrossSpaceCompacti if (!InnerSpaceCompactionUtils.deleteTsFilesInDisk( remainSourceTsFileResources, fullStorageGroupName)) { LOGGER.error( - "{}-{} [Compaction][Recover] fail to delete remaining source files.", - logicalStorageGroupName, - virtualStorageGroup); + "{} [Compaction][Recover] fail to delete remaining source files.", fullStorageGroupName); handleSuccess = false; } return handleSuccess; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java index f81341aacd..96459cf34e 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java @@ -54,11 +54,8 @@ import static org.apache.iotdb.db.engine.compaction.cross.rewrite.recover.Rewrit public class RewriteCrossSpaceCompactionTask extends AbstractCrossSpaceCompactionTask { private static final Logger logger = LoggerFactory.getLogger("COMPACTION"); - protected String storageGroupDir; protected List selectedSeqTsFileResourceList; protected List selectedUnSeqTsFileResourceList; - protected String logicalStorageGroupName; - protected String virtualStorageGroupName; protected TsFileManager tsFileManager; private File logFile; @@ -69,13 +66,10 @@ public class RewriteCrossSpaceCompactionTask extends AbstractCrossSpaceCompactio private final long ACQUIRE_WRITE_LOCK_TIMEOUT = IoTDBDescriptor.getInstance().getConfig().getCompactionAcquireWriteLockTimeout(); - String storageGroupName; - public RewriteCrossSpaceCompactionTask( String logicalStorageGroupName, String virtualStorageGroupName, long timePartitionId, - String storageGroupDir, TsFileManager tsFileManager, List selectedSeqTsFileResourceList, List selectedUnSeqTsFileResourceList, @@ -86,9 +80,6 @@ public class RewriteCrossSpaceCompactionTask extends AbstractCrossSpaceCompactio currentTaskNum, selectedSeqTsFileResourceList, selectedUnSeqTsFileResourceList); - this.logicalStorageGroupName = logicalStorageGroupName; - this.virtualStorageGroupName = virtualStorageGroupName; - this.storageGroupDir = storageGroupDir; this.selectedSeqTsFileResourceList = selectedSeqTsFileResourceList; this.selectedUnSeqTsFileResourceList = selectedUnSeqTsFileResourceList; this.tsFileManager = tsFileManager; @@ -101,7 +92,7 @@ public class RewriteCrossSpaceCompactionTask extends AbstractCrossSpaceCompactio } catch (Throwable throwable) { // catch throwable instead of exception to handle OOM errors CrossSpaceCompactionExceptionHandler.handleException( - storageGroupName, + fullStorageGroupName, logFile, targetTsfileResourceList, selectedSeqTsFileResourceList, @@ -128,13 +119,14 @@ public class RewriteCrossSpaceCompactionTask extends AbstractCrossSpaceCompactio || selectedSeqTsFileResourceList.isEmpty() || selectedUnSeqTsFileResourceList.isEmpty()) { logger.info( - "{} [Compaction] Cross space compaction file list is empty, end it", storageGroupName); + "{} [Compaction] Cross space compaction file list is empty, end it", + fullStorageGroupName); return; } logger.info( "{} [Compaction] CrossSpaceCompactionTask start. Sequence files : {}, unsequence files : {}", - storageGroupName, + fullStorageGroupName, selectedSeqTsFileResourceList, selectedUnSeqTsFileResourceList); logFile = @@ -154,7 +146,7 @@ public class RewriteCrossSpaceCompactionTask extends AbstractCrossSpaceCompactio CompactionUtils.compact( selectedSeqTsFileResourceList, selectedUnSeqTsFileResourceList, targetTsfileResourceList); - CompactionUtils.moveTargetFile(targetTsfileResourceList, false, storageGroupName); + CompactionUtils.moveTargetFile(targetTsfileResourceList, false, fullStorageGroupName); // indicates that the cross compaction is complete and the result can be reused during a // restart recovery @@ -193,7 +185,7 @@ public class RewriteCrossSpaceCompactionTask extends AbstractCrossSpaceCompactio } logger.info( "{} [Compaction] CrossSpaceCompactionTask Costs {} s", - storageGroupName, + fullStorageGroupName, (System.currentTimeMillis() - startTime) / 1000); } } @@ -266,7 +258,7 @@ public class RewriteCrossSpaceCompactionTask extends AbstractCrossSpaceCompactio } public String getStorageGroupName() { - return storageGroupName; + return fullStorageGroupName; } private void removeCompactionModification() { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/CompactionRecoverTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/CompactionRecoverTask.java index c0f5c65417..3e367eddaf 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/CompactionRecoverTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/CompactionRecoverTask.java @@ -72,7 +72,6 @@ public class CompactionRecoverTask { logicalStorageGroupName, virtualStorageGroupId, timePartition, - storageGroupDir, compactionLog, tsFileManager) .call(); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTest.java index 9726d3f535..1fc248cebe 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTest.java @@ -429,13 +429,10 @@ public class CrossSpaceCompactionTest { COMPACTION_TEST_SG, "0", 0, - "target", new TsFileManager( "root.compactionTest", "0", "target\\data\\sequence\\test\\root.compactionTest\\0\\0\\"), - seqTsFileResourceList, - unseqTsFileResourceList, mergeResource.getSeqFiles(), mergeResource.getUnseqFiles()); compactionTask.call(); @@ -735,13 +732,10 @@ public class CrossSpaceCompactionTest { COMPACTION_TEST_SG, "0", 0, - "target", new TsFileManager( "root.compactionTest", "0", "target\\data\\sequence\\test\\root.compactionTest\\0\\0\\"), - seqTsFileResourceList, - unseqTsFileResourceList, mergeResource.getSeqFiles(), mergeResource.getUnseqFiles()); compactionTask.call(); @@ -1040,13 +1034,10 @@ public class CrossSpaceCompactionTest { COMPACTION_TEST_SG, "0", 0, - "target", new TsFileManager( "root.compactionTest", "0", "target\\data\\sequence\\test\\root.compactionTest\\0\\0\\"), - seqTsFileResourceList, - unseqTsFileResourceList, mergeResource.getSeqFiles(), mergeResource.getUnseqFiles()); compactionTask.call(); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionRecoverTest.java index 8e0080b9f9..bdbbc0b3c8 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionRecoverTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionRecoverTest.java @@ -100,7 +100,6 @@ public class RewriteCrossSpaceCompactionRecoverTest extends AbstractCompactionTe COMPACTION_LOG_NAME, "0", 0, - SEQ_DIRS.getPath(), compactionLogFile, CompactionTaskManager.currentTaskNum, tsFileManager) @@ -171,7 +170,6 @@ public class RewriteCrossSpaceCompactionRecoverTest extends AbstractCompactionTe COMPACTION_LOG_NAME, "0", 0, - SEQ_DIRS.getPath(), compactionLogFile, CompactionTaskManager.currentTaskNum, tsFileManager) @@ -243,7 +241,6 @@ public class RewriteCrossSpaceCompactionRecoverTest extends AbstractCompactionTe COMPACTION_LOG_NAME, "0", 0, - SEQ_DIRS.getPath(), compactionLogFile, CompactionTaskManager.currentTaskNum, tsFileManager) @@ -334,7 +331,6 @@ public class RewriteCrossSpaceCompactionRecoverTest extends AbstractCompactionTe COMPACTION_LOG_NAME, "0", 0, - SEQ_DIRS.getPath(), compactionLogFile, CompactionTaskManager.currentTaskNum, tsFileManager) @@ -438,7 +434,6 @@ public class RewriteCrossSpaceCompactionRecoverTest extends AbstractCompactionTe COMPACTION_LOG_NAME, "0", 0, - SEQ_DIRS.getPath(), compactionLogFile, CompactionTaskManager.currentTaskNum, tsFileManager) diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java index aaa8d82fef..6e2de99522 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java @@ -224,7 +224,6 @@ public class RewriteCrossSpaceCompactionTest extends AbstractCompactionTest { COMPACTION_TEST_SG, "0", 0, - STORAGE_GROUP_DIR.getPath(), tsFileManager, seqResources, unseqResources, @@ -462,7 +461,6 @@ public class RewriteCrossSpaceCompactionTest extends AbstractCompactionTest { COMPACTION_TEST_SG, "0", 0, - STORAGE_GROUP_DIR.getPath(), tsFileManager, seqResources, unseqResources, @@ -610,7 +608,6 @@ public class RewriteCrossSpaceCompactionTest extends AbstractCompactionTest { COMPACTION_TEST_SG, "0", 0, - STORAGE_GROUP_DIR.getPath(), vsgp.getTsFileResourceManager(), seqResources, unseqResources, @@ -720,7 +717,6 @@ public class RewriteCrossSpaceCompactionTest extends AbstractCompactionTest { COMPACTION_TEST_SG, "0", 0, - STORAGE_GROUP_DIR.getPath(), vsgp.getTsFileResourceManager(), seqResources, unseqResources, diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTask.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTask.java index fd32a78da8..315ffc51e3 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTask.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTask.java @@ -32,7 +32,6 @@ public class FakedCrossSpaceCompactionTask extends RewriteCrossSpaceCompactionTa String logicalStorageGroupName, String virtualStorageGroupName, long timePartitionId, - String storageGroupDir, TsFileManager tsFileManager, List selectedSeqTsFileResourceList, List selectedUnSeqTsFileResourceList) { @@ -40,7 +39,6 @@ public class FakedCrossSpaceCompactionTask extends RewriteCrossSpaceCompactionTa logicalStorageGroupName, virtualStorageGroupName, timePartitionId, - storageGroupDir, tsFileManager, selectedSeqTsFileResourceList, selectedUnSeqTsFileResourceList, diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTaskFactory.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTaskFactory.java index 910cd3a883..1fb4b2e3fc 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTaskFactory.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTaskFactory.java @@ -22,7 +22,6 @@ package org.apache.iotdb.db.engine.compaction.task; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; -import org.apache.iotdb.db.engine.storagegroup.TsFileResourceList; import java.util.List; @@ -31,10 +30,7 @@ public class FakedCrossSpaceCompactionTaskFactory { String logicalStorageGroupName, String virtualStorageGroupName, long timePartitionId, - String storageGroupDir, TsFileManager tsFileManager, - TsFileResourceList seqTsFileResourceList, - TsFileResourceList unseqTsFileResourceList, List selectedSeqTsFileResourceList, List selectedUnSeqTsFileResourceList) { return IoTDBDescriptor.getInstance() @@ -44,7 +40,6 @@ public class FakedCrossSpaceCompactionTaskFactory { logicalStorageGroupName, virtualStorageGroupName, timePartitionId, - storageGroupDir, tsFileManager, selectedSeqTsFileResourceList, selectedUnSeqTsFileResourceList); -- GitLab