Skip to content

I
Iotdb

apache / Iotdb
12 个月 前同步成功

通知 25
Star 3344
Fork 916
I
Iotdb

体验新版 GitCode，发现更多精彩内容 >>

未验证 提交 f9011617 编写于 作者: 周 周沛辰 提交者： GitHub
浏览文件
操作

[IOTDB-2468][Compaction]There is 'null' in logs (#4958)

上级 ed816e44
隐藏空白更改
内联 并排
Showing with 15 addition and 74 deletion
server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossCompactionStrategy.java
浏览文件 @ f9011617
......@@ -42,7 +42,6 @@ public enum CrossCompactionStrategy {
String logicalStorageGroupName,
String virtualStorageGroupName,
long timePartitionId,
String storageGroupDir,
TsFileManager tsFileManager,
List<TsFileResource> selectedSeqTsFileResourceList,
List<TsFileResource> selectedUnSeqTsFileResourceList) {
......@@ -53,7 +52,6 @@ public enum CrossCompactionStrategy {
logicalStorageGroupName,
virtualStorageGroupName,
timePartitionId,
storageGroupDir,
tsFileManager,
selectedSeqTsFileResourceList,
selectedUnSeqTsFileResourceList,
......@@ -65,7 +63,6 @@ public enum CrossCompactionStrategy {
String logicalStorageGroupName,
String virtualStorageGroupName,
long timePartitionId,
String storageGroupDir,
File logFile,
TsFileManager tsFileManager) {
switch (this) {
......@@ -75,7 +72,6 @@ public enum CrossCompactionStrategy {
logicalStorageGroupName,
virtualStorageGroupName,
timePartitionId,
storageGroupDir,
logFile,
CompactionTaskManager.currentTaskNum,
tsFileManager);
......
server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTaskFactory.java
浏览文件 @ f9011617
......@@ -23,7 +23,6 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask;
import org.apache.iotdb.db.engine.storagegroup.TsFileManager;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.engine.storagegroup.TsFileResourceList;
import java.util.List;
......@@ -33,10 +32,7 @@ public class CrossSpaceCompactionTaskFactory {
String logicalStorageGroupName,
String virtualStorageGroupName,
long timePartitionId,
String storageGroupDir,
TsFileManager tsFileManager,
TsFileResourceList seqTsFileResourceList,
TsFileResourceList unseqTsFileResourceList,
List<TsFileResource> selectedSeqTsFileResourceList,
List<TsFileResource> selectedUnSeqTsFileResourceList) {
return IoTDBDescriptor.getInstance()
......@@ -46,7 +42,6 @@ public class CrossSpaceCompactionTaskFactory {
logicalStorageGroupName,
virtualStorageGroupName,
timePartitionId,
storageGroupDir,
tsFileManager,
selectedSeqTsFileResourceList,
selectedUnSeqTsFileResourceList);
......
server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java
浏览文件 @ f9011617
......@@ -134,10 +134,7 @@ public class RewriteCrossSpaceCompactionSelector extends AbstractCrossSpaceCompa
logicalStorageGroupName,
virtualGroupId,
timePartition,
storageGroupDir,
tsFileManager,
sequenceFileList,
unsequenceFileList,
mergeFiles[0],
mergeFiles[1]);
CompactionTaskManager.getInstance().addTaskToWaitingQueue(compactionTask);
......
server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossCompactionRecoverTask.java
浏览文件 @ f9011617
......@@ -45,14 +45,11 @@ public class RewriteCrossCompactionRecoverTask extends RewriteCrossSpaceCompacti
private static final Logger LOGGER =
LoggerFactory.getLogger(RewriteCrossCompactionRecoverTask.class);
private File compactionLogFile;
private String logicalStorageGroupName;
private String virtualStorageGroup;
public RewriteCrossCompactionRecoverTask(
String logicalStorageGroupName,
String virtualStorageGroupName,
long timePartitionId,
String storageGroupDir,
File logFile,
AtomicInteger currentTaskNum,
TsFileManager tsFileManager) {
......@@ -60,14 +57,11 @@ public class RewriteCrossCompactionRecoverTask extends RewriteCrossSpaceCompacti
logicalStorageGroupName,
virtualStorageGroupName,
timePartitionId,
storageGroupDir,
tsFileManager,
null,
null,
currentTaskNum);
this.compactionLogFile = logFile;
this.logicalStorageGroupName = logicalStorageGroupName;
this.virtualStorageGroup = virtualStorageGroupName;
}
@Override
......@@ -80,9 +74,8 @@ public class RewriteCrossCompactionRecoverTask extends RewriteCrossSpaceCompacti
try {
if (compactionLogFile.exists()) {
LOGGER.info(
"{}-{} [Compaction][Recover] cross space compaction log file {} exists, start to recover it",
logicalStorageGroupName,
virtualStorageGroup,
"{} [Compaction][Recover] cross space compaction log file {} exists, start to recover it",
fullStorageGroupName,
compactionLogFile);
RewriteCrossSpaceCompactionLogAnalyzer logAnalyzer =
new RewriteCrossSpaceCompactionLogAnalyzer(compactionLogFile);
......@@ -93,9 +86,7 @@ public class RewriteCrossCompactionRecoverTask extends RewriteCrossSpaceCompacti
// compaction log file is incomplete
if (targetFileIdentifiers.isEmpty() || sourceFileIdentifiers.isEmpty()) {
LOGGER.info(
"{}-{} [Compaction][Recover] incomplete log file, abort recover",
logicalStorageGroupName,
virtualStorageGroup);
"{} [Compaction][Recover] incomplete log file, abort recover", fullStorageGroupName);
return;
}
......@@ -216,17 +207,15 @@ public class RewriteCrossCompactionRecoverTask extends RewriteCrossSpaceCompacti
getFileFromDataDirs(sourceFileIdentifier.getFilePath() + ModificationFile.FILE_SUFFIX);
if (compactionModFile != null && !compactionModFile.delete()) {
LOGGER.error(
"{}-{} [Compaction][Recover] fail to delete target file {}, this may cause data incorrectness",
logicalStorageGroupName,
virtualStorageGroup,
"{} [Compaction][Recover] fail to delete target file {}, this may cause data incorrectness",
fullStorageGroupName,
compactionModFile);
handleSuccess = false;
}
if (modFile != null && !modFile.delete()) {
LOGGER.error(
"{}-{} [Compaction][Recover] fail to delete target file {}, this may cause data incorrectness",
logicalStorageGroupName,
virtualStorageGroup,
"{} [Compaction][Recover] fail to delete target file {}, this may cause data incorrectness",
fullStorageGroupName,
modFile);
handleSuccess = false;
}
......@@ -235,9 +224,7 @@ public class RewriteCrossCompactionRecoverTask extends RewriteCrossSpaceCompacti
if (!InnerSpaceCompactionUtils.deleteTsFilesInDisk(
remainSourceTsFileResources, fullStorageGroupName)) {
LOGGER.error(
"{}-{} [Compaction][Recover] fail to delete remaining source files.",
logicalStorageGroupName,
virtualStorageGroup);
"{} [Compaction][Recover] fail to delete remaining source files.", fullStorageGroupName);
handleSuccess = false;
}
return handleSuccess;
......
server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java
浏览文件 @ f9011617
......@@ -54,11 +54,8 @@ import static org.apache.iotdb.db.engine.compaction.cross.rewrite.recover.Rewrit
public class RewriteCrossSpaceCompactionTask extends AbstractCrossSpaceCompactionTask {
private static final Logger logger = LoggerFactory.getLogger("COMPACTION");
protected String storageGroupDir;
protected List<TsFileResource> selectedSeqTsFileResourceList;
protected List<TsFileResource> selectedUnSeqTsFileResourceList;
protected String logicalStorageGroupName;
protected String virtualStorageGroupName;
protected TsFileManager tsFileManager;
private File logFile;
......@@ -69,13 +66,10 @@ public class RewriteCrossSpaceCompactionTask extends AbstractCrossSpaceCompactio
private final long ACQUIRE_WRITE_LOCK_TIMEOUT =
IoTDBDescriptor.getInstance().getConfig().getCompactionAcquireWriteLockTimeout();
String storageGroupName;
public RewriteCrossSpaceCompactionTask(
String logicalStorageGroupName,
String virtualStorageGroupName,
long timePartitionId,
String storageGroupDir,
TsFileManager tsFileManager,
List<TsFileResource> selectedSeqTsFileResourceList,
List<TsFileResource> selectedUnSeqTsFileResourceList,
......@@ -86,9 +80,6 @@ public class RewriteCrossSpaceCompactionTask extends AbstractCrossSpaceCompactio
currentTaskNum,
selectedSeqTsFileResourceList,
selectedUnSeqTsFileResourceList);
this.logicalStorageGroupName = logicalStorageGroupName;
this.virtualStorageGroupName = virtualStorageGroupName;
this.storageGroupDir = storageGroupDir;
this.selectedSeqTsFileResourceList = selectedSeqTsFileResourceList;
this.selectedUnSeqTsFileResourceList = selectedUnSeqTsFileResourceList;
this.tsFileManager = tsFileManager;
......@@ -101,7 +92,7 @@ public class RewriteCrossSpaceCompactionTask extends AbstractCrossSpaceCompactio
} catch (Throwable throwable) {
// catch throwable instead of exception to handle OOM errors
CrossSpaceCompactionExceptionHandler.handleException(
storageGroupName,
fullStorageGroupName,
logFile,
targetTsfileResourceList,
selectedSeqTsFileResourceList,
......@@ -128,13 +119,14 @@ public class RewriteCrossSpaceCompactionTask extends AbstractCrossSpaceCompactio
|| selectedSeqTsFileResourceList.isEmpty()
|| selectedUnSeqTsFileResourceList.isEmpty()) {
logger.info(
"{} [Compaction] Cross space compaction file list is empty, end it", storageGroupName);
"{} [Compaction] Cross space compaction file list is empty, end it",
fullStorageGroupName);
return;
}
logger.info(
"{} [Compaction] CrossSpaceCompactionTask start. Sequence files : {}, unsequence files : {}",
storageGroupName,
fullStorageGroupName,
selectedSeqTsFileResourceList,
selectedUnSeqTsFileResourceList);
logFile =
......@@ -154,7 +146,7 @@ public class RewriteCrossSpaceCompactionTask extends AbstractCrossSpaceCompactio
CompactionUtils.compact(
selectedSeqTsFileResourceList, selectedUnSeqTsFileResourceList, targetTsfileResourceList);
CompactionUtils.moveTargetFile(targetTsfileResourceList, false, storageGroupName);
CompactionUtils.moveTargetFile(targetTsfileResourceList, false, fullStorageGroupName);
// indicates that the cross compaction is complete and the result can be reused during a
// restart recovery
......@@ -193,7 +185,7 @@ public class RewriteCrossSpaceCompactionTask extends AbstractCrossSpaceCompactio
}
logger.info(
"{} [Compaction] CrossSpaceCompactionTask Costs {} s",
storageGroupName,
fullStorageGroupName,
(System.currentTimeMillis() - startTime) / 1000);
}
}
......@@ -266,7 +258,7 @@ public class RewriteCrossSpaceCompactionTask extends AbstractCrossSpaceCompactio
}
public String getStorageGroupName() {
return storageGroupName;
return fullStorageGroupName;
}
private void removeCompactionModification() {
......
server/src/main/java/org/apache/iotdb/db/engine/compaction/task/CompactionRecoverTask.java
浏览文件 @ f9011617
......@@ -72,7 +72,6 @@ public class CompactionRecoverTask {
logicalStorageGroupName,
virtualStorageGroupId,
timePartition,
storageGroupDir,
compactionLog,
tsFileManager)
.call();
......
server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTest.java
浏览文件 @ f9011617
......@@ -429,13 +429,10 @@ public class CrossSpaceCompactionTest {
COMPACTION_TEST_SG,
"0",
0,
"target",
new TsFileManager(
"root.compactionTest",
"0",
"target\\data\\sequence\\test\\root.compactionTest\\0\\0\\"),
seqTsFileResourceList,
unseqTsFileResourceList,
mergeResource.getSeqFiles(),
mergeResource.getUnseqFiles());
compactionTask.call();
......@@ -735,13 +732,10 @@ public class CrossSpaceCompactionTest {
COMPACTION_TEST_SG,
"0",
0,
"target",
new TsFileManager(
"root.compactionTest",
"0",
"target\\data\\sequence\\test\\root.compactionTest\\0\\0\\"),
seqTsFileResourceList,
unseqTsFileResourceList,
mergeResource.getSeqFiles(),
mergeResource.getUnseqFiles());
compactionTask.call();
......@@ -1040,13 +1034,10 @@ public class CrossSpaceCompactionTest {
COMPACTION_TEST_SG,
"0",
0,
"target",
new TsFileManager(
"root.compactionTest",
"0",
"target\\data\\sequence\\test\\root.compactionTest\\0\\0\\"),
seqTsFileResourceList,
unseqTsFileResourceList,
mergeResource.getSeqFiles(),
mergeResource.getUnseqFiles());
compactionTask.call();
......
server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionRecoverTest.java
浏览文件 @ f9011617
......@@ -100,7 +100,6 @@ public class RewriteCrossSpaceCompactionRecoverTest extends AbstractCompactionTe
COMPACTION_LOG_NAME,
"0",
0,
SEQ_DIRS.getPath(),
compactionLogFile,
CompactionTaskManager.currentTaskNum,
tsFileManager)
......@@ -171,7 +170,6 @@ public class RewriteCrossSpaceCompactionRecoverTest extends AbstractCompactionTe
COMPACTION_LOG_NAME,
"0",
0,
SEQ_DIRS.getPath(),
compactionLogFile,
CompactionTaskManager.currentTaskNum,
tsFileManager)
......@@ -243,7 +241,6 @@ public class RewriteCrossSpaceCompactionRecoverTest extends AbstractCompactionTe
COMPACTION_LOG_NAME,
"0",
0,
SEQ_DIRS.getPath(),
compactionLogFile,
CompactionTaskManager.currentTaskNum,
tsFileManager)
......@@ -334,7 +331,6 @@ public class RewriteCrossSpaceCompactionRecoverTest extends AbstractCompactionTe
COMPACTION_LOG_NAME,
"0",
0,
SEQ_DIRS.getPath(),
compactionLogFile,
CompactionTaskManager.currentTaskNum,
tsFileManager)
......@@ -438,7 +434,6 @@ public class RewriteCrossSpaceCompactionRecoverTest extends AbstractCompactionTe
COMPACTION_LOG_NAME,
"0",
0,
SEQ_DIRS.getPath(),
compactionLogFile,
CompactionTaskManager.currentTaskNum,
tsFileManager)
......
server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java
浏览文件 @ f9011617
......@@ -224,7 +224,6 @@ public class RewriteCrossSpaceCompactionTest extends AbstractCompactionTest {
COMPACTION_TEST_SG,
"0",
0,
STORAGE_GROUP_DIR.getPath(),
tsFileManager,
seqResources,
unseqResources,
......@@ -462,7 +461,6 @@ public class RewriteCrossSpaceCompactionTest extends AbstractCompactionTest {
COMPACTION_TEST_SG,
"0",
0,
STORAGE_GROUP_DIR.getPath(),
tsFileManager,
seqResources,
unseqResources,
......@@ -610,7 +608,6 @@ public class RewriteCrossSpaceCompactionTest extends AbstractCompactionTest {
COMPACTION_TEST_SG,
"0",
0,
STORAGE_GROUP_DIR.getPath(),
vsgp.getTsFileResourceManager(),
seqResources,
unseqResources,
......@@ -720,7 +717,6 @@ public class RewriteCrossSpaceCompactionTest extends AbstractCompactionTest {
COMPACTION_TEST_SG,
"0",
0,
STORAGE_GROUP_DIR.getPath(),
vsgp.getTsFileResourceManager(),
seqResources,
unseqResources,
......
server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTask.java
浏览文件 @ f9011617
......@@ -32,7 +32,6 @@ public class FakedCrossSpaceCompactionTask extends RewriteCrossSpaceCompactionTa
String logicalStorageGroupName,
String virtualStorageGroupName,
long timePartitionId,
String storageGroupDir,
TsFileManager tsFileManager,
List<TsFileResource> selectedSeqTsFileResourceList,
List<TsFileResource> selectedUnSeqTsFileResourceList) {
......@@ -40,7 +39,6 @@ public class FakedCrossSpaceCompactionTask extends RewriteCrossSpaceCompactionTa
logicalStorageGroupName,
virtualStorageGroupName,
timePartitionId,
storageGroupDir,
tsFileManager,
selectedSeqTsFileResourceList,
selectedUnSeqTsFileResourceList,
......
server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTaskFactory.java
浏览文件 @ f9011617
......@@ -22,7 +22,6 @@ package org.apache.iotdb.db.engine.compaction.task;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.storagegroup.TsFileManager;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.engine.storagegroup.TsFileResourceList;
import java.util.List;
......@@ -31,10 +30,7 @@ public class FakedCrossSpaceCompactionTaskFactory {
String logicalStorageGroupName,
String virtualStorageGroupName,
long timePartitionId,
String storageGroupDir,
TsFileManager tsFileManager,
TsFileResourceList seqTsFileResourceList,
TsFileResourceList unseqTsFileResourceList,
List<TsFileResource> selectedSeqTsFileResourceList,
List<TsFileResource> selectedUnSeqTsFileResourceList) {
return IoTDBDescriptor.getInstance()
......@@ -44,7 +40,6 @@ public class FakedCrossSpaceCompactionTaskFactory {
logicalStorageGroupName,
virtualStorageGroupName,
timePartitionId,
storageGroupDir,
tsFileManager,
selectedSeqTsFileResourceList,
selectedUnSeqTsFileResourceList);
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或