Optimize the speed of delete by pattern (#10566)

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java

@@ -1851,14 +1851,26 @@ public class DataRegion implements IDataRegionForQuery {
       List<TsFileResource> sealedTsFileResource = new ArrayList<>();
       List<TsFileResource> unsealedTsFileResource = new ArrayList<>();
       separateTsFile(sealedTsFileResource, unsealedTsFileResource);
-
+      // deviceMatchInfo is used for filter the matched deviceId in TsFileResource
+      // deviceMatchInfo contains the DeviceId means this device matched the pattern
+      Set<String> deviceMatchInfo = new HashSet<>();
       deleteDataInFiles(
-          unsealedTsFileResource, deletion, devicePaths, updatedModFiles, timePartitionFilter);
+          unsealedTsFileResource,
+          deletion,
+          devicePaths,
+          updatedModFiles,
+          timePartitionFilter,
+          deviceMatchInfo);
       writeUnlock();
       hasReleasedLock = true;
 
       deleteDataInFiles(
-          sealedTsFileResource, deletion, devicePaths, updatedModFiles, timePartitionFilter);
+          sealedTsFileResource,
+          deletion,
+          devicePaths,
+          updatedModFiles,
+          timePartitionFilter,
+          deviceMatchInfo);
 
     } catch (Exception e) {
       // roll back
@@ -1916,16 +1928,32 @@ public class DataRegion implements IDataRegionForQuery {
       Set<PartialPath> devicePaths,
       long deleteStart,
       long deleteEnd,
-      TimePartitionFilter timePartitionFilter) {
+      TimePartitionFilter timePartitionFilter,
+      Set<String> deviceMatchInfo) {
     if (timePartitionFilter != null
         && !timePartitionFilter.satisfy(databaseName, tsFileResource.getTimePartition())) {
       return true;
     }
+    long fileStartTime = tsFileResource.getTimeIndex().getMinStartTime();
+    long fileEndTime = tsFileResource.getTimeIndex().getMaxEndTime();
 
     for (PartialPath device : devicePaths) {
       long deviceStartTime, deviceEndTime;
       if (device.hasWildcard()) {
-        Pair<Long, Long> startAndEndTime = tsFileResource.getPossibleStartTimeAndEndTime(device);
+        if (!tsFileResource.isClosed() && fileEndTime == Long.MIN_VALUE) {
+          // unsealed seq file
+          if (deleteEnd < fileStartTime) {
+            // time range of file has not overlapped with the deletion
+            return true;
+          }
+        } else {
+          if (deleteEnd < fileStartTime || deleteStart > fileEndTime) {
+            // time range of file has not overlapped with the deletion
+            return true;
+          }
+        }
+        Pair<Long, Long> startAndEndTime =
+            tsFileResource.getPossibleStartTimeAndEndTime(device, deviceMatchInfo);
         if (startAndEndTime == null) {
           continue;
         }
@@ -1962,7 +1990,8 @@ public class DataRegion implements IDataRegionForQuery {
       Deletion deletion,
       Set<PartialPath> devicePaths,
       List<ModificationFile> updatedModFiles,
-      TimePartitionFilter timePartitionFilter)
+      TimePartitionFilter timePartitionFilter,
+      Set<String> deviceMatchInfo)
       throws IOException {
     for (TsFileResource tsFileResource : tsFileResourceList) {
       if (canSkipDelete(
@@ -1970,7 +1999,8 @@ public class DataRegion implements IDataRegionForQuery {
           devicePaths,
           deletion.getStartTime(),
           deletion.getEndTime(),
-          timePartitionFilter)) {
+          timePartitionFilter,
+          deviceMatchInfo)) {
         continue;
       }
 

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java

@@ -503,8 +503,9 @@ public class TsFileResource {
    * Get the min start time and max end time of devices matched by given devicePattern. If there's
    * no device matched by given pattern, return null.
    */
-  public Pair<Long, Long> getPossibleStartTimeAndEndTime(PartialPath devicePattern) {
-    return timeIndex.getPossibleStartTimeAndEndTime(devicePattern);
+  public Pair<Long, Long> getPossibleStartTimeAndEndTime(
+      PartialPath devicePattern, Set<String> deviceMatchInfo) {
+    return timeIndex.getPossibleStartTimeAndEndTime(devicePattern, deviceMatchInfo);
   }
 
   public boolean isClosed() {

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java

@@ -23,6 +23,7 @@ import org.apache.iotdb.commons.exception.IllegalPathException;
 import org.apache.iotdb.commons.path.PartialPath;
 import org.apache.iotdb.commons.utils.SerializeUtils;
 import org.apache.iotdb.db.exception.PartitionViolationException;
+import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache;
 import org.apache.iotdb.db.storageengine.StorageEngine;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
 import org.apache.iotdb.tsfile.utils.FilePathUtils;
@@ -377,13 +378,14 @@ public class DeviceTimeIndex implements ITimeIndex {
   }
 
   @Override
-  public Pair<Long, Long> getPossibleStartTimeAndEndTime(PartialPath devicePattern) {
+  public Pair<Long, Long> getPossibleStartTimeAndEndTime(
+      PartialPath devicePattern, Set<String> deviceMatchInfo) {
     boolean hasMatchedDevice = false;
     long startTime = Long.MAX_VALUE;
     long endTime = Long.MIN_VALUE;
     for (Entry<String, Integer> entry : deviceToIndex.entrySet()) {
       try {
-        if (devicePattern.matchFullPath(new PartialPath(entry.getKey()))) {
+        if (deviceMatchInfo.contains(entry.getKey())) {
           hasMatchedDevice = true;
           if (startTimes[entry.getValue()] < startTime) {
             startTime = startTimes[entry.getValue()];
@@ -391,6 +393,18 @@ public class DeviceTimeIndex implements ITimeIndex {
           if (endTimes[entry.getValue()] > endTime) {
             endTime = endTimes[entry.getValue()];
           }
+        } else {
+          if (devicePattern.matchFullPath(
+              DataNodeDevicePathCache.getInstance().getPartialPath(entry.getKey()))) {
+            deviceMatchInfo.add(entry.getKey());
+            hasMatchedDevice = true;
+            if (startTimes[entry.getValue()] < startTime) {
+              startTime = startTimes[entry.getValue()];
+            }
+            if (endTimes[entry.getValue()] > endTime) {
+              endTime = endTimes[entry.getValue()];
+            }
+          }
         }
       } catch (IllegalPathException e) {
         // won't reach here

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java

@@ -234,7 +234,8 @@ public class FileTimeIndex implements ITimeIndex {
   }
 
   @Override
-  public Pair<Long, Long> getPossibleStartTimeAndEndTime(PartialPath devicePattern) {
+  public Pair<Long, Long> getPossibleStartTimeAndEndTime(
+      PartialPath devicePattern, Set<String> deviceMatchInfo) {
     return new Pair<>(startTime, endTime);
   }
 

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ITimeIndex.java

@@ -198,7 +198,8 @@ public interface ITimeIndex {
    */
   long[] getStartAndEndTime(String deviceId);
 
-  Pair<Long, Long> getPossibleStartTimeAndEndTime(PartialPath devicePattern);
+  Pair<Long, Long> getPossibleStartTimeAndEndTime(
+      PartialPath devicePattern, Set<String> deviceMatchInfo);
 
   /**
    * Get TimeIndex Type