Add UT for DirectoriesStrategy (#197)

* add ut for DirectoriesStrategy

iotdb/pom.xml

@@ -70,6 +70,25 @@
             <artifactId>commons-lang3</artifactId>
             <version>${common.lang3.version}</version>
         </dependency>
+        <!-- for mocked test-->
+        <dependency>
+            <groupId>org.powermock</groupId>
+            <artifactId>powermock-core</artifactId>
+            <version>2.0.2</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.powermock</groupId>
+            <artifactId>powermock-module-junit4</artifactId>
+            <version>2.0.2</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.powermock</groupId>
+            <artifactId>powermock-api-mockito2</artifactId>
+            <version>2.0.2</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
     <build>
         <plugins>

iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategy.java

@@ -18,15 +18,17 @@
  */
 package org.apache.iotdb.db.conf.directories.strategy;
 
-import java.io.File;
 import java.util.List;
 import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.db.utils.CommonUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
  * The basic class of all the strategies of multiple directories. If a user wants to define his own
  * strategy, his strategy has to extend this class and implement the abstract method.
+ *
+ * @author East
  */
 public abstract class DirectoryStrategy {
 
@@ -39,11 +41,24 @@ public abstract class DirectoryStrategy {
 
   /**
    * To init folders. Do not recommend to overwrite.
+   * This method guarantees that at least one folder has available space.
    *
    * @param folders the folders from conf
    */
   public void init(List<String> folders) throws DiskSpaceInsufficientException {
     this.folders = folders;
+
+    boolean hasSpace = false;
+    for (String folder : folders) {
+      if (CommonUtils.hasSpace(folder)) {
+        hasSpace = true;
+        break;
+      }
+    }
+    if (!hasSpace) {
+      throw new DiskSpaceInsufficientException(
+          String.format("All disks of folders %s are full, can't init.", folders));
+    }
   }
 
   /**
@@ -72,18 +87,4 @@ public abstract class DirectoryStrategy {
   public void setFolderForTest(String path) {
     folders.set(0, path);
   }
-
-  protected long getUsableSpace(String dir) {
-    File file = new File(dir);
-    if (!file.exists() && !file.mkdirs()) {
-      return 0;
-    }
-    long space = file.getFreeSpace();
-    logger.trace("Folder {} has {} available bytes.", dir, space);
-    return space;
-  }
-
-  protected boolean hasSpace(String dir) {
-    return getUsableSpace(dir) > 0;
-  }
 }

iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/MaxDiskUsableSpaceFirstStrategy.java

@@ -19,6 +19,7 @@
 package org.apache.iotdb.db.conf.directories.strategy;
 
 import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.db.utils.CommonUtils;
 
 public class MaxDiskUsableSpaceFirstStrategy extends DirectoryStrategy {
 
@@ -35,7 +36,12 @@ public class MaxDiskUsableSpaceFirstStrategy extends DirectoryStrategy {
     long maxSpace = 0;
 
     for (int i = 0; i < folders.size(); i++) {
-      long space = getUsableSpace(folders.get(i));
+      String folder = folders.get(i);
+      if (!CommonUtils.hasSpace(folder)) {
+        continue;
+      }
+
+      long space = CommonUtils.getUsableSpace(folder);
       if (space > maxSpace) {
         maxSpace = space;
         maxIndex = i;

iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/MinFolderOccupiedSpaceFirstStrategy.java

@@ -19,17 +19,11 @@
 package org.apache.iotdb.db.conf.directories.strategy;
 
 import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.stream.Stream;
 import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.db.utils.CommonUtils;
 
 public class MinFolderOccupiedSpaceFirstStrategy extends DirectoryStrategy {
 
-  // directory space is measured by MB
-  private static final long DATA_SIZE_SHIFT = 1024L * 1024;
-
   @Override
   public int nextFolderIndex() throws DiskSpaceInsufficientException {
     return getMinOccupiedSpaceFolder();
@@ -37,15 +31,20 @@ public class MinFolderOccupiedSpaceFirstStrategy extends DirectoryStrategy {
 
   private int getMinOccupiedSpaceFolder() throws DiskSpaceInsufficientException {
     int minIndex = -1;
-    long minSpace = Integer.MAX_VALUE;
+    long minSpace = Long.MAX_VALUE;
 
     for (int i = 0; i < folders.size(); i++) {
       String folder = folders.get(i);
-      if (!hasSpace(folder)) {
+      if (!CommonUtils.hasSpace(folder)) {
         continue;
       }
 
-      long space = getOccupiedSpace(folder);
+      long space = 0;
+      try {
+        space = CommonUtils.getOccupiedSpace(folder);
+      } catch (IOException e) {
+        logger.error("Cannot calculate occupied space for path {}.", folder, e);
+      }
       if (space < minSpace) {
         minSpace = space;
         minIndex = i;
@@ -58,20 +57,4 @@ public class MinFolderOccupiedSpaceFirstStrategy extends DirectoryStrategy {
 
     return minIndex;
   }
-
-  private long getOccupiedSpace(String path) {
-    Path folder = Paths.get(path);
-    long size = Long.MAX_VALUE;
-    try {
-      try (Stream<Path> stream = Files.walk(folder)) {
-        size = stream.filter(p -> p.toFile().isFile())
-            .mapToLong(p -> p.toFile().length())
-            .sum();
-      }
-    } catch (IOException e) {
-      logger.error("Cannot calculate occupied space for seriesPath {}.", path, e);
-    }
-
-    return size / DATA_SIZE_SHIFT;
-  }
 }

iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/SequenceStrategy.java

@@ -20,6 +20,7 @@ package org.apache.iotdb.db.conf.directories.strategy;
 
 import java.util.List;
 import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.db.utils.CommonUtils;
 
 public class SequenceStrategy extends DirectoryStrategy {
 
@@ -29,18 +30,15 @@ public class SequenceStrategy extends DirectoryStrategy {
   public void init(List<String> folders) throws DiskSpaceInsufficientException {
     super.init(folders);
 
+    // super.init() ensures at least one folder is not full,
+    // so currentIndex will not be -1 after loop
     currentIndex = -1;
     for (int i = 0; i < folders.size(); i++) {
-      if (hasSpace(folders.get(i))) {
+      if (CommonUtils.hasSpace(folders.get(i))) {
         currentIndex = i;
         break;
       }
     }
-
-    if (currentIndex == -1) {
-      throw new DiskSpaceInsufficientException(
-          String.format("All disks of folders %s are full, can't init.", folders));
-    }
   }
 
   @Override
@@ -53,7 +51,7 @@ public class SequenceStrategy extends DirectoryStrategy {
 
   private int tryGetNextIndex(int start) throws DiskSpaceInsufficientException {
     int index = (start + 1) % folders.size();
-    while (!hasSpace(folders.get(index))) {
+    while (!CommonUtils.hasSpace(folders.get(index))) {
       index = (index + 1) % folders.size();
       if (index == start) {
         throw new DiskSpaceInsufficientException(folders);

iotdb/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java

@@ -18,6 +18,12 @@
  */
 package org.apache.iotdb.db.utils;
 
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
 public class CommonUtils {
 
   private CommonUtils(){}
@@ -35,4 +41,18 @@ public class CommonUtils {
       return Integer.parseInt(javaVersionElements[0]);
     }
   }
+
+  public static long getUsableSpace(String dir) {
+    return new File(dir).getFreeSpace();
+  }
+
+  public static boolean hasSpace(String dir) {
+    return getUsableSpace(dir) > 0;
+  }
+
+  public static long getOccupiedSpace(String folderPath) throws IOException {
+    Path folder = Paths.get(folderPath);
+    return Files.walk(folder).filter(p -> p.toFile().isFile())
+        .mapToLong(p -> p.toFile().length()).sum();
+  }
 }

iotdb/src/test/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategyTest.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.conf.directories.strategy;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.db.utils.CommonUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+@RunWith(PowerMockRunner.class)
+@PowerMockIgnore({"com.sun.org.apache.xerces.*", "javax.xml.*", "org.xml.*", "org.w3c.*"})
+@PrepareForTest(CommonUtils.class)
+public class DirectoryStrategyTest {
+
+  List<String> dataDirList;
+  Set<Integer> fullDirIndexSet;
+
+  @Before
+  public void setUp() throws DiskSpaceInsufficientException, IOException {
+    dataDirList = new ArrayList<>();
+    for (int i = 0; i < 5; i++) {
+      dataDirList.add("data" + i);
+    }
+
+    fullDirIndexSet = new HashSet<>();
+    fullDirIndexSet.add(1);
+    fullDirIndexSet.add(3);
+
+    PowerMockito.mockStatic(CommonUtils.class);
+    for (int i = 0; i < dataDirList.size(); i++) {
+      boolean res = !fullDirIndexSet.contains(i);
+      PowerMockito.when(CommonUtils.hasSpace(dataDirList.get(i))).thenReturn(res);
+      PowerMockito.when(CommonUtils.getUsableSpace(dataDirList.get(i))).thenReturn(res ? (long) (i + 1) : 0L);
+      PowerMockito.when(CommonUtils.getOccupiedSpace(dataDirList.get(i))).thenReturn(res ? (long) (i + 1) : Long.MAX_VALUE);
+    }
+  }
+
+  @After
+  public void tearDown() {
+
+  }
+
+  @Test
+  public void testSequenceStrategy() throws DiskSpaceInsufficientException {
+    SequenceStrategy sequenceStrategy = new SequenceStrategy();
+    sequenceStrategy.init(dataDirList);
+
+    // loop two times of data dir size to fully loop
+    int index = 0;
+    for (int i = 0; i < dataDirList.size() * 2; i++, index++) {
+      index = index % dataDirList.size();
+      while (fullDirIndexSet.contains(index)) {
+        index = (index + 1) % dataDirList.size();
+      }
+      assertEquals(index, sequenceStrategy.nextFolderIndex());
+    }
+  }
+
+  @Test
+  public void testMaxDiskUsableSpaceFirstStrategy() throws DiskSpaceInsufficientException {
+    MaxDiskUsableSpaceFirstStrategy maxDiskUsableSpaceFirstStrategy = new MaxDiskUsableSpaceFirstStrategy();
+    maxDiskUsableSpaceFirstStrategy.init(dataDirList);
+
+    int maxIndex = getIndexOfMaxSpace();
+    for (int i = 0; i < dataDirList.size(); i++) {
+      assertEquals(maxIndex, maxDiskUsableSpaceFirstStrategy.nextFolderIndex());
+    }
+
+    PowerMockito.when(CommonUtils.getUsableSpace(dataDirList.get(maxIndex))).thenReturn(0L);
+    maxIndex = getIndexOfMaxSpace();
+    for (int i = 0; i < dataDirList.size(); i++) {
+      assertEquals(maxIndex, maxDiskUsableSpaceFirstStrategy.nextFolderIndex());
+    }
+  }
+
+  private int getIndexOfMaxSpace() {
+    int index = -1;
+    long maxSpace = -1;
+    for (int i = 0; i < dataDirList.size(); i++) {
+      long space = CommonUtils.getUsableSpace(dataDirList.get(i));
+      if (maxSpace < space) {
+        index = i;
+        maxSpace = space;
+      }
+    }
+    return index;
+  }
+
+  @Test
+  public void testMinFolderOccupiedSpaceFirstStrategy()
+      throws DiskSpaceInsufficientException, IOException {
+    MinFolderOccupiedSpaceFirstStrategy minFolderOccupiedSpaceFirstStrategy = new MinFolderOccupiedSpaceFirstStrategy();
+    minFolderOccupiedSpaceFirstStrategy.init(dataDirList);
+
+    int minIndex = getIndexOfMinOccupiedSpace();
+    for (int i = 0; i < dataDirList.size(); i++) {
+      assertEquals(minIndex, minFolderOccupiedSpaceFirstStrategy.nextFolderIndex());
+    }
+
+    PowerMockito.when(CommonUtils.getOccupiedSpace(dataDirList.get(minIndex))).thenReturn(Long.MAX_VALUE);
+    minIndex = getIndexOfMinOccupiedSpace();
+    for (int i = 0; i < dataDirList.size(); i++) {
+      assertEquals(minIndex, minFolderOccupiedSpaceFirstStrategy.nextFolderIndex());
+    }
+  }
+
+  private int getIndexOfMinOccupiedSpace() throws IOException {
+    int index = -1;
+    long minOccupied = Long.MAX_VALUE;
+    for (int i = 0; i < dataDirList.size(); i++) {
+      long space = CommonUtils.getOccupiedSpace(dataDirList.get(i));
+      if (minOccupied > space) {
+        index = i;
+        minOccupied = space;
+      }
+    }
+    return index;
+  }
+
+  @Test
+  public void testAllDiskFull() {
+    for (int i = 0; i < dataDirList.size(); i++) {
+      PowerMockito.when(CommonUtils.hasSpace(dataDirList.get(i))).thenReturn(false);
+    }
+
+    SequenceStrategy sequenceStrategy = new SequenceStrategy();
+    try {
+      sequenceStrategy.init(dataDirList);
+      fail();
+    } catch (DiskSpaceInsufficientException e) {
+    }
+
+    MaxDiskUsableSpaceFirstStrategy maxDiskUsableSpaceFirstStrategy = new MaxDiskUsableSpaceFirstStrategy();
+    try {
+      maxDiskUsableSpaceFirstStrategy.init(dataDirList);
+      fail();
+    } catch (DiskSpaceInsufficientException e) {
+    }
+
+    MinFolderOccupiedSpaceFirstStrategy minFolderOccupiedSpaceFirstStrategy = new MinFolderOccupiedSpaceFirstStrategy();
+    try {
+      minFolderOccupiedSpaceFirstStrategy.init(dataDirList);
+      fail();
+    } catch (DiskSpaceInsufficientException e) {
+    }
+  }
+}
\ No newline at end of file

iotdb/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java

@@ -152,7 +152,7 @@ public class EnvironmentUtils {
     }
     // create unsequential files
     for (String path : directoryManager.getAllUnSequenceFileFolders()) {
-      cleanDir(path);
+      createDir(path);
     }
     // create storage group
     createDir(config.getSystemDir());