Add feature to control the maximum agent log files (#3475)

* Add feature to control the maximum history agent log files.

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java

@@ -175,6 +175,12 @@ public class Config {
          */
         public static int MAX_FILE_SIZE = 300 * 1024 * 1024;
 
+        /**
+         * The max history log files. When rollover happened, if log files exceed this number,
+         * then the oldest file will be delete. Negative or zero means off, by default.
+         */
+        public static int MAX_HISTORY_FILES = -1;
+
         /**
          * The log level. Default is debug.
          */

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/logging/core/FileWriter.java

@@ -19,21 +19,22 @@
 
 package org.apache.skywalking.apm.agent.core.logging.core;
 
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
+import org.apache.skywalking.apm.agent.core.boot.DefaultNamedThreadFactory;
+import org.apache.skywalking.apm.agent.core.conf.Config;
+import org.apache.skywalking.apm.agent.core.conf.Constants;
+import org.apache.skywalking.apm.util.RunnableWithExceptionProtection;
+
+import java.io.*;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
 import java.util.Date;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.Callable;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
-import org.apache.skywalking.apm.agent.core.boot.DefaultNamedThreadFactory;
-import org.apache.skywalking.apm.agent.core.conf.Config;
-import org.apache.skywalking.apm.agent.core.conf.Constants;
-import org.apache.skywalking.apm.util.RunnableWithExceptionProtection;
+import java.util.regex.Pattern;
 
 /**
  * The <code>FileWriter</code> support async file output, by using a queue as buffer.
@@ -46,6 +47,7 @@ public class FileWriter implements IWriter {
     private FileOutputStream fileOutputStream;
     private ArrayBlockingQueue logBuffer;
     private volatile int fileSize;
+    private Pattern filenamePattern = Pattern.compile(Config.Logging.FILE_NAME + "\\.\\d{4}_\\d{2}_\\d{2}_\\d{2}_\\d{2}_\\d{2}");
 
     public static FileWriter get() {
         if (INSTANCE == null) {
@@ -134,6 +136,47 @@ public class FileWriter implements IWriter {
                     return null;
                 }
             });
+
+            if (Config.Logging.MAX_HISTORY_FILES > 0) {
+                deleteExpiredFiles();
+            }
+        }
+    }
+
+    /**
+     * load history log file name array
+     * @return history log file name array
+     */
+    private String[] getHistoryFilePath() {
+        File path = new File(Config.Logging.DIR);
+        String[] pathArr = path.list(new FilenameFilter() {
+            @Override
+            public boolean accept(File dir, String name) {
+                return filenamePattern.matcher(name).matches();
+            }
+        });
+
+        return pathArr;
+    }
+
+    /**
+     * delete expired log files
+     */
+    private void deleteExpiredFiles() {
+        String[] historyFileArr = getHistoryFilePath();
+        if (historyFileArr != null && historyFileArr.length > Config.Logging.MAX_HISTORY_FILES) {
+
+            Arrays.sort(historyFileArr, new Comparator<String>() {
+                @Override
+                public int compare(String o1, String o2) {
+                    return o2.compareTo(o1);
+                }
+            });
+
+            for (int i = Config.Logging.MAX_HISTORY_FILES; i < historyFileArr.length; i++) {
+                File expiredFile = new File(Config.Logging.DIR, historyFileArr[i]);
+                expiredFile.delete();
+            }
         }
     }
 

apm-sniffer/apm-agent-core/src/test/java/org/apache/skywalking/apm/agent/core/logging/core/FileWriterTest.java

@@ -26,7 +26,12 @@ import org.apache.skywalking.apm.agent.core.conf.Config;
 import org.apache.skywalking.apm.agent.core.conf.Constants;
 
 import java.io.File;
+import java.io.FilenameFilter;
 import java.io.IOException;
+import java.util.UUID;
+import java.util.regex.Pattern;
+
+import static org.junit.Assert.assertEquals;
 
 /**
  * @author wusheng
@@ -37,7 +42,8 @@ public class FileWriterTest {
     public static void beforeTestFile() throws IOException {
         Config.Logging.MAX_FILE_SIZE = 10;
         File directory = new File(System.getProperty("java.io.tmpdir", "/tmp"));
-        Config.Logging.DIR = directory.getCanonicalPath() + Constants.PATH_SEPARATOR + "/log-test/";
+        String dirName4Unique = UUID.randomUUID().toString();
+        Config.Logging.DIR = directory.getCanonicalPath() + Constants.PATH_SEPARATOR + "log-test_" + dirName4Unique;
     }
 
     @Test
@@ -50,6 +56,27 @@ public class FileWriterTest {
         Thread.sleep(10000L);
     }
 
+    @Test
+    public void testDeleteWhenRollover() throws InterruptedException {
+        Config.Logging.MAX_HISTORY_FILES = 3;
+        FileWriter writer = FileWriter.get();
+        for (int i = 0; i < 4; i++) {
+            writer.write("abcdefghij");
+            Thread.sleep(1000);
+        }
+
+        final Pattern filenamePattern = Pattern.compile(Config.Logging.FILE_NAME + "\\.\\d{4}_\\d{2}_\\d{2}_\\d{2}_\\d{2}_\\d{2}");
+        File path = new File(Config.Logging.DIR);
+        String[] pathArr = path.list(new FilenameFilter() {
+            @Override
+            public boolean accept(File dir, String name) {
+                return filenamePattern.matcher(name).matches();
+            }
+        });
+
+        assertEquals(3, pathArr.length);
+    }
+
     @AfterClass
     public static void clear() {
         Config.Logging.MAX_FILE_SIZE = 300 * 1024 * 1024;

apm-sniffer/config/agent.config

@@ -56,5 +56,9 @@ logging.level=${SW_LOGGING_LEVEL:DEBUG}
 # Logging max_file_size, default: 300 * 1024 * 1024 = 314572800
 # logging.max_file_size=${SW_LOGGING_MAX_FILE_SIZE:314572800}
 
+# The max history log files. When rollover happened, if log files exceed this number,
+# then the oldest file will be delete. Negative or zero means off, by default.
+# logging.max_history_files=${SW_LOGGING_MAX_HISTORY_FILES:-1}
+
 # mysql plugin configuration
 # plugin.mysql.trace_sql_parameters=${SW_MYSQL_TRACE_SQL_PARAMETERS:false}
\ No newline at end of file

docs/en/setup/service-agent/java-agent/README.md

@@ -88,6 +88,7 @@ property key | Description | Default |
 `logging.dir`|Log files directory. Default is blank string, means, use "system.out" to output logs.|`""`|
 `logging.pattern `|logging format. There are all conversion specifiers: <br>&nbsp;&nbsp;* `%level` means log level. <br>&nbsp;&nbsp;*  `%timestamp` means now of time with format `yyyy-MM-dd HH:mm:ss:SSS`.<br>&nbsp;&nbsp;*   `%thread` means name of current thread.<br>&nbsp;&nbsp;*   `%msg` means some message which user logged. <br>&nbsp;&nbsp;*  `%class` means SimpleName of TargetClass. <br>&nbsp;&nbsp;*  `%throwable` means a throwable which user called. <br>&nbsp;&nbsp;*  `%agent_name` means `agent.service_name`  |`%level %timestamp %thread %class : %msg %throwable`|
 `logging.max_file_size`|The max size of log file. If the size is bigger than this, archive the current file, and write into a new file.|`300 * 1024 * 1024`|
+`logging.max_history_files`|The max history log files. When rollover happened, if log files exceed this number,then the oldest file will be delete. Negative or zero means off, by default.|`-1`|
 `jvm.buffer_size`|The buffer size of collected JVM info.|`60 * 10`|
 `buffer.channel_size`|The buffer channel size.|`5`|
 `buffer.buffer_size`|The buffer size.|`300`|