automatic AOF rewrite first implementation. Still to be tested.

redis.conf

@@ -292,6 +292,26 @@ appendfsync everysec
 # "no" that is the safest pick from the point of view of durability.
 no-appendfsync-on-rewrite no
 
+# Automatic rewrite of the append only file.
+# Redis is able to automatically rewrite the log file implicitly calling
+# BGREWRITEAOF when the AOF log size will growth by the specified percentage.
+# 
+# This is how it works: Redis remembers the size of the AOF file after the
+# latest rewrite (or if no rewrite happened since the restart, the size of
+# the AOF at startup is used).
+#
+# This base size is compared to the current size. If the current size is
+# bigger than the specified percentage, the rewrite is triggered. Also
+# you need to specify a minimal size for the AOF file to be rewritten, this
+# is useful to avoid rewriting the AOF file even if the percentage increase
+# is reached but it is still pretty small.
+#
+# Specify a precentage of zero in order to disable the automatic AOF
+# rewrite feature.
+
+auto-aof-rewrite-percentage 100
+auto-aof-rewrite-min-size 64mb
+
 #################################### DISK STORE ###############################
 
 # When disk store is active Redis works as an on-disk database, where memory

src/aof.c

@@ -8,6 +8,8 @@
 #include <sys/resource.h>
 #include <sys/wait.h>
 
+void aofUpdateCurrentSize(void);
+
 /* Called when the user switches from "appendonly yes" to "appendonly no"
  * at runtime using the CONFIG command. */
 void stopAppendOnly(void) {
@@ -19,15 +21,15 @@ void stopAppendOnly(void) {
     server.appendseldb = -1;
     server.appendonly = 0;
     /* rewrite operation in progress? kill it, wait child exit */
-    if (server.bgsavechildpid != -1) {
+    if (server.bgrewritechildpid != -1) {
         int statloc;
 
-        if (kill(server.bgsavechildpid,SIGKILL) != -1)
+        if (kill(server.bgrewritechildpid,SIGKILL) != -1)
             wait3(&statloc,0,NULL);
         /* reset the buffer accumulating changes while the child saves */
         sdsfree(server.bgrewritebuf);
         server.bgrewritebuf = sdsempty();
-        server.bgsavechildpid = -1;
+        server.bgrewritechildpid = -1;
     }
 }
 
@@ -82,6 +84,7 @@ void flushAppendOnlyFile(void) {
     }
     sdsfree(server.aofbuf);
     server.aofbuf = sdsempty();
+    server.appendonly_current_size += nwritten;
 
     /* Don't Fsync if no-appendfsync-on-rewrite is set to yes and we have
      * childs performing heavy I/O on disk. */
@@ -221,6 +224,7 @@ int loadAppendOnlyFile(char *filename) {
     long loops = 0;
 
     if (fp && redis_fstat(fileno(fp),&sb) != -1 && sb.st_size == 0) {
+        server.appendonly_current_size = 0;
         fclose(fp);
         return REDIS_ERR;
     }
@@ -299,6 +303,7 @@ int loadAppendOnlyFile(char *filename) {
     freeFakeClient(fakeClient);
     server.appendonly = appendonly;
     stopLoading();
+    aofUpdateCurrentSize();
     return REDIS_OK;
 
 readerr:
@@ -611,9 +616,10 @@ int rewriteAppendOnlyFileBackground(void) {
 void bgrewriteaofCommand(redisClient *c) {
     if (server.bgrewritechildpid != -1) {
         addReplyError(c,"Background append only file rewriting already in progress");
-        return;
-    }
-    if (rewriteAppendOnlyFileBackground() == REDIS_OK) {
+    } else if (server.bgsavechildpid != -1) {
+        server.aofrewrite_scheduled = 1;
+        addReplyStatus(c,"Background append only file rewriting started");
+    } else if (rewriteAppendOnlyFileBackground() == REDIS_OK) {
         addReplyStatus(c,"Background append only file rewriting started");
     } else {
         addReply(c,shared.err);
@@ -627,6 +633,21 @@ void aofRemoveTempFile(pid_t childpid) {
     unlink(tmpfile);
 }
 
+/* Update the server.appendonly_current_size filed explicitly using stat(2)
+ * to check the size of the file. This is useful after a rewrite or after
+ * a restart, normally the size is updated just adding the write length
+ * to the current lenght, that is much faster. */
+void aofUpdateCurrentSize(void) {
+    struct redis_stat sb;
+
+    if (redis_fstat(server.appendfd,&sb) == -1) {
+        redisLog(REDIS_WARNING,"Unable to check the AOF length: %s",
+            strerror(errno));
+    } else {
+        server.appendonly_current_size = sb.st_size;
+    }
+}
+
 /* A background append only file rewriting (BGREWRITEAOF) terminated its work.
  * Handle this. */
 void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
@@ -667,6 +688,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
             if (server.appendfsync != APPENDFSYNC_NO) aof_fsync(fd);
             server.appendseldb = -1; /* Make sure it will issue SELECT */
             redisLog(REDIS_NOTICE,"The new append only file was selected for future appends.");
+            aofUpdateCurrentSize();
         } else {
             /* If append only is disabled we just generate a dump in this
              * format. Why not? */

src/config.c

@@ -231,6 +231,18 @@ void loadServerConfig(char *filename) {
                 err = "argument must be 'no', 'always' or 'everysec'";
                 goto loaderr;
             }
+        } else if (!strcasecmp(argv[0],"auto-aof-rewrite-percentage") &&
+                   argc == 2)
+        {
+            server.auto_aofrewrite_perc = atoi(argv[2]);
+            if (server.auto_aofrewrite_perc < 0) {
+                err = "Invalid negative percentage for AOF auto rewrite";
+                goto loaderr;
+            }
+        } else if (!strcasecmp(argv[0],"auto-aof-rewrite-min-size") &&
+                   argc == 2)
+        {
+            server.auto_aofrewrite_min_size = memtoll(argv[1],NULL);
         } else if (!strcasecmp(argv[0],"requirepass") && argc == 2) {
             server.requirepass = zstrdup(argv[1]);
         } else if (!strcasecmp(argv[0],"pidfile") && argc == 2) {

src/rdb.c

@@ -1039,6 +1039,9 @@ void bgsaveCommand(redisClient *c) {
     if (server.bgsavechildpid != -1 || server.bgsavethread != (pthread_t)-1) {
         addReplyError(c,"Background save already in progress");
         return;
+    } else if (server.bgrewritechildpid != -1) {
+        addReplyError(c,"Can't BGSAVE while AOF log rewriting is in progress");
+        return;
     }
     if (rdbSaveBackground(server.dbfilename) == REDIS_OK) {
         addReplyStatus(c,"Background saving started");

src/redis.c

@@ -633,6 +633,14 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
     if ((server.maxidletime && !(loops % 100)) || server.bpop_blocked_clients)
         closeTimedoutClients();
 
+    /* Start a scheduled AOF rewrite if this was requested by the user while
+     * a BGSAVE was in progress. */
+    if (server.bgsavechildpid == -1 && server.bgrewritechildpid == -1 &&
+        server.aofrewrite_scheduled)
+    {
+        rewriteAppendOnlyFileBackground();
+    }
+
     /* Check if a background saving or AOF rewrite in progress terminated. */
     if (server.bgsavechildpid != -1 || server.bgrewritechildpid != -1) {
         int statloc;
@@ -667,9 +675,10 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
             }
         }
     } else if (!server.ds_enabled) {
-        /* If there is not a background saving in progress check if
-         * we have to save now */
          time_t now = time(NULL);
+
+        /* If there is not a background saving/rewrite in progress check if
+         * we have to save/rewrite now */
          for (j = 0; j < server.saveparamslen; j++) {
             struct saveparam *sp = server.saveparams+j;
 
@@ -681,6 +690,18 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
                 break;
             }
          }
+
+         /* Trigger an AOF rewrite if needed */
+         if (server.auto_aofrewrite_perc &&
+             server.appendonly_current_size > server.auto_aofrewrite_min_size)
+         {
+            int growth = (server.appendonly_current_size*100/
+                          server.auto_aofrewrite_base_size);
+            if (growth >= server.auto_aofrewrite_perc) {
+                redisLog(REDIS_NOTICE,"Starting automatic rewriting of AOF on %d growth",growth);
+                rewriteAppendOnlyFileBackground();
+            }
+         }
     }
 
     /* Expire a few keys per cycle, only if this is a master.
@@ -828,6 +849,10 @@ void initServerConfig() {
     server.appendonly = 0;
     server.appendfsync = APPENDFSYNC_EVERYSEC;
     server.no_appendfsync_on_rewrite = 0;
+    server.auto_aofrewrite_perc = REDIS_AUTO_AOFREWRITE_PERC;
+    server.auto_aofrewrite_min_size = REDIS_AUTO_AOFREWRITE_MIN_SIZE;
+    server.auto_aofrewrite_base_size = 0;
+    server.aofrewrite_scheduled = 0;
     server.lastfsync = time(NULL);
     server.appendfd = -1;
     server.appendseldb = -1; /* Make sure the first time will not match */

src/redis.h

@@ -50,6 +50,8 @@
 #define REDIS_SHARED_INTEGERS 10000
 #define REDIS_REPLY_CHUNK_BYTES (5*1500) /* 5 TCP packets with default MTU */
 #define REDIS_MAX_LOGMSG_LEN    1024 /* Default maximum length of syslog messages */
+#define REDIS_AUTO_AOFREWRITE_PERC  100
+#define REDIS_AUTO_AOFREWRITE_MIN_SIZE (1024*1024)
 
 /* Hash table parameters */
 #define REDIS_HT_MINFILL        10      /* Minimal hash table fill 10% */
@@ -555,6 +557,11 @@ struct redisServer {
     int appendonly;
     int appendfsync;
     int no_appendfsync_on_rewrite;
+    int auto_aofrewrite_perc;       /* Rewrite AOF if % growth is > M and... */
+    off_t auto_aofrewrite_min_size; /* the AOF file is at least N bytes. */
+    off_t auto_aofrewrite_base_size;/* AOF size on latest startup or rewrite. */
+    off_t appendonly_current_size;  /* AOF current size. */
+    int aofrewrite_scheduled;       /* Rewrite once BGSAVE terminates. */
     int shutdown_asap;
     int activerehashing;
     char *requirepass;