merge conflicts resolved

src/Makefile

@@ -61,7 +61,7 @@ QUIET_CC = @printf '    %b %b\n' $(CCCOLOR)CC$(ENDCOLOR) $(SRCCOLOR)$@$(ENDCOLOR
 QUIET_LINK = @printf '    %b %b\n' $(LINKCOLOR)LINK$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR);
 endif
 
-OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endian.o slowlog.o scripting.o bio.o
+OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endian.o slowlog.o scripting.o bio.o rio.o
 BENCHOBJ = ae.o anet.o redis-benchmark.o sds.o adlist.o zmalloc.o
 CLIOBJ = anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o
 CHECKDUMPOBJ = redis-check-dump.o lzf_c.o lzf_d.o
@@ -130,6 +130,7 @@ replication.o: replication.c redis.h fmacros.h config.h ae.h sds.h dict.h \
 scripting.o: scripting.c redis.h fmacros.h config.h ae.h sds.h dict.h \
   adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h \
   sha1.h
+rio.o: rio.c sds.h
 sds.o: sds.c sds.h zmalloc.h
 sha1.o: sha1.c sha1.h config.h
 slowlog.o: slowlog.c redis.h fmacros.h config.h ae.h sds.h dict.h \

src/aof.c

 #include "redis.h"
 #include "bio.h"
+#include "rio.h"
 
 #include <signal.h>
 #include <fcntl.h>
@@ -386,11 +387,26 @@ fmterr:
     exit(1);
 }
 
+/* Delegate writing an object to writing a bulk string or bulk long long.
+ * This is not placed in rio.c since that adds the redis.h dependency. */
+int rioWriteBulkObject(rio *r, robj *obj) {
+    /* Avoid using getDecodedObject to help copy-on-write (we are often
+     * in a child process when this function is called). */
+    if (obj->encoding == REDIS_ENCODING_INT) {
+        return rioWriteBulkLongLong(r,(long)obj->ptr);
+    } else if (obj->encoding == REDIS_ENCODING_RAW) {
+        return rioWriteBulkString(r,obj->ptr,sdslen(obj->ptr));
+    } else {
+        redisPanic("Unknown string encoding");
+    }
+}
+
 /* Write a sequence of commands able to fully rebuild the dataset into
  * "filename". Used both by REWRITEAOF and BGREWRITEAOF. */
 int rewriteAppendOnlyFile(char *filename) {
     dictIterator *di = NULL;
     dictEntry *de;
+    rio aof;
     FILE *fp;
     char tmpfile[256];
     int j;
@@ -404,6 +420,8 @@ int rewriteAppendOnlyFile(char *filename) {
         redisLog(REDIS_WARNING, "Failed rewriting the append only file: %s", strerror(errno));
         return REDIS_ERR;
     }
+
+    aof = rioInitWithFile(fp);
     for (j = 0; j < server.dbnum; j++) {
         char selectcmd[] = "*2\r\n$6\r\nSELECT\r\n";
         redisDb *db = server.db+j;
@@ -416,8 +434,8 @@ int rewriteAppendOnlyFile(char *filename) {
         }
 
         /* SELECT the new DB */
-        if (fwrite(selectcmd,sizeof(selectcmd)-1,1,fp) == 0) goto werr;
-        if (fwriteBulkLongLong(fp,j) == 0) goto werr;
+        if (rioWrite(&aof,selectcmd,sizeof(selectcmd)-1) == 0) goto werr;
+        if (rioWriteBulkLongLong(&aof,j) == 0) goto werr;
 
         /* Iterate this DB writing every entry */
         while((de = dictNext(di)) != NULL) {
@@ -435,10 +453,10 @@ int rewriteAppendOnlyFile(char *filename) {
             if (o->type == REDIS_STRING) {
                 /* Emit a SET command */
                 char cmd[]="*3\r\n$3\r\nSET\r\n";
-                if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
+                if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr;
                 /* Key and value */
-                if (fwriteBulkObject(fp,&key) == 0) goto werr;
-                if (fwriteBulkObject(fp,o) == 0) goto werr;
+                if (rioWriteBulkObject(&aof,&key) == 0) goto werr;
+                if (rioWriteBulkObject(&aof,o) == 0) goto werr;
             } else if (o->type == REDIS_LIST) {
                 /* Emit the RPUSHes needed to rebuild the list */
                 char cmd[]="*3\r\n$5\r\nRPUSH\r\n";
@@ -450,13 +468,13 @@ int rewriteAppendOnlyFile(char *filename) {
                     long long vlong;
 
                     while(ziplistGet(p,&vstr,&vlen,&vlong)) {
-                        if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
-                        if (fwriteBulkObject(fp,&key) == 0) goto werr;
+                        if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr;
+                        if (rioWriteBulkObject(&aof,&key) == 0) goto werr;
                         if (vstr) {
-                            if (fwriteBulkString(fp,(char*)vstr,vlen) == 0)
+                            if (rioWriteBulkString(&aof,(char*)vstr,vlen) == 0)
                                 goto werr;
                         } else {
-                            if (fwriteBulkLongLong(fp,vlong) == 0)
+                            if (rioWriteBulkLongLong(&aof,vlong) == 0)
                                 goto werr;
                         }
                         p = ziplistNext(zl,p);
@@ -470,9 +488,9 @@ int rewriteAppendOnlyFile(char *filename) {
                     while((ln = listNext(&li))) {
                         robj *eleobj = listNodeValue(ln);
 
-                        if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
-                        if (fwriteBulkObject(fp,&key) == 0) goto werr;
-                        if (fwriteBulkObject(fp,eleobj) == 0) goto werr;
+                        if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr;
+                        if (rioWriteBulkObject(&aof,&key) == 0) goto werr;
+                        if (rioWriteBulkObject(&aof,eleobj) == 0) goto werr;
                     }
                 } else {
                     redisPanic("Unknown list encoding");
@@ -485,18 +503,18 @@ int rewriteAppendOnlyFile(char *filename) {
                     int ii = 0;
                     int64_t llval;
                     while(intsetGet(o->ptr,ii++,&llval)) {
-                        if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
-                        if (fwriteBulkObject(fp,&key) == 0) goto werr;
-                        if (fwriteBulkLongLong(fp,llval) == 0) goto werr;
+                        if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr;
+                        if (rioWriteBulkObject(&aof,&key) == 0) goto werr;
+                        if (rioWriteBulkLongLong(&aof,llval) == 0) goto werr;
                     }
                 } else if (o->encoding == REDIS_ENCODING_HT) {
                     dictIterator *di = dictGetIterator(o->ptr);
                     dictEntry *de;
                     while((de = dictNext(di)) != NULL) {
                         robj *eleobj = dictGetEntryKey(de);
-                        if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
-                        if (fwriteBulkObject(fp,&key) == 0) goto werr;
-                        if (fwriteBulkObject(fp,eleobj) == 0) goto werr;
+                        if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr;
+                        if (rioWriteBulkObject(&aof,&key) == 0) goto werr;
+                        if (rioWriteBulkObject(&aof,eleobj) == 0) goto werr;
                     }
                     dictReleaseIterator(di);
                 } else {
@@ -523,14 +541,14 @@ int rewriteAppendOnlyFile(char *filename) {
                         redisAssert(ziplistGet(eptr,&vstr,&vlen,&vll));
                         score = zzlGetScore(sptr);
 
-                        if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
-                        if (fwriteBulkObject(fp,&key) == 0) goto werr;
-                        if (fwriteBulkDouble(fp,score) == 0) goto werr;
+                        if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr;
+                        if (rioWriteBulkObject(&aof,&key) == 0) goto werr;
+                        if (rioWriteBulkDouble(&aof,score) == 0) goto werr;
                         if (vstr != NULL) {
-                            if (fwriteBulkString(fp,(char*)vstr,vlen) == 0)
+                            if (rioWriteBulkString(&aof,(char*)vstr,vlen) == 0)
                                 goto werr;
                         } else {
-                            if (fwriteBulkLongLong(fp,vll) == 0)
+                            if (rioWriteBulkLongLong(&aof,vll) == 0)
                                 goto werr;
                         }
                         zzlNext(zl,&eptr,&sptr);
@@ -544,10 +562,10 @@ int rewriteAppendOnlyFile(char *filename) {
                         robj *eleobj = dictGetEntryKey(de);
                         double *score = dictGetEntryVal(de);
 
-                        if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
-                        if (fwriteBulkObject(fp,&key) == 0) goto werr;
-                        if (fwriteBulkDouble(fp,*score) == 0) goto werr;
-                        if (fwriteBulkObject(fp,eleobj) == 0) goto werr;
+                        if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr;
+                        if (rioWriteBulkObject(&aof,&key) == 0) goto werr;
+                        if (rioWriteBulkDouble(&aof,*score) == 0) goto werr;
+                        if (rioWriteBulkObject(&aof,eleobj) == 0) goto werr;
                     }
                     dictReleaseIterator(di);
                 } else {
@@ -563,11 +581,11 @@ int rewriteAppendOnlyFile(char *filename) {
                     unsigned int flen, vlen;
 
                     while((p = zipmapNext(p,&field,&flen,&val,&vlen)) != NULL) {
-                        if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
-                        if (fwriteBulkObject(fp,&key) == 0) goto werr;
-                        if (fwriteBulkString(fp,(char*)field,flen) == 0)
+                        if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr;
+                        if (rioWriteBulkObject(&aof,&key) == 0) goto werr;
+                        if (rioWriteBulkString(&aof,(char*)field,flen) == 0)
                             goto werr;
-                        if (fwriteBulkString(fp,(char*)val,vlen) == 0)
+                        if (rioWriteBulkString(&aof,(char*)val,vlen) == 0)
                             goto werr;
                     }
                 } else {
@@ -578,10 +596,10 @@ int rewriteAppendOnlyFile(char *filename) {
                         robj *field = dictGetEntryKey(de);
                         robj *val = dictGetEntryVal(de);
 
-                        if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
-                        if (fwriteBulkObject(fp,&key) == 0) goto werr;
-                        if (fwriteBulkObject(fp,field) == 0) goto werr;
-                        if (fwriteBulkObject(fp,val) == 0) goto werr;
+                        if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr;
+                        if (rioWriteBulkObject(&aof,&key) == 0) goto werr;
+                        if (rioWriteBulkObject(&aof,field) == 0) goto werr;
+                        if (rioWriteBulkObject(&aof,val) == 0) goto werr;
                     }
                     dictReleaseIterator(di);
                 }
@@ -593,9 +611,9 @@ int rewriteAppendOnlyFile(char *filename) {
                 char cmd[]="*3\r\n$8\r\nEXPIREAT\r\n";
                 /* If this key is already expired skip it */
                 if (expiretime < now) continue;
-                if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
-                if (fwriteBulkObject(fp,&key) == 0) goto werr;
-                if (fwriteBulkLongLong(fp,expiretime) == 0) goto werr;
+                if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr;
+                if (rioWriteBulkObject(&aof,&key) == 0) goto werr;
+                if (rioWriteBulkLongLong(&aof,expiretime) == 0) goto werr;
             }
         }
         dictReleaseIterator(di);

src/cluster.c

@@ -1383,11 +1383,10 @@ void clusterCommand(redisClient *c) {
 
 /* RESTORE key ttl serialized-value */
 void restoreCommand(redisClient *c) {
-    FILE *fp;
-    char buf[64];
-    robj *o;
-    unsigned char *data;
     long ttl;
+    rio payload;
+    int type;
+    robj *obj;
 
     /* Make sure this key does not already exist here... */
     if (lookupKeyWrite(c->db,c->argv[1]) != NULL) {
@@ -1403,44 +1402,16 @@ void restoreCommand(redisClient *c) {
         return;
     }
 
-    /* rdbLoadObject() only works against file descriptors so we need to
-     * dump the serialized object into a file and reload. */
-    snprintf(buf,sizeof(buf),"redis-restore-%d.tmp",getpid());
-    fp = fopen(buf,"w+");
-    if (!fp) {
-        redisLog(REDIS_WARNING,"Can't open tmp file for RESTORE: %s",
-            strerror(errno));
-        addReplyErrorFormat(c,"RESTORE failed, tmp file creation error: %s",
-            strerror(errno));
-        return;
-    }
-    unlink(buf);
-
-    /* Write the actual data and rewind the file */
-    data = (unsigned char*) c->argv[3]->ptr;
-    if (fwrite(data+1,sdslen((sds)data)-1,1,fp) != 1) {
-        redisLog(REDIS_WARNING,"Can't write against tmp file for RESTORE: %s",
-            strerror(errno));
-        addReplyError(c,"RESTORE failed, tmp file I/O error.");
-        fclose(fp);
-        return;
-    }
-    rewind(fp);
-
-    /* Finally create the object from the serialized dump and
-     * store it at the specified key. */
-    if ((data[0] > 4 && data[0] < 9) ||
-         data[0] > 11 ||
-        (o = rdbLoadObject(data[0],fp)) == NULL)
+    payload = rioInitWithBuffer(c->argv[3]->ptr);
+    if (((type = rdbLoadObjectType(&payload)) == -1) ||
+        ((obj = rdbLoadObject(type,&payload)) == NULL))
     {
-        addReplyError(c,"Bad data format.");
-        fclose(fp);
+        addReplyError(c,"Bad data format");
         return;
     }
-    fclose(fp);
 
     /* Create the key and set the TTL if any */
-    dbAdd(c->db,c->argv[1],o);
+    dbAdd(c->db,c->argv[1],obj);
     if (ttl) setExpire(c->db,c->argv[1],time(NULL)+ttl);
     addReply(c,shared.ok);
 }
@@ -1450,12 +1421,9 @@ void migrateCommand(redisClient *c) {
     int fd;
     long timeout;
     long dbid;
-    char buf[64];
-    FILE *fp;
     time_t ttl;
     robj *o;
-    unsigned char type;
-    off_t payload_len;
+    rio cmd, payload;
 
     /* Sanity check */
     if (getLongFromObjectOrReply(c,c->argv[5],&timeout,NULL) != REDIS_OK)
@@ -1485,54 +1453,41 @@ void migrateCommand(redisClient *c) {
         return;
     }
 
-    /* Create temp file */
-    snprintf(buf,sizeof(buf),"redis-migrate-%d.tmp",getpid());
-    fp = fopen(buf,"w+");
-    if (!fp) {
-        redisLog(REDIS_WARNING,"Can't open tmp file for MIGRATE: %s",
-            strerror(errno));
-        addReplyErrorFormat(c,"MIGRATE failed, tmp file creation error: %s.",
-            strerror(errno));
-        return;
-    }
-    unlink(buf);
-
-    /* Build the SELECT + RESTORE query writing it in our temp file. */
-    if (fwriteBulkCount(fp,'*',2) == 0) goto file_wr_err;
-    if (fwriteBulkString(fp,"SELECT",6) == 0) goto file_wr_err;
-    if (fwriteBulkLongLong(fp,dbid) == 0) goto file_wr_err;
+    cmd = rioInitWithBuffer(sdsempty());
+    redisAssert(rioWriteBulkCount(&cmd,'*',2));
+    redisAssert(rioWriteBulkString(&cmd,"SELECT",6));
+    redisAssert(rioWriteBulkLongLong(&cmd,dbid));
 
     ttl = getExpire(c->db,c->argv[3]);
-    type = o->type;
-    if (fwriteBulkCount(fp,'*',4) == 0) goto file_wr_err;
-    if (fwriteBulkString(fp,"RESTORE",7) == 0) goto file_wr_err;
-    if (fwriteBulkObject(fp,c->argv[3]) == 0) goto file_wr_err;
-    if (fwriteBulkLongLong(fp, (ttl == -1) ? 0 : ttl) == 0) goto file_wr_err;
+    redisAssert(rioWriteBulkCount(&cmd,'*',4));
+    redisAssert(rioWriteBulkString(&cmd,"RESTORE",7));
+    redisAssert(c->argv[3]->encoding == REDIS_ENCODING_RAW);
+    redisAssert(rioWriteBulkString(&cmd,c->argv[3]->ptr,sdslen(c->argv[3]->ptr)));
+    redisAssert(rioWriteBulkLongLong(&cmd,(ttl == -1) ? 0 : ttl));
 
     /* Finally the last argument that is the serailized object payload
-     * in the form: <type><rdb-serailized-object>. */
-    payload_len = rdbSavedObjectLen(o);
-    if (fwriteBulkCount(fp,'$',payload_len+1) == 0) goto file_wr_err;
-    if (fwrite(&type,1,1,fp) == 0) goto file_wr_err;
-    if (rdbSaveObject(fp,o) == -1) goto file_wr_err;
-    if (fwrite("\r\n",2,1,fp) == 0) goto file_wr_err;
-
-    /* Tranfer the query to the other node */
-    rewind(fp);
+     * in the form: <type><rdb-serialized-object>. */
+    payload = rioInitWithBuffer(sdsempty());
+    redisAssert(rdbSaveObjectType(&payload,o));
+    redisAssert(rdbSaveObject(&payload,o) != -1);
+    redisAssert(rioWriteBulkString(&cmd,payload.io.buffer.ptr,sdslen(payload.io.buffer.ptr)));
+    sdsfree(payload.io.buffer.ptr);
+
+    /* Tranfer the query to the other node in 64K chunks. */
     {
-        char buf[4096];
-        size_t nread;
-
-        while ((nread = fread(buf,1,sizeof(buf),fp)) != 0) {
-            int nwritten;
-
-            nwritten = syncWrite(fd,buf,nread,timeout);
-            if (nwritten != (signed)nread) goto socket_wr_err;
+        sds buf = cmd.io.buffer.ptr;
+        size_t pos = 0, towrite;
+        int nwritten = 0;
+
+        while ((towrite = sdslen(buf)-pos) > 0) {
+            towrite = (towrite > (64*1024) ? (64*1024) : towrite);
+            nwritten = syncWrite(fd,buf+nwritten,towrite,timeout);
+            if (nwritten != (signed)towrite) goto socket_wr_err;
+            pos += nwritten;
         }
-        if (ferror(fp)) goto file_rd_err;
     }
 
-    /* Read back the reply */
+    /* Read back the reply. */
     {
         char buf1[1024];
         char buf2[1024];
@@ -1541,7 +1496,7 @@ void migrateCommand(redisClient *c) {
         if (syncReadLine(fd, buf1, sizeof(buf1), timeout) <= 0)
             goto socket_rd_err;
         if (syncReadLine(fd, buf2, sizeof(buf2), timeout) <= 0)
-                goto socket_rd_err;
+            goto socket_rd_err;
         if (buf1[0] == '-' || buf2[0] == '-') {
             addReplyErrorFormat(c,"Target instance replied with error: %s",
                 (buf1[0] == '-') ? buf1+1 : buf2+1);
@@ -1550,25 +1505,8 @@ void migrateCommand(redisClient *c) {
             addReply(c,shared.ok);
         }
     }
-    fclose(fp);
-    close(fd);
-    return;
 
-file_wr_err:
-    redisLog(REDIS_WARNING,"Can't write on tmp file for MIGRATE: %s",
-        strerror(errno));
-    addReplyErrorFormat(c,"MIGRATE failed, tmp file write error: %s.",
-        strerror(errno));
-    fclose(fp);
-    close(fd);
-    return;
-
-file_rd_err:
-    redisLog(REDIS_WARNING,"Can't read from tmp file for MIGRATE: %s",
-        strerror(errno));
-    addReplyErrorFormat(c,"MIGRATE failed, tmp file read error: %s.",
-        strerror(errno));
-    fclose(fp);
+    sdsfree(cmd.io.buffer.ptr);
     close(fd);
     return;
 
@@ -1577,7 +1515,7 @@ socket_wr_err:
         strerror(errno));
     addReplyErrorFormat(c,"MIGRATE failed, writing to target node: %s.",
         strerror(errno));
-    fclose(fp);
+    sdsfree(cmd.io.buffer.ptr);
     close(fd);
     return;
 
@@ -1586,7 +1524,7 @@ socket_rd_err:
         strerror(errno));
     addReplyErrorFormat(c,"MIGRATE failed, reading from target node: %s.",
         strerror(errno));
-    fclose(fp);
+    sdsfree(cmd.io.buffer.ptr);
     close(fd);
     return;
 }
@@ -1595,74 +1533,26 @@ socket_rd_err:
  * DUMP is actually not used by Redis Cluster but it is the obvious
  * complement of RESTORE and can be useful for different applications. */
 void dumpCommand(redisClient *c) {
-    char buf[64];
-    FILE *fp;
     robj *o, *dumpobj;
-    sds dump = NULL;
-    off_t payload_len;
-    unsigned int type;
+    rio payload;
 
     /* Check if the key is here. */
     if ((o = lookupKeyRead(c->db,c->argv[1])) == NULL) {
         addReply(c,shared.nullbulk);
         return;
     }
-    
-    /* Create temp file */
-    snprintf(buf,sizeof(buf),"redis-dump-%d.tmp",getpid());
-    fp = fopen(buf,"w+");
-    if (!fp) {
-        redisLog(REDIS_WARNING,"Can't open tmp file for MIGRATE: %s",
-            strerror(errno));
-        addReplyErrorFormat(c,"DUMP failed, tmp file creation error: %s.",
-            strerror(errno));
-        return;
-    }
-    unlink(buf);
-
-    /* Dump the serailized object and read it back in memory.
-     * We prefix it with a one byte containing the type ID.
-     * This is the serialization format understood by RESTORE. */
-    if (rdbSaveObject(fp,o) == -1) goto file_wr_err;
-    payload_len = ftello(fp);
-    if (fseeko(fp,0,SEEK_SET) == -1) goto file_rd_err;
-    dump = sdsnewlen(NULL,payload_len+1);
-    if (payload_len && fread(dump+1,payload_len,1,fp) != 1) goto file_rd_err;
-    fclose(fp);
-    type = o->type;
-    if (type == REDIS_LIST && o->encoding == REDIS_ENCODING_ZIPLIST)
-        type = REDIS_LIST_ZIPLIST;
-    else if (type == REDIS_HASH && o->encoding == REDIS_ENCODING_ZIPMAP)
-        type = REDIS_HASH_ZIPMAP;
-    else if (type == REDIS_SET && o->encoding == REDIS_ENCODING_INTSET)
-        type = REDIS_SET_INTSET;
-    else
-        type = o->type;
-    dump[0] = type;
+
+    /* Serialize the object in a RDB-like format. It consist of an object type
+     * byte followed by the serialized object. This is understood by RESTORE. */
+    payload = rioInitWithBuffer(sdsempty());
+    redisAssert(rdbSaveObjectType(&payload,o));
+    redisAssert(rdbSaveObject(&payload,o));
 
     /* Transfer to the client */
-    dumpobj = createObject(REDIS_STRING,dump);
+    dumpobj = createObject(REDIS_STRING,payload.io.buffer.ptr);
     addReplyBulk(c,dumpobj);
     decrRefCount(dumpobj);
     return;
-
-file_wr_err:
-    redisLog(REDIS_WARNING,"Can't write on tmp file for DUMP: %s",
-        strerror(errno));
-    addReplyErrorFormat(c,"DUMP failed, tmp file write error: %s.",
-        strerror(errno));
-    sdsfree(dump);
-    fclose(fp);
-    return;
-
-file_rd_err:
-    redisLog(REDIS_WARNING,"Can't read from tmp file for DUMP: %s",
-        strerror(errno));
-    addReplyErrorFormat(c,"DUMP failed, tmp file read error: %s.",
-        strerror(errno));
-    sdsfree(dump);
-    fclose(fp);
-    return;
 }
 
 /* -----------------------------------------------------------------------------

src/rdb.h

+#ifndef __REDIS_RDB_H
+#define __REDIS_RDB_H
+
+#include <stdio.h>
+#include "rio.h"
+
+/* TBD: include only necessary headers. */
+#include "redis.h"
+
+/* Defines related to the dump file format. To store 32 bits lengths for short
+ * keys requires a lot of space, so we check the most significant 2 bits of
+ * the first byte to interpreter the length:
+ *
+ * 00|000000 => if the two MSB are 00 the len is the 6 bits of this byte
+ * 01|000000 00000000 =>  01, the len is 14 byes, 6 bits + 8 bits of next byte
+ * 10|000000 [32 bit integer] => if it's 01, a full 32 bit len will follow
+ * 11|000000 this means: specially encoded object will follow. The six bits
+ *           number specify the kind of object that follows.
+ *           See the REDIS_RDB_ENC_* defines.
+ *
+ * Lenghts up to 63 are stored using a single byte, most DB keys, and may
+ * values, will fit inside. */
+#define REDIS_RDB_6BITLEN 0
+#define REDIS_RDB_14BITLEN 1
+#define REDIS_RDB_32BITLEN 2
+#define REDIS_RDB_ENCVAL 3
+#define REDIS_RDB_LENERR UINT_MAX
+
+/* When a length of a string object stored on disk has the first two bits
+ * set, the remaining two bits specify a special encoding for the object
+ * accordingly to the following defines: */
+#define REDIS_RDB_ENC_INT8 0        /* 8 bit signed integer */
+#define REDIS_RDB_ENC_INT16 1       /* 16 bit signed integer */
+#define REDIS_RDB_ENC_INT32 2       /* 32 bit signed integer */
+#define REDIS_RDB_ENC_LZF 3         /* string compressed with FASTLZ */
+
+/* Dup object types to RDB object types. Only reason is readability (are we
+ * dealing with RDB types or with in-memory object types?). */
+#define REDIS_RDB_TYPE_STRING 0
+#define REDIS_RDB_TYPE_LIST   1
+#define REDIS_RDB_TYPE_SET    2
+#define REDIS_RDB_TYPE_ZSET   3
+#define REDIS_RDB_TYPE_HASH   4
+
+/* Object types for encoded objects. */
+#define REDIS_RDB_TYPE_HASH_ZIPMAP    9
+#define REDIS_RDB_TYPE_LIST_ZIPLIST  10
+#define REDIS_RDB_TYPE_SET_INTSET    11
+#define REDIS_RDB_TYPE_ZSET_ZIPLIST  12
+
+/* Test if a type is an object type. */
+#define rdbIsObjectType(t) ((t >= 0 && t <= 4) || (t >= 9 && t <= 12))
+
+/* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */
+#define REDIS_RDB_OPCODE_EXPIRETIME 253
+#define REDIS_RDB_OPCODE_SELECTDB   254
+#define REDIS_RDB_OPCODE_EOF        255
+
+/* Test if a type is an opcode. */
+#define rdbIsOpcode(t) (t >= 253 && t <= 255)
+
+int rdbSaveType(rio *rdb, unsigned char type);
+int rdbLoadType(rio *rdb);
+int rdbSaveTime(rio *rdb, time_t t);
+time_t rdbLoadTime(rio *rdb);
+int rdbSaveLen(rio *rdb, uint32_t len);
+uint32_t rdbLoadLen(rio *rdb, int *isencoded);
+int rdbSaveObjectType(rio *rdb, robj *o);
+int rdbLoadObjectType(rio *rdb);
+int rdbLoad(char *filename);
+int rdbSaveBackground(char *filename);
+void rdbRemoveTempFile(pid_t childpid);
+int rdbSave(char *filename);
+int rdbSaveObject(rio *rdb, robj *o);
+off_t rdbSavedObjectLen(robj *o);
+off_t rdbSavedObjectPages(robj *o);
+robj *rdbLoadObject(int type, rio *rdb);
+void backgroundSaveDoneHandler(int exitcode, int bysignal);
+int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, time_t expireitme, time_t now);
+robj *rdbLoadStringObject(rio *rdb);
+
+#endif

src/redis.h

@@ -77,12 +77,6 @@
 #define REDIS_HASH 4
 #define REDIS_VMPOINTER 8
 
-/* Object types only used for persistence in .rdb files */
-#define REDIS_HASH_ZIPMAP 9
-#define REDIS_LIST_ZIPLIST 10
-#define REDIS_SET_INTSET 11
-#define REDIS_ZSET_ZIPLIST 12
-
 /* Objects encoding. Some kind of objects like Strings and Hashes can be
  * internally represented in multiple ways. The 'encoding' field of the object
  * is set to one of this fields for this object. */
@@ -836,11 +830,6 @@ unsigned long estimateObjectIdleTime(robj *o);
 int syncWrite(int fd, char *ptr, ssize_t size, int timeout);
 int syncRead(int fd, char *ptr, ssize_t size, int timeout);
 int syncReadLine(int fd, char *ptr, ssize_t size, int timeout);
-int fwriteBulkString(FILE *fp, char *s, unsigned long len);
-int fwriteBulkDouble(FILE *fp, double d);
-int fwriteBulkLongLong(FILE *fp, long long l);
-int fwriteBulkObject(FILE *fp, robj *obj);
-int fwriteBulkCount(FILE *fp, char prefix, int count);
 
 /* Replication */
 void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc);
@@ -854,21 +843,7 @@ void loadingProgress(off_t pos);
 void stopLoading(void);
 
 /* RDB persistence */
-int rdbLoad(char *filename);
-int rdbSaveBackground(char *filename);
-void rdbRemoveTempFile(pid_t childpid);
-int rdbSave(char *filename);
-int rdbSaveObject(FILE *fp, robj *o);
-off_t rdbSavedObjectLen(robj *o);
-off_t rdbSavedObjectPages(robj *o);
-robj *rdbLoadObject(int type, FILE *fp);
-void backgroundSaveDoneHandler(int exitcode, int bysignal);
-int rdbSaveKeyValuePair(FILE *fp, robj *key, robj *val, time_t expireitme, time_t now);
-int rdbLoadType(FILE *fp);
-time_t rdbLoadTime(FILE *fp);
-robj *rdbLoadStringObject(FILE *fp);
-int rdbSaveType(FILE *fp, unsigned char type);
-int rdbSaveLen(FILE *fp, uint32_t len);
+#include "rdb.h"
 
 /* AOF persistence */
 void flushAppendOnlyFile(int force);

src/rio.c

+#include <string.h>
+#include "rio.h"
+#include "util.h"
+
+/* Returns 1 or 0 for success/failure. */
+static size_t rioBufferWrite(rio *r, const void *buf, size_t len) {
+    r->io.buffer.ptr = sdscatlen(r->io.buffer.ptr,(char*)buf,len);
+    r->io.buffer.pos += len;
+    return len;
+}
+
+/* Returns 1 or 0 for success/failure. */
+static size_t rioBufferRead(rio *r, void *buf, size_t len) {
+    if (sdslen(r->io.buffer.ptr)-r->io.buffer.pos < len)
+        return 0;
+    memcpy(buf,r->io.buffer.ptr+r->io.buffer.pos,len);
+    r->io.buffer.pos += len;
+    return 1;
+}
+
+/* Returns read/write position in buffer. */
+static off_t rioBufferTell(rio *r) {
+    return r->io.buffer.pos;
+}
+
+/* Returns 1 or 0 for success/failure. */
+static size_t rioFileWrite(rio *r, const void *buf, size_t len) {
+    return fwrite(buf,len,1,r->io.file.fp);
+}
+
+/* Returns 1 or 0 for success/failure. */
+static size_t rioFileRead(rio *r, void *buf, size_t len) {
+    return fread(buf,len,1,r->io.file.fp);
+}
+
+/* Returns read/write position in file. */
+static off_t rioFileTell(rio *r) {
+    return ftello(r->io.file.fp);
+}
+
+static const rio rioBufferIO = {
+    rioBufferRead,
+    rioBufferWrite,
+    rioBufferTell,
+    { { NULL, 0 } } /* union for io-specific vars */
+};
+
+static const rio rioFileIO = {
+    rioFileRead,
+    rioFileWrite,
+    rioFileTell,
+    { { NULL, 0 } } /* union for io-specific vars */
+};
+
+rio rioInitWithFile(FILE *fp) {
+    rio r = rioFileIO;
+    r.io.file.fp = fp;
+    return r;
+}
+rio rioInitWithBuffer(sds s) {
+    rio r = rioBufferIO;
+    r.io.buffer.ptr = s;
+    r.io.buffer.pos = 0;
+    return r;
+}
+
+/* Write multi bulk count in the format: "*<count>\r\n". */
+size_t rioWriteBulkCount(rio *r, char prefix, int count) {
+    char cbuf[128];
+    int clen;
+
+    cbuf[0] = prefix;
+    clen = 1+ll2string(cbuf+1,sizeof(cbuf)-1,count);
+    cbuf[clen++] = '\r';
+    cbuf[clen++] = '\n';
+    if (rioWrite(r,cbuf,clen) == 0) return 0;
+    return clen;
+}
+
+/* Write binary-safe string in the format: "$<count>\r\n<payload>\r\n". */
+size_t rioWriteBulkString(rio *r, const char *buf, size_t len) {
+    size_t nwritten;
+
+    if ((nwritten = rioWriteBulkCount(r,'$',len)) == 0) return 0;
+    if (len > 0 && rioWrite(r,buf,len) == 0) return 0;
+    if (rioWrite(r,"\r\n",2) == 0) return 0;
+    return nwritten+len+2;
+}
+
+/* Write a long long value in format: "$<count>\r\n<payload>\r\n". */
+size_t rioWriteBulkLongLong(rio *r, long long l) {
+    char lbuf[32];
+    unsigned int llen;
+
+    llen = ll2string(lbuf,sizeof(lbuf),l);
+    return rioWriteBulkString(r,lbuf,llen);
+}
+
+/* Write a double value in the format: "$<count>\r\n<payload>\r\n" */
+size_t rioWriteBulkDouble(rio *r, double d) {
+    char dbuf[128];
+    unsigned int dlen;
+
+    dlen = snprintf(dbuf,sizeof(dbuf),"%.17g",d);
+    return rioWriteBulkString(r,dbuf,dlen);
+}

src/rio.h

+#ifndef __REDIS_RIO_H
+#define __REDIS_RIO_H
+
+#include <stdio.h>
+#include "sds.h"
+
+struct _rio {
+    /* Backend functions. Both read and write should return 0 for short reads
+     * or writes, identical to the return values of fread/fwrite. */
+    size_t (*read)(struct _rio *, void *buf, size_t len);
+    size_t (*write)(struct _rio *, const void *buf, size_t len);
+    off_t (*tell)(struct _rio *);
+
+    /* Backend-specific vars. */
+    union {
+        struct {
+            sds ptr;
+            off_t pos;
+        } buffer;
+        struct {
+            FILE *fp;
+        } file;
+    } io;
+};
+
+typedef struct _rio rio;
+
+#define rioWrite(rio,buf,len) ((rio)->write((rio),(buf),(len)))
+#define rioRead(rio,buf,len) ((rio)->read((rio),(buf),(len)))
+
+rio rioInitWithFile(FILE *fp);
+rio rioInitWithBuffer(sds s);
+
+size_t rioWriteBulkCount(rio *r, char prefix, int count);
+size_t rioWriteBulkString(rio *r, const char *buf, size_t len);
+size_t rioWriteBulkLongLong(rio *r, long long l);
+size_t rioWriteBulkDouble(rio *r, double d);
+
+#endif

src/syncio.c

@@ -99,70 +99,3 @@ int syncReadLine(int fd, char *ptr, ssize_t size, int timeout) {
     }
     return nread;
 }
-
-/* ----------------- Blocking sockets I/O with timeouts --------------------- */
-
-/* Write binary-safe string into a file in the bulkformat
- * $<count>\r\n<payload>\r\n */
-int fwriteBulkString(FILE *fp, char *s, unsigned long len) {
-    char cbuf[128];
-    int clen;
-
-    cbuf[0] = '$';
-    clen = 1+ll2string(cbuf+1,sizeof(cbuf)-1,len);
-    cbuf[clen++] = '\r';
-    cbuf[clen++] = '\n';
-    if (fwrite(cbuf,clen,1,fp) == 0) return 0;
-    if (len > 0 && fwrite(s,len,1,fp) == 0) return 0;
-    if (fwrite("\r\n",2,1,fp) == 0) return 0;
-    return 1;
-}
-
-/* Write a multi bulk count in the form "*<count>\r\n" */
-int fwriteBulkCount(FILE *fp, char prefix, int count) {
-    char cbuf[128];
-    int clen;
-
-    cbuf[0] = prefix;
-    clen = 1+ll2string(cbuf+1,sizeof(cbuf)-1,count);
-    cbuf[clen++] = '\r';
-    cbuf[clen++] = '\n';
-    if (fwrite(cbuf,clen,1,fp) == 0) return 0;
-    return 1;
-}
-
-/* Write a double value in bulk format $<count>\r\n<payload>\r\n */
-int fwriteBulkDouble(FILE *fp, double d) {
-    char buf[128], dbuf[128];
-
-    snprintf(dbuf,sizeof(dbuf),"%.17g\r\n",d);
-    snprintf(buf,sizeof(buf),"$%lu\r\n",(unsigned long)strlen(dbuf)-2);
-    if (fwrite(buf,strlen(buf),1,fp) == 0) return 0;
-    if (fwrite(dbuf,strlen(dbuf),1,fp) == 0) return 0;
-    return 1;
-}
-
-/* Write a long value in bulk format $<count>\r\n<payload>\r\n */
-int fwriteBulkLongLong(FILE *fp, long long l) {
-    char bbuf[128], lbuf[128];
-    unsigned int blen, llen;
-    llen = ll2string(lbuf,32,l);
-    blen = snprintf(bbuf,sizeof(bbuf),"$%u\r\n%s\r\n",llen,lbuf);
-    if (fwrite(bbuf,blen,1,fp) == 0) return 0;
-    return 1;
-}
-
-/* Delegate writing an object to writing a bulk string or bulk long long. */
-int fwriteBulkObject(FILE *fp, robj *obj) {
-    /* Avoid using getDecodedObject to help copy-on-write (we are often
-     * in a child process when this function is called). */
-    if (obj->encoding == REDIS_ENCODING_INT) {
-        return fwriteBulkLongLong(fp,(long)obj->ptr);
-    } else if (obj->encoding == REDIS_ENCODING_RAW) {
-        return fwriteBulkString(fp,obj->ptr,sdslen(obj->ptr));
-    } else {
-        redisPanic("Unknown string encoding");
-    }
-}
-
-