aof.c 48.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
/*
 * Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *   * Redistributions of source code must retain the above copyright notice,
 *     this list of conditions and the following disclaimer.
 *   * Redistributions in binary form must reproduce the above copyright
 *     notice, this list of conditions and the following disclaimer in the
 *     documentation and/or other materials provided with the distribution.
 *   * Neither the name of Redis nor the names of its contributors may be used
 *     to endorse or promote products derived from this software without
 *     specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

30
#include "redis.h"
31
#include "bio.h"
A
antirez 已提交
32
#include "rio.h"
33 34 35 36

#include <signal.h>
#include <fcntl.h>
#include <sys/stat.h>
37
#include <sys/types.h>
H
Henry Rawas 已提交
38
#ifndef _WIN32
39 40 41
#include <sys/time.h>
#include <sys/resource.h>
#include <sys/wait.h>
H
Henry Rawas 已提交
42
#endif
43

44 45
void aofUpdateCurrentSize(void);

46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79
/* ----------------------------------------------------------------------------
 * AOF rewrite buffer implementation.
 *
 * The following code implement a simple buffer used in order to accumulate
 * changes while the background process is rewriting the AOF file.
 *
 * We only need to append, but can't just use realloc with a large block
 * because 'huge' reallocs are not always handled as one could expect
 * (via remapping of pages at OS level) but may involve copying data.
 *
 * For this reason we use a list of blocks, every block is
 * AOF_RW_BUF_BLOCK_SIZE bytes.
 * ------------------------------------------------------------------------- */

#define AOF_RW_BUF_BLOCK_SIZE (1024*1024*10)    /* 10 MB per block */

typedef struct aofrwblock {
    unsigned long used, free;
    char buf[AOF_RW_BUF_BLOCK_SIZE];
} aofrwblock;

/* This function free the old AOF rewrite buffer if needed, and initialize
 * a fresh new one. It tests for server.aof_rewrite_buf_blocks equal to NULL
 * so can be used for the first initialization as well. */
void aofRewriteBufferReset(void) {
    if (server.aof_rewrite_buf_blocks)
        listRelease(server.aof_rewrite_buf_blocks);

    server.aof_rewrite_buf_blocks = listCreate();
    listSetFreeMethod(server.aof_rewrite_buf_blocks,zfree);
}

/* Return the current size of the AOF rerwite buffer. */
unsigned long aofRewriteBufferSize(void) {
H
Henry Rawas 已提交
80
    unsigned long size;
81 82 83 84
    listNode *ln = listLast(server.aof_rewrite_buf_blocks);
    aofrwblock *block = ln ? ln->value : NULL;

    if (block == NULL) return 0;
H
Henry Rawas 已提交
85
    size =
86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155
        (listLength(server.aof_rewrite_buf_blocks)-1) * AOF_RW_BUF_BLOCK_SIZE;
    size += block->used;
    return size;
}

/* Append data to the AOF rewrite buffer, allocating new blocks if needed. */
void aofRewriteBufferAppend(unsigned char *s, unsigned long len) {
    listNode *ln = listLast(server.aof_rewrite_buf_blocks);
    aofrwblock *block = ln ? ln->value : NULL;

    while(len) {
        /* If we already got at least an allocated block, try appending
         * at least some piece into it. */
        if (block) {
            unsigned long thislen = (block->free < len) ? block->free : len;
            if (thislen) {  /* The current block is not already full. */
                memcpy(block->buf+block->used, s, thislen);
                block->used += thislen;
                block->free -= thislen;
                s += thislen;
                len -= thislen;
            }
        }

        if (len) { /* First block to allocate, or need another block. */
            int numblocks;

            block = zmalloc(sizeof(*block));
            block->free = AOF_RW_BUF_BLOCK_SIZE;
            block->used = 0;
            listAddNodeTail(server.aof_rewrite_buf_blocks,block);

            /* Log every time we cross more 10 or 100 blocks, respectively
             * as a notice or warning. */
            numblocks = listLength(server.aof_rewrite_buf_blocks);
            if (((numblocks+1) % 10) == 0) {
                int level = ((numblocks+1) % 100) == 0 ? REDIS_WARNING :
                                                         REDIS_NOTICE;
                redisLog(level,"Background AOF buffer size: %lu MB",
                    aofRewriteBufferSize()/(1024*1024));
            }
        }
    }
}

/* Write the buffer (possibly composed of multiple blocks) into the specified
 * fd. If no short write or any other error happens -1 is returned,
 * otherwise the number of bytes written is returned. */
ssize_t aofRewriteBufferWrite(int fd) {
    listNode *ln;
    listIter li;
    ssize_t count = 0;

    listRewind(server.aof_rewrite_buf_blocks,&li);
    while((ln = listNext(&li))) {
        aofrwblock *block = listNodeValue(ln);
        ssize_t nwritten;

        if (block->used) {
            nwritten = write(fd,block->buf,block->used);
            if (nwritten != block->used) {
                if (nwritten == 0) errno = EIO;
                return -1;
            }
            count += nwritten;
        }
    }
    return count;
}

156 157 158 159 160 161
/* ----------------------------------------------------------------------------
 * AOF file implementation
 * ------------------------------------------------------------------------- */

/* Starts a background task that performs fsync() against the specified
 * file descriptor (the one of the AOF file) in another thread. */
162
void aof_background_fsync(int fd) {
A
antirez 已提交
163
    bioCreateBackgroundJob(REDIS_BIO_AOF_FSYNC,(void*)(long)fd,NULL,NULL);
164 165
}

166 167 168
/* Called when the user switches from "appendonly yes" to "appendonly no"
 * at runtime using the CONFIG command. */
void stopAppendOnly(void) {
169
    redisAssert(server.aof_state != REDIS_AOF_OFF);
170
    flushAppendOnlyFile(1);
A
antirez 已提交
171 172
    aof_fsync(server.aof_fd);
    close(server.aof_fd);
173

A
antirez 已提交
174 175
    server.aof_fd = -1;
    server.aof_selected_db = -1;
176
    server.aof_state = REDIS_AOF_OFF;
177
    /* rewrite operation in progress? kill it, wait child exit */
A
antirez 已提交
178
    if (server.aof_child_pid != -1) {
H
Henry Rawas 已提交
179 180
        redisLog(REDIS_NOTICE,"Killing running AOF rewrite child: %ld",
            (long) server.aof_child_pid);
181 182
#ifdef _WIN32
        AbortForkOperation();
H
Henry Rawas 已提交
183
#else
184 185 186 187 188
        {
            int statloc;
            if (kill(server.aof_child_pid,SIGUSR1) != -1)
                wait3(&statloc,0,NULL);
        }
H
Henry Rawas 已提交
189
#endif
190
        /* reset the buffer accumulating changes while the child saves */
191
        aofRewriteBufferReset();
A
antirez 已提交
192 193
        aofRemoveTempFile(server.aof_child_pid);
        server.aof_child_pid = -1;
194
        server.aof_rewrite_time_start = -1;
195 196 197 198 199 200
    }
}

/* Called when the user switches from "appendonly no" to "appendonly yes"
 * at runtime using the CONFIG command. */
int startAppendOnly(void) {
201
    server.aof_last_fsync = server.unixtime;
H
Henry Rawas 已提交
202 203 204
#ifdef _WIN32
    server.aof_fd = open(server.aof_filename,O_WRONLY|O_APPEND|O_CREAT|_O_BINARY,_S_IREAD|_S_IWRITE);
#else
A
antirez 已提交
205
    server.aof_fd = open(server.aof_filename,O_WRONLY|O_APPEND|O_CREAT,0644);
H
Henry Rawas 已提交
206
#endif
207
    redisAssert(server.aof_state == REDIS_AOF_OFF);
A
antirez 已提交
208
    if (server.aof_fd == -1) {
209
        redisLog(REDIS_WARNING,"Redis needs to enable the AOF but can't open the append only file: %s",strerror(errno));
210 211 212
        return REDIS_ERR;
    }
    if (rewriteAppendOnlyFileBackground() == REDIS_ERR) {
A
antirez 已提交
213
        close(server.aof_fd);
214
        server.aof_fd = -1;
215
        redisLog(REDIS_WARNING,"Redis needs to enable the AOF but can't trigger a background AOF rewrite operation. Check the above logs for more info about the error.");
216 217
        return REDIS_ERR;
    }
218 219
    /* We correctly switched on AOF, now wait for the rerwite to be complete
     * in order to append data on disk. */
220
    server.aof_state = REDIS_AOF_WAIT_REWRITE;
221 222 223 224 225 226 227 228 229
    return REDIS_OK;
}

/* Write the append only file buffer on disk.
 *
 * Since we are required to write the AOF before replying to the client,
 * and the only way the client socket can get a write is entering when the
 * the event loop, we accumulate all the AOF writes in a memory
 * buffer and write it on disk using this function just before entering
230 231 232 233 234 235 236 237 238 239 240 241 242
 * the event loop again.
 *
 * About the 'force' argument:
 *
 * When the fsync policy is set to 'everysec' we may delay the flush if there
 * is still an fsync() going on in the background thread, since for instance
 * on Linux write(2) will be blocked by the background fsync anyway.
 * When this happens we remember that there is some aof buffer to be
 * flushed ASAP, and will try to do that in the serverCron() function.
 *
 * However if force is set to 1 we'll write regardless of the background
 * fsync. */
void flushAppendOnlyFile(int force) {
243
    ssize_t nwritten;
244
    int sync_in_progress = 0;
245

A
antirez 已提交
246
    if (sdslen(server.aof_buf) == 0) return;
247

248
    if (server.aof_fsync == AOF_FSYNC_EVERYSEC)
249 250
        sync_in_progress = bioPendingJobsOfType(REDIS_BIO_AOF_FSYNC) != 0;

251
    if (server.aof_fsync == AOF_FSYNC_EVERYSEC && !force) {
252 253 254 255 256 257 258 259 260 261
        /* With this append fsync policy we do background fsyncing.
         * If the fsync is still in progress we can try to delay
         * the write for a couple of seconds. */
        if (sync_in_progress) {
            if (server.aof_flush_postponed_start == 0) {
                /* No previous write postponinig, remember that we are
                 * postponing the flush and return. */
                server.aof_flush_postponed_start = server.unixtime;
                return;
            } else if (server.unixtime - server.aof_flush_postponed_start < 2) {
A
antirez 已提交
262
                /* We were already waiting for fsync to finish, but for less
263 264 265 266 267
                 * than two seconds this is still ok. Postpone again. */
                return;
            }
            /* Otherwise fall trough, and go write since we can't wait
             * over two seconds. */
268
            server.aof_delayed_fsync++;
269
            redisLog(REDIS_NOTICE,"Asynchronous AOF fsync is taking too long (disk is busy?). Writing the AOF buffer without waiting for fsync to complete, this may slow down Redis.");
270 271 272 273 274 275
        }
    }
    /* If you are following this code path, then we are going to write so
     * set reset the postponed flush sentinel to zero. */
    server.aof_flush_postponed_start = 0;

276 277 278 279 280
    /* We want to perform a single write. This should be guaranteed atomic
     * at least if the filesystem we are writing is a real physical one.
     * While this will save us against the server being killed I don't think
     * there is much to do about the whole server stopping for power problems
     * or alike */
H
Henry Rawas 已提交
281
    nwritten = write(server.aof_fd,server.aof_buf,(unsigned int)sdslen(server.aof_buf));
A
antirez 已提交
282
    if (nwritten != (signed)sdslen(server.aof_buf)) {
283 284 285
        /* Ooops, we are in troubles. The best thing to do for now is
         * aborting instead of giving the illusion that everything is
         * working as expected. */
P
Pieter Noordhuis 已提交
286
        if (nwritten == -1) {
287
            redisLog(REDIS_WARNING,"Exiting on error writing to the append-only file: %s",strerror(errno));
P
Pieter Noordhuis 已提交
288
        } else {
A
antirez 已提交
289 290 291 292 293 294
            redisLog(REDIS_WARNING,"Exiting on short write while writing to "
                                   "the append-only file: %s (nwritten=%ld, "
                                   "expected=%ld)",
                                   strerror(errno),
                                   (long)nwritten,
                                   (long)sdslen(server.aof_buf));
295 296 297 298 299 300 301

            if (ftruncate(server.aof_fd, server.aof_current_size) == -1) {
                redisLog(REDIS_WARNING, "Could not remove short write "
                         "from the append-only file.  Redis may refuse "
                         "to load the AOF the next time it starts.  "
                         "ftruncate: %s", strerror(errno));
            }
P
Pieter Noordhuis 已提交
302 303
        }
        exit(1);
304
    }
305
    server.aof_current_size += nwritten;
306

307 308
    /* Re-use AOF buffer when it is small enough. The maximum comes from the
     * arena size of 4k minus some overhead (but is otherwise arbitrary). */
A
antirez 已提交
309 310
    if ((sdslen(server.aof_buf)+sdsavail(server.aof_buf)) < 4000) {
        sdsclear(server.aof_buf);
311
    } else {
A
antirez 已提交
312 313
        sdsfree(server.aof_buf);
        server.aof_buf = sdsempty();
314 315
    }

316 317
    /* Don't fsync if no-appendfsync-on-rewrite is set to yes and there are
     * children doing I/O in the background. */
318
    if (server.aof_no_fsync_on_rewrite &&
A
antirez 已提交
319
        (server.aof_child_pid != -1 || server.rdb_child_pid != -1))
320
            return;
321 322

    /* Perform the fsync if needed. */
323
    if (server.aof_fsync == AOF_FSYNC_ALWAYS) {
324 325
        /* aof_fsync is defined as fdatasync() for Linux in order to avoid
         * flushing metadata. */
A
antirez 已提交
326 327
        aof_fsync(server.aof_fd); /* Let's try to get this data on the disk */
        server.aof_last_fsync = server.unixtime;
328
    } else if ((server.aof_fsync == AOF_FSYNC_EVERYSEC &&
A
antirez 已提交
329 330 331
                server.unixtime > server.aof_last_fsync)) {
        if (!sync_in_progress) aof_background_fsync(server.aof_fd);
        server.aof_last_fsync = server.unixtime;
332 333 334
    }
}

335 336 337 338 339 340 341 342 343 344 345
sds catAppendOnlyGenericCommand(sds dst, int argc, robj **argv) {
    char buf[32];
    int len, j;
    robj *o;

    buf[0] = '*';
    len = 1+ll2string(buf+1,sizeof(buf)-1,argc);
    buf[len++] = '\r';
    buf[len++] = '\n';
    dst = sdscatlen(dst,buf,len);

346
    for (j = 0; j < argc; j++) {
347 348 349 350 351 352 353 354
        o = getDecodedObject(argv[j]);
        buf[0] = '$';
        len = 1+ll2string(buf+1,sizeof(buf)-1,sdslen(o->ptr));
        buf[len++] = '\r';
        buf[len++] = '\n';
        dst = sdscatlen(dst,buf,len);
        dst = sdscatlen(dst,o->ptr,sdslen(o->ptr));
        dst = sdscatlen(dst,"\r\n",2);
355 356
        decrRefCount(o);
    }
357
    return dst;
358 359
}

360 361 362 363 364 365 366 367 368
/* Create the sds representation of an PEXPIREAT command, using
 * 'seconds' as time to live and 'cmd' to understand what command
 * we are translating into a PEXPIREAT.
 *
 * This command is used in order to translate EXPIRE and PEXPIRE commands
 * into PEXPIREAT command so that we retain precision in the append only
 * file, and the time is always absolute and not relative. */
sds catAppendOnlyExpireAtCommand(sds buf, struct redisCommand *cmd, robj *key, robj *seconds) {
    long long when;
369 370 371 372
    robj *argv[3];

    /* Make sure we can use strtol */
    seconds = getDecodedObject(seconds);
373 374 375 376 377 378 379 380 381 382 383 384 385
    when = strtoll(seconds->ptr,NULL,10);
    /* Convert argument into milliseconds for EXPIRE, SETEX, EXPIREAT */
    if (cmd->proc == expireCommand || cmd->proc == setexCommand ||
        cmd->proc == expireatCommand)
    {
        when *= 1000;
    }
    /* Convert into absolute time for EXPIRE, PEXPIRE, SETEX, PSETEX */
    if (cmd->proc == expireCommand || cmd->proc == pexpireCommand ||
        cmd->proc == setexCommand || cmd->proc == psetexCommand)
    {
        when += mstime();
    }
386 387
    decrRefCount(seconds);

388
    argv[0] = createStringObject("PEXPIREAT",9);
389
    argv[1] = key;
390 391
    argv[2] = createStringObjectFromLongLong(when);
    buf = catAppendOnlyGenericCommand(buf, 3, argv);
392 393 394 395 396 397
    decrRefCount(argv[0]);
    decrRefCount(argv[2]);
    return buf;
}

void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int argc) {
398
    sds buf = sdsempty();
399 400
    robj *tmpargv[3];

G
guiquanz 已提交
401
    /* The DB this command was targeting is not the same as the last command
402
     * we appendend. To issue a SELECT command is needed. */
A
antirez 已提交
403
    if (dictid != server.aof_selected_db) {
404 405 406 407 408
        char seldb[64];

        snprintf(seldb,sizeof(seldb),"%d",dictid);
        buf = sdscatprintf(buf,"*2\r\n$6\r\nSELECT\r\n$%lu\r\n%s\r\n",
            (unsigned long)strlen(seldb),seldb);
A
antirez 已提交
409
        server.aof_selected_db = dictid;
410 411
    }

412 413 414 415 416 417
    if (cmd->proc == expireCommand || cmd->proc == pexpireCommand ||
        cmd->proc == expireatCommand) {
        /* Translate EXPIRE/PEXPIRE/EXPIREAT into PEXPIREAT */
        buf = catAppendOnlyExpireAtCommand(buf,cmd,argv[1],argv[2]);
    } else if (cmd->proc == setexCommand || cmd->proc == psetexCommand) {
        /* Translate SETEX/PSETEX to SET and PEXPIREAT */
418 419 420 421 422
        tmpargv[0] = createStringObject("SET",3);
        tmpargv[1] = argv[1];
        tmpargv[2] = argv[3];
        buf = catAppendOnlyGenericCommand(buf,3,tmpargv);
        decrRefCount(tmpargv[0]);
423
        buf = catAppendOnlyExpireAtCommand(buf,cmd,argv[1],argv[2]);
424
    } else {
425 426 427
        /* All the other commands don't need translation or need the
         * same translation already operated in the command vector
         * for the replication itself. */
428 429 430 431 432
        buf = catAppendOnlyGenericCommand(buf,argc,argv);
    }

    /* Append to the AOF buffer. This will be flushed on disk just before
     * of re-entering the event loop, so before the client will get a
433 434
     * positive reply about the operation performed. */
    if (server.aof_state == REDIS_AOF_ON)
A
antirez 已提交
435
        server.aof_buf = sdscatlen(server.aof_buf,buf,sdslen(buf));
436 437 438 439 440

    /* If a background append only file rewriting is in progress we want to
     * accumulate the differences between the child DB and the current one
     * in a buffer, so that when the child process will do its work we
     * can append the differences to the new append only file. */
A
antirez 已提交
441
    if (server.aof_child_pid != -1)
H
Henry Rawas 已提交
442
        aofRewriteBufferAppend((unsigned char*)buf,(unsigned long)sdslen(buf));
443 444 445 446

    sdsfree(buf);
}

447 448 449 450
/* ----------------------------------------------------------------------------
 * AOF loading
 * ------------------------------------------------------------------------- */

451 452 453 454 455 456 457
/* In Redis commands are always executed in the context of a client, so in
 * order to load the append only file we need to create a fake client. */
struct redisClient *createFakeClient(void) {
    struct redisClient *c = zmalloc(sizeof(*c));

    selectDb(c,0);
    c->fd = -1;
458
    c->name = NULL;
459
    c->querybuf = sdsempty();
460
    c->querybuf_peak = 0;
461 462
    c->argc = 0;
    c->argv = NULL;
463
    c->bufpos = 0;
464 465 466 467 468
    c->flags = 0;
    /* We set the fake client as a slave waiting for the synchronization
     * so that Redis will not try to send replies to this client. */
    c->replstate = REDIS_REPL_WAIT_BGSAVE_START;
    c->reply = listCreate();
469
    c->reply_bytes = 0;
470
    c->obuf_soft_limit_reached_time = 0;
471
    c->watched_keys = listCreate();
472 473 474 475 476 477 478 479 480
    listSetFreeMethod(c->reply,decrRefCount);
    listSetDupMethod(c->reply,dupClientReplyValue);
    initClientMultiState(c);
    return c;
}

void freeFakeClient(struct redisClient *c) {
    sdsfree(c->querybuf);
    listRelease(c->reply);
481
    listRelease(c->watched_keys);
482 483 484 485 486 487 488 489 490
    freeClientMultiState(c);
    zfree(c);
}

/* Replay the append log file. On error REDIS_OK is returned. On non fatal
 * error (the append only file is zero-length) REDIS_ERR is returned. On
 * fatal error an error message is logged and the program exists. */
int loadAppendOnlyFile(char *filename) {
    struct redisClient *fakeClient;
H
Henry Rawas 已提交
491 492 493
#ifdef _WIN32
    FILE *fp = fopen(filename,"rb");
#else
494
    FILE *fp = fopen(filename,"r");
H
Henry Rawas 已提交
495
#endif
496
    struct redis_stat sb;
497
    int old_aof_state = server.aof_state;
498
    long loops = 0;
499

A
antirez 已提交
500
    if (fp && redis_fstat(fileno(fp),&sb) != -1 && sb.st_size == 0) {
501
        server.aof_current_size = 0;
A
antirez 已提交
502
        fclose(fp);
503
        return REDIS_ERR;
A
antirez 已提交
504
    }
505 506 507 508 509 510 511 512

    if (fp == NULL) {
        redisLog(REDIS_WARNING,"Fatal error: can't open the append log file for reading: %s",strerror(errno));
        exit(1);
    }

    /* Temporarily disable AOF, to prevent EXEC from feeding a MULTI
     * to the same file we're about to read. */
513
    server.aof_state = REDIS_AOF_OFF;
514 515

    fakeClient = createFakeClient();
516 517
    startLoading(fp);

518 519
    while(1) {
        int argc, j;
H
Henry Rawas 已提交
520 521 522
#ifdef _WIN32
        size_t len;
#else
523
        unsigned long len;
H
Henry Rawas 已提交
524
#endif
525 526 527 528 529
        robj **argv;
        char buf[128];
        sds argsds;
        struct redisCommand *cmd;

530 531
        /* Serve the clients from time to time */
        if (!(loops++ % 1000)) {
H
Henry Rawas 已提交
532
            loadingProgress((off_t)ftello(fp));
533 534 535
            aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT);
        }

536 537 538 539 540 541 542 543
        if (fgets(buf,sizeof(buf),fp) == NULL) {
            if (feof(fp))
                break;
            else
                goto readerr;
        }
        if (buf[0] != '*') goto fmterr;
        argc = atoi(buf+1);
544 545
        if (argc < 1) goto fmterr;

546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566
        argv = zmalloc(sizeof(robj*)*argc);
        for (j = 0; j < argc; j++) {
            if (fgets(buf,sizeof(buf),fp) == NULL) goto readerr;
            if (buf[0] != '$') goto fmterr;
            len = strtol(buf+1,NULL,10);
            argsds = sdsnewlen(NULL,len);
            if (len && fread(argsds,len,1,fp) == 0) goto fmterr;
            argv[j] = createObject(REDIS_STRING,argsds);
            if (fread(buf,2,1,fp) == 0) goto fmterr; /* discard CRLF */
        }

        /* Command lookup */
        cmd = lookupCommand(argv[0]->ptr);
        if (!cmd) {
            redisLog(REDIS_WARNING,"Unknown command '%s' reading the append only file", argv[0]->ptr);
            exit(1);
        }
        /* Run the command in the context of a fake client */
        fakeClient->argc = argc;
        fakeClient->argv = argv;
        cmd->proc(fakeClient);
567 568 569

        /* The fake client should not have a reply */
        redisAssert(fakeClient->bufpos == 0 && listLength(fakeClient->reply) == 0);
570 571
        /* The fake client should never get blocked */
        redisAssert((fakeClient->flags & REDIS_BLOCKED) == 0);
572

573 574 575 576 577
        /* Clean up. Command code may have changed argv/argc so we use the
         * argv/argc of the client instead of the local variables. */
        for (j = 0; j < fakeClient->argc; j++)
            decrRefCount(fakeClient->argv[j]);
        zfree(fakeClient->argv);
578 579 580 581 582 583 584 585
    }

    /* This point can only be reached when EOF is reached without errors.
     * If the client is in the middle of a MULTI/EXEC, log error and quit. */
    if (fakeClient->flags & REDIS_MULTI) goto readerr;

    fclose(fp);
    freeFakeClient(fakeClient);
586
    server.aof_state = old_aof_state;
587
    stopLoading();
588
    aofUpdateCurrentSize();
589
    server.aof_rewrite_base_size = server.aof_current_size;
590 591 592 593 594 595 596 597 598 599
    return REDIS_OK;

readerr:
    if (feof(fp)) {
        redisLog(REDIS_WARNING,"Unexpected end of file reading the append only file");
    } else {
        redisLog(REDIS_WARNING,"Unrecoverable error reading the append only file: %s", strerror(errno));
    }
    exit(1);
fmterr:
A
antirez 已提交
600
    redisLog(REDIS_WARNING,"Bad file format reading the append only file: make a backup of your AOF file, then use ./redis-check-aof --fix <filename>");
601 602 603
    exit(1);
}

604 605 606 607
/* ----------------------------------------------------------------------------
 * AOF rewrite
 * ------------------------------------------------------------------------- */

P
Pieter Noordhuis 已提交
608 609 610 611 612 613
/* Delegate writing an object to writing a bulk string or bulk long long.
 * This is not placed in rio.c since that adds the redis.h dependency. */
int rioWriteBulkObject(rio *r, robj *obj) {
    /* Avoid using getDecodedObject to help copy-on-write (we are often
     * in a child process when this function is called). */
    if (obj->encoding == REDIS_ENCODING_INT) {
H
Henry Rawas 已提交
614
        return (int)rioWriteBulkLongLong(r,(long)obj->ptr);
P
Pieter Noordhuis 已提交
615
    } else if (obj->encoding == REDIS_ENCODING_RAW) {
H
Henry Rawas 已提交
616
        return (int)rioWriteBulkString(r,obj->ptr,sdslen(obj->ptr));
P
Pieter Noordhuis 已提交
617 618 619 620 621
    } else {
        redisPanic("Unknown string encoding");
    }
}

622 623 624 625 626 627 628 629 630 631 632 633 634 635
/* Emit the commands needed to rebuild a list object.
 * The function returns 0 on error, 1 on success. */
int rewriteListObject(rio *r, robj *key, robj *o) {
    long long count = 0, items = listTypeLength(o);

    if (o->encoding == REDIS_ENCODING_ZIPLIST) {
        unsigned char *zl = o->ptr;
        unsigned char *p = ziplistIndex(zl,0);
        unsigned char *vstr;
        unsigned int vlen;
        long long vlong;

        while(ziplistGet(p,&vstr,&vlen,&vlong)) {
            if (count == 0) {
636
                int cmd_items = (items > REDIS_AOF_REWRITE_ITEMS_PER_CMD) ?
H
Henry Rawas 已提交
637
                    REDIS_AOF_REWRITE_ITEMS_PER_CMD : (int)items;
638

639 640 641 642 643 644 645 646 647 648
                if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0;
                if (rioWriteBulkString(r,"RPUSH",5) == 0) return 0;
                if (rioWriteBulkObject(r,key) == 0) return 0;
            }
            if (vstr) {
                if (rioWriteBulkString(r,(char*)vstr,vlen) == 0) return 0;
            } else {
                if (rioWriteBulkLongLong(r,vlong) == 0) return 0;
            }
            p = ziplistNext(zl,p);
649
            if (++count == REDIS_AOF_REWRITE_ITEMS_PER_CMD) count = 0;
650 651 652 653 654 655 656 657 658 659 660
            items--;
        }
    } else if (o->encoding == REDIS_ENCODING_LINKEDLIST) {
        list *list = o->ptr;
        listNode *ln;
        listIter li;

        listRewind(list,&li);
        while((ln = listNext(&li))) {
            robj *eleobj = listNodeValue(ln);

661
            if (count == 0) {
662
                int cmd_items = (items > REDIS_AOF_REWRITE_ITEMS_PER_CMD) ?
H
Henry Rawas 已提交
663
                    REDIS_AOF_REWRITE_ITEMS_PER_CMD : (int)items;
664

665 666 667 668
                if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0;
                if (rioWriteBulkString(r,"RPUSH",5) == 0) return 0;
                if (rioWriteBulkObject(r,key) == 0) return 0;
            }
669
            if (rioWriteBulkObject(r,eleobj) == 0) return 0;
670
            if (++count == REDIS_AOF_REWRITE_ITEMS_PER_CMD) count = 0;
671
            items--;
672 673 674 675 676 677 678
        }
    } else {
        redisPanic("Unknown list encoding");
    }
    return 1;
}

679 680 681 682 683 684 685 686 687 688 689
/* Emit the commands needed to rebuild a set object.
 * The function returns 0 on error, 1 on success. */
int rewriteSetObject(rio *r, robj *key, robj *o) {
    long long count = 0, items = setTypeSize(o);

    if (o->encoding == REDIS_ENCODING_INTSET) {
        int ii = 0;
        int64_t llval;

        while(intsetGet(o->ptr,ii++,&llval)) {
            if (count == 0) {
690
                int cmd_items = (items > REDIS_AOF_REWRITE_ITEMS_PER_CMD) ?
H
Henry Rawas 已提交
691
                    REDIS_AOF_REWRITE_ITEMS_PER_CMD : (int)items;
692 693 694 695 696 697

                if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0;
                if (rioWriteBulkString(r,"SADD",4) == 0) return 0;
                if (rioWriteBulkObject(r,key) == 0) return 0;
            }
            if (rioWriteBulkLongLong(r,llval) == 0) return 0;
698
            if (++count == REDIS_AOF_REWRITE_ITEMS_PER_CMD) count = 0;
699 700 701 702 703 704 705 706 707
            items--;
        }
    } else if (o->encoding == REDIS_ENCODING_HT) {
        dictIterator *di = dictGetIterator(o->ptr);
        dictEntry *de;

        while((de = dictNext(di)) != NULL) {
            robj *eleobj = dictGetKey(de);
            if (count == 0) {
708
                int cmd_items = (items > REDIS_AOF_REWRITE_ITEMS_PER_CMD) ?
H
Henry Rawas 已提交
709
                    REDIS_AOF_REWRITE_ITEMS_PER_CMD : (int)items;
710 711 712 713 714 715

                if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0;
                if (rioWriteBulkString(r,"SADD",4) == 0) return 0;
                if (rioWriteBulkObject(r,key) == 0) return 0;
            }
            if (rioWriteBulkObject(r,eleobj) == 0) return 0;
716
            if (++count == REDIS_AOF_REWRITE_ITEMS_PER_CMD) count = 0;
717 718 719 720 721 722 723 724 725
            items--;
        }
        dictReleaseIterator(di);
    } else {
        redisPanic("Unknown set encoding");
    }
    return 1;
}

726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748
/* Emit the commands needed to rebuild a sorted set object.
 * The function returns 0 on error, 1 on success. */
int rewriteSortedSetObject(rio *r, robj *key, robj *o) {
    long long count = 0, items = zsetLength(o);

    if (o->encoding == REDIS_ENCODING_ZIPLIST) {
        unsigned char *zl = o->ptr;
        unsigned char *eptr, *sptr;
        unsigned char *vstr;
        unsigned int vlen;
        long long vll;
        double score;

        eptr = ziplistIndex(zl,0);
        redisAssert(eptr != NULL);
        sptr = ziplistNext(zl,eptr);
        redisAssert(sptr != NULL);

        while (eptr != NULL) {
            redisAssert(ziplistGet(eptr,&vstr,&vlen,&vll));
            score = zzlGetScore(sptr);

            if (count == 0) {
749
                int cmd_items = (items > REDIS_AOF_REWRITE_ITEMS_PER_CMD) ?
H
Henry Rawas 已提交
750
                    REDIS_AOF_REWRITE_ITEMS_PER_CMD : (int)items;
751 752 753 754 755 756 757 758 759 760 761 762

                if (rioWriteBulkCount(r,'*',2+cmd_items*2) == 0) return 0;
                if (rioWriteBulkString(r,"ZADD",4) == 0) return 0;
                if (rioWriteBulkObject(r,key) == 0) return 0;
            }
            if (rioWriteBulkDouble(r,score) == 0) return 0;
            if (vstr != NULL) {
                if (rioWriteBulkString(r,(char*)vstr,vlen) == 0) return 0;
            } else {
                if (rioWriteBulkLongLong(r,vll) == 0) return 0;
            }
            zzlNext(zl,&eptr,&sptr);
763
            if (++count == REDIS_AOF_REWRITE_ITEMS_PER_CMD) count = 0;
764 765 766 767 768 769 770 771 772 773 774 775
            items--;
        }
    } else if (o->encoding == REDIS_ENCODING_SKIPLIST) {
        zset *zs = o->ptr;
        dictIterator *di = dictGetIterator(zs->dict);
        dictEntry *de;

        while((de = dictNext(di)) != NULL) {
            robj *eleobj = dictGetKey(de);
            double *score = dictGetVal(de);

            if (count == 0) {
776
                int cmd_items = (items > REDIS_AOF_REWRITE_ITEMS_PER_CMD) ?
H
Henry Rawas 已提交
777
                    REDIS_AOF_REWRITE_ITEMS_PER_CMD : (int)items;
778 779 780 781 782 783 784

                if (rioWriteBulkCount(r,'*',2+cmd_items*2) == 0) return 0;
                if (rioWriteBulkString(r,"ZADD",4) == 0) return 0;
                if (rioWriteBulkObject(r,key) == 0) return 0;
            }
            if (rioWriteBulkDouble(r,*score) == 0) return 0;
            if (rioWriteBulkObject(r,eleobj) == 0) return 0;
785
            if (++count == REDIS_AOF_REWRITE_ITEMS_PER_CMD) count = 0;
786 787 788 789 790 791 792 793 794
            items--;
        }
        dictReleaseIterator(di);
    } else {
        redisPanic("Unknown sorted zset encoding");
    }
    return 1;
}

795 796 797 798 799 800
/* Write either the key or the value of the currently selected item of an hash.
 * The 'hi' argument passes a valid Redis hash iterator.
 * The 'what' filed specifies if to write a key or a value and can be
 * either REDIS_HASH_KEY or REDIS_HASH_VALUE.
 *
 * The function returns 0 on error, non-zero on success. */
801 802 803 804 805 806 807 808
static int rioWriteHashIteratorCursor(rio *r, hashTypeIterator *hi, int what) {
    if (hi->encoding == REDIS_ENCODING_ZIPLIST) {
        unsigned char *vstr = NULL;
        unsigned int vlen = UINT_MAX;
        long long vll = LLONG_MAX;

        hashTypeCurrentFromZiplist(hi, what, &vstr, &vlen, &vll);
        if (vstr) {
H
Henry Rawas 已提交
809
            return (int)rioWriteBulkString(r, (char*)vstr, vlen);
810
        } else {
H
Henry Rawas 已提交
811
            return (int)rioWriteBulkLongLong(r, vll);
812 813 814 815 816 817 818 819 820 821 822 823 824
        }

    } else if (hi->encoding == REDIS_ENCODING_HT) {
        robj *value;

        hashTypeCurrentFromHashTable(hi, what, &value);
        return rioWriteBulkObject(r, value);
    }

    redisPanic("Unknown hash encoding");
    return 0;
}

A
antirez 已提交
825 826 827
/* Emit the commands needed to rebuild a hash object.
 * The function returns 0 on error, 1 on success. */
int rewriteHashObject(rio *r, robj *key, robj *o) {
828
    hashTypeIterator *hi;
A
antirez 已提交
829 830
    long long count = 0, items = hashTypeLength(o);

831 832 833 834 835
    hi = hashTypeInitIterator(o);
    while (hashTypeNext(hi) != REDIS_ERR) {
        if (count == 0) {
            int cmd_items = (items > REDIS_AOF_REWRITE_ITEMS_PER_CMD) ?
                REDIS_AOF_REWRITE_ITEMS_PER_CMD : items;
A
antirez 已提交
836

837 838 839
            if (rioWriteBulkCount(r,'*',2+cmd_items*2) == 0) return 0;
            if (rioWriteBulkString(r,"HMSET",5) == 0) return 0;
            if (rioWriteBulkObject(r,key) == 0) return 0;
A
antirez 已提交
840 841
        }

842 843 844 845 846
        if (rioWriteHashIteratorCursor(r, hi, REDIS_HASH_KEY) == 0) return 0;
        if (rioWriteHashIteratorCursor(r, hi, REDIS_HASH_VALUE) == 0) return 0;
        if (++count == REDIS_AOF_REWRITE_ITEMS_PER_CMD) count = 0;
        items--;
    }
A
antirez 已提交
847

848
    hashTypeReleaseIterator(hi);
A
antirez 已提交
849 850 851 852

    return 1;
}

853
/* Write a sequence of commands able to fully rebuild the dataset into
854 855 856 857
 * "filename". Used both by REWRITEAOF and BGREWRITEAOF.
 *
 * In order to minimize the number of commands needed in the rewritten
 * log Redis uses variadic commands when possible, such as RPUSH, SADD
858
 * and ZADD. However at max REDIS_AOF_REWRITE_ITEMS_PER_CMD items per time
859
 * are inserted using a single command. */
860 861 862
int rewriteAppendOnlyFile(char *filename) {
    dictIterator *di = NULL;
    dictEntry *de;
P
Pieter Noordhuis 已提交
863
    rio aof;
864 865 866
    FILE *fp;
    char tmpfile[256];
    int j;
867
    long long now = mstime();
H
Henry Rawas 已提交
868
    dictIterator * expIter = NULL;
869 870 871 872

    /* Note that we have to use a different temp name here compared to the
     * one used by rewriteAppendOnlyFileBackground() function. */
    snprintf(tmpfile,256,"temp-rewriteaof-%d.aof", (int) getpid());
H
Henry Rawas 已提交
873 874 875
#ifdef _WIN32
    fp = fopen(tmpfile,"wb");
#else
876
    fp = fopen(tmpfile,"w");
H
Henry Rawas 已提交
877
#endif
878
    if (!fp) {
A
antirez 已提交
879
        redisLog(REDIS_WARNING, "Opening the temp file for AOF rewrite in rewriteAppendOnlyFile(): %s", strerror(errno));
880 881
        return REDIS_ERR;
    }
P
Pieter Noordhuis 已提交
882

883
    rioInitWithFile(&aof,fp);
884 885
    if (server.aof_rewrite_incremental_fsync)
        rioSetAutoSync(&aof,REDIS_AOF_AUTOSYNC_BYTES);
886 887 888
    for (j = 0; j < server.dbnum; j++) {
        char selectcmd[] = "*2\r\n$6\r\nSELECT\r\n";
        redisDb *db = server.db+j;
H
Henry Rawas 已提交
889 890 891
        dict *d;

        d = db->dict;
892
        if (dictSize(d) == 0) continue;
893
        di = dictGetSafeIterator(d);
894 895 896 897 898 899
        if (!di) {
            fclose(fp);
            return REDIS_ERR;
        }

        /* SELECT the new DB */
P
Pieter Noordhuis 已提交
900 901
        if (rioWrite(&aof,selectcmd,sizeof(selectcmd)-1) == 0) goto werr;
        if (rioWriteBulkLongLong(&aof,j) == 0) goto werr;
902 903 904

        /* Iterate this DB writing every entry */
        while((de = dictNext(di)) != NULL) {
A
antirez 已提交
905
            sds keystr;
906
            robj key, *o;
907
            long long expiretime;
908

909 910
            keystr = dictGetKey(de);
            o = dictGetVal(de);
911 912 913
            initStaticStringObject(key,keystr);
            expiretime = getExpire(db,&key);

914 915 916
            /* If this key is already expired skip it */
            if (expiretime != -1 && expiretime < now) continue;

917 918 919 920
            /* Save the key and associated value */
            if (o->type == REDIS_STRING) {
                /* Emit a SET command */
                char cmd[]="*3\r\n$3\r\nSET\r\n";
P
Pieter Noordhuis 已提交
921
                if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr;
922
                /* Key and value */
P
Pieter Noordhuis 已提交
923 924
                if (rioWriteBulkObject(&aof,&key) == 0) goto werr;
                if (rioWriteBulkObject(&aof,o) == 0) goto werr;
925
            } else if (o->type == REDIS_LIST) {
926
                if (rewriteListObject(&aof,&key,o) == 0) goto werr;
927
            } else if (o->type == REDIS_SET) {
928
                if (rewriteSetObject(&aof,&key,o) == 0) goto werr;
929
            } else if (o->type == REDIS_ZSET) {
930
                if (rewriteSortedSetObject(&aof,&key,o) == 0) goto werr;
931
            } else if (o->type == REDIS_HASH) {
A
antirez 已提交
932
                if (rewriteHashObject(&aof,&key,o) == 0) goto werr;
933 934 935 936 937
            } else {
                redisPanic("Unknown object type");
            }
            /* Save the expire time */
            if (expiretime != -1) {
938
                char cmd[]="*3\r\n$9\r\nPEXPIREAT\r\n";
P
Pieter Noordhuis 已提交
939 940
                if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr;
                if (rioWriteBulkObject(&aof,&key) == 0) goto werr;
941
                if (rioWriteBulkLongLong(&aof,expiretime) == 0) goto werr;
942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974
            }
        }
        dictReleaseIterator(di);
    }

    /* Make sure data will not remain on the OS's output buffers */
    fflush(fp);
    aof_fsync(fileno(fp));
    fclose(fp);

    /* Use RENAME to make sure the DB file is changed atomically only
     * if the generate DB file is ok. */
    if (rename(tmpfile,filename) == -1) {
        redisLog(REDIS_WARNING,"Error moving temp append only file on the final destination: %s", strerror(errno));
        unlink(tmpfile);
        return REDIS_ERR;
    }
    redisLog(REDIS_NOTICE,"SYNC append only file rewrite performed");
    return REDIS_OK;

werr:
    fclose(fp);
    unlink(tmpfile);
    redisLog(REDIS_WARNING,"Write error writing append only file on disk: %s", strerror(errno));
    if (di) dictReleaseIterator(di);
    return REDIS_ERR;
}

/* This is how rewriting of the append only file in background works:
 *
 * 1) The user calls BGREWRITEAOF
 * 2) Redis calls this function, that forks():
 *    2a) the child rewrite the append only file in a temp file.
A
antirez 已提交
975
 *    2b) the parent accumulates differences in server.aof_rewrite_buf.
976 977
 * 3) When the child finished '2a' exists.
 * 4) The parent will trap the exit code, if it's OK, will append the
A
antirez 已提交
978
 *    data accumulated into server.aof_rewrite_buf into the temp file, and
979 980 981
 *    finally will rename(2) the temp file in the actual file name.
 *    The the new file is reopened as the new append only file. Profit!
 */
H
Henry Rawas 已提交
982 983
#ifdef _WIN32
int rewriteAppendOnlyFileBackground(void) {
984
    long long start;
H
Henry Rawas 已提交
985 986 987
    char tmpfile[256];

    if (server.aof_child_pid != -1) return REDIS_ERR;
988
    start = ustime();
H
Henry Rawas 已提交
989

990 991
    snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid());
    if (BeginForkOperation(otAOF, tmpfile, &server, sizeof(server), &server.aof_child_pid) == FALSE) {
H
Henry Rawas 已提交
992
            redisLog(REDIS_WARNING,
993
                "Can't rewrite append only file in background: fork: %s",
H
Henry Rawas 已提交
994 995 996
                strerror(errno));
            return REDIS_ERR;
    }
997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008
    server.stat_fork_time = ustime()-start;

    redisLog(REDIS_NOTICE,
        "Background append only file rewriting started by pid %d",server.aof_child_pid);
    server.aof_rewrite_scheduled = 0;
    server.aof_rewrite_time_start = time(NULL);
    updateDictResizePolicy();
    /* We set appendseldb to -1 in order to force the next call to the
        * feedAppendOnlyFile() to issue a SELECT command, so the differences
        * accumulated by the parent into server.aof_rewrite_buf will start
        * with a SELECT statement and it will be safe to merge. */
    server.aof_selected_db = -1;
1009
    return REDIS_OK;
H
Henry Rawas 已提交
1010 1011
}
#else
1012 1013
int rewriteAppendOnlyFileBackground(void) {
    pid_t childpid;
1014
    long long start;
1015

A
antirez 已提交
1016
    if (server.aof_child_pid != -1) return REDIS_ERR;
1017
    start = ustime();
1018 1019 1020
    if ((childpid = fork()) == 0) {
        char tmpfile[256];

1021
        /* Child */
1022 1023
        if (server.ipfd > 0) close(server.ipfd);
        if (server.sofd > 0) close(server.sofd);
1024 1025
        snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid());
        if (rewriteAppendOnlyFile(tmpfile) == REDIS_OK) {
1026 1027 1028 1029 1030 1031 1032
            size_t private_dirty = zmalloc_get_private_dirty();

            if (private_dirty) {
                redisLog(REDIS_NOTICE,
                    "AOF rewrite: %lu MB of memory used by copy-on-write",
                    private_dirty/(1024*1024));
            }
1033
            exitFromChild(0);
1034
        } else {
1035
            exitFromChild(1);
1036 1037 1038
        }
    } else {
        /* Parent */
1039
        server.stat_fork_time = ustime()-start;
1040 1041 1042 1043 1044 1045 1046 1047
        if (childpid == -1) {
            redisLog(REDIS_WARNING,
                "Can't rewrite append only file in background: fork: %s",
                strerror(errno));
            return REDIS_ERR;
        }
        redisLog(REDIS_NOTICE,
            "Background append only file rewriting started by pid %d",childpid);
1048
        server.aof_rewrite_scheduled = 0;
1049
        server.aof_rewrite_time_start = time(NULL);
A
antirez 已提交
1050
        server.aof_child_pid = childpid;
1051 1052 1053
        updateDictResizePolicy();
        /* We set appendseldb to -1 in order to force the next call to the
         * feedAppendOnlyFile() to issue a SELECT command, so the differences
A
antirez 已提交
1054
         * accumulated by the parent into server.aof_rewrite_buf will start
1055
         * with a SELECT statement and it will be safe to merge. */
A
antirez 已提交
1056
        server.aof_selected_db = -1;
1057 1058 1059 1060
        return REDIS_OK;
    }
    return REDIS_OK; /* unreached */
}
H
Henry Rawas 已提交
1061
#endif
1062 1063

void bgrewriteaofCommand(redisClient *c) {
A
antirez 已提交
1064
    if (server.aof_child_pid != -1) {
1065
        addReplyError(c,"Background append only file rewriting already in progress");
A
antirez 已提交
1066
    } else if (server.rdb_child_pid != -1) {
1067
        server.aof_rewrite_scheduled = 1;
1068
        addReplyStatus(c,"Background append only file rewriting scheduled");
1069
    } else if (rewriteAppendOnlyFileBackground() == REDIS_OK) {
1070
        addReplyStatus(c,"Background append only file rewriting started");
1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082
    } else {
        addReply(c,shared.err);
    }
}

void aofRemoveTempFile(pid_t childpid) {
    char tmpfile[256];

    snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) childpid);
    unlink(tmpfile);
}

1083
/* Update the server.aof_current_size filed explicitly using stat(2)
1084 1085
 * to check the size of the file. This is useful after a rewrite or after
 * a restart, normally the size is updated just adding the write length
A
antirez 已提交
1086
 * to the current length, that is much faster. */
1087 1088 1089
void aofUpdateCurrentSize(void) {
    struct redis_stat sb;

H
Henry Rawas 已提交
1090 1091 1092 1093 1094 1095
#ifdef _WIN32
    if (server.aof_fd == -1) {
        redisLog(REDIS_NOTICE,"Unable to check the AOF length: %s", "appendfd is -1");
        return;
    }
#endif
A
antirez 已提交
1096
    if (redis_fstat(server.aof_fd,&sb) == -1) {
A
antirez 已提交
1097
        redisLog(REDIS_WARNING,"Unable to obtain the AOF file length. stat: %s",
1098 1099
            strerror(errno));
    } else {
1100
        server.aof_current_size = sb.st_size;
1101 1102 1103
    }
}

1104 1105
/* A background append only file rewriting (BGREWRITEAOF) terminated its work.
 * Handle this. */
1106
void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
1107
    if (!bysignal && exitcode == 0) {
1108
        int newfd, oldfd;
1109
        char tmpfile[256];
1110
        long long now = ustime();
H
Henry Rawas 已提交
1111 1112 1113
#ifdef _WIN32
        char tmpfile_old[256];
#endif
1114 1115

        redisLog(REDIS_NOTICE,
1116 1117
            "Background AOF rewrite terminated with success");

1118 1119
        /* Flush the differences accumulated by the parent to the
         * rewritten AOF. */
H
Henry Rawas 已提交
1120
#ifdef _WIN32
1121 1122
        snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof",
            getpid());
H
Henry Rawas 已提交
1123 1124
        newfd = open(tmpfile,O_WRONLY|O_APPEND|O_CREAT|_O_BINARY,_S_IREAD|_S_IWRITE);
#else
1125 1126
        snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof",
            (int)server.aof_child_pid);
1127
        newfd = open(tmpfile,O_WRONLY|O_APPEND);
H
Henry Rawas 已提交
1128
#endif
1129 1130 1131
        if (newfd == -1) {
            redisLog(REDIS_WARNING,
                "Unable to open the temporary AOF produced by the child: %s", strerror(errno));
1132 1133
            goto cleanup;
        }
1134

1135 1136 1137
        if (aofRewriteBufferWrite(newfd) == -1) {
            redisLog(REDIS_WARNING,
                "Error trying to flush the parent diff to the rewritten AOF: %s", strerror(errno));
1138
            close(newfd);
1139 1140
            goto cleanup;
        }
1141 1142

        redisLog(REDIS_NOTICE,
1143
            "Parent diff successfully flushed to the rewritten AOF (%lu bytes)", aofRewriteBufferSize());
1144 1145 1146

        /* The only remaining thing to do is to rename the temporary file to
         * the configured file and switch the file descriptor used to do AOF
1147 1148 1149 1150
         * writes. We don't want close(2) or rename(2) calls to block the
         * server on old file deletion.
         *
         * There are two possible scenarios:
1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164
         *
         * 1) AOF is DISABLED and this was a one time rewrite. The temporary
         * file will be renamed to the configured file. When this file already
         * exists, it will be unlinked, which may block the server.
         *
         * 2) AOF is ENABLED and the rewritten AOF will immediately start
         * receiving writes. After the temporary file is renamed to the
         * configured file, the original AOF file descriptor will be closed.
         * Since this will be the last reference to that file, closing it
         * causes the underlying file to be unlinked, which may block the
         * server.
         *
         * To mitigate the blocking effect of the unlink operation (either
         * caused by rename(2) in scenario 1, or by close(2) in scenario 2), we
1165
         * use a background thread to take care of this. First, we
1166 1167 1168 1169 1170 1171
         * make scenario 1 identical to scenario 2 by opening the target file
         * when it exists. The unlink operation after the rename(2) will then
         * be executed upon calling close(2) for its descriptor. Everything to
         * guarantee atomicity for this switch has already happened by then, so
         * we don't care what the outcome or duration of that close operation
         * is, as long as the file descriptor is released again. */
H
Henry Rawas 已提交
1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199
#ifdef _WIN32
        oldfd = -1; /* We'll set this to the current AOF filedes later. */

        /* Close files before renaming */
        close(newfd);
        if (server.aof_fd != -1) close(server.aof_fd);
        /* now rename the existing file to allow new file to be renamed */
        snprintf(tmpfile_old,256,"temp-rewriteaof-old-%d.aof",
            (int)server.aof_child_pid);
        if (server.aof_fd != -1) {
            if (rename(server.aof_filename, tmpfile_old) == -1) {
                redisLog(REDIS_WARNING,
                    "Error trying to rename the existing AOF to old tempfile: %s", strerror(errno));
            }
        }
        if (rename(tmpfile,server.aof_filename) == -1) {
            redisLog(REDIS_WARNING,
                "Error trying to rename the temporary AOF: %s", strerror(errno));
            if (server.aof_fd != -1) {
                if (rename(tmpfile_old, server.aof_filename) == -1) {
                    redisLog(REDIS_WARNING,
                        "Error trying to rename the existing AOF from old tempfile: %s", strerror(errno));
                }
            }
            if (oldfd != -1) close(oldfd);
            goto cleanup;
        }
        /* now open the files again with new names */
1200
        newfd = open(server.aof_filename, O_WRONLY|O_APPEND|_O_BINARY,0);
H
Henry Rawas 已提交
1201 1202 1203 1204 1205 1206 1207 1208 1209
        if (newfd == -1) {
            /* Windows fix: More info */
            redisLog(REDIS_WARNING, "Not able to reopen the temporary AOF file after rename");
            goto cleanup;
        }
        if (server.aof_fd != -1) {
            server.aof_fd = open(tmpfile_old, O_WRONLY|O_APPEND|O_CREAT|_O_BINARY,0644);
        }
#else
A
antirez 已提交
1210
        if (server.aof_fd == -1) {
1211 1212
            /* AOF disabled */

1213 1214 1215
             /* Don't care if this fails: oldfd will be -1 and we handle that.
              * One notable case of -1 return is if the old file does
              * not exist. */
1216
             oldfd = open(server.aof_filename,O_RDONLY|O_NONBLOCK);
1217 1218
        } else {
            /* AOF enabled */
1219
            oldfd = -1; /* We'll set this to the current AOF filedes later. */
1220 1221 1222 1223
        }

        /* Rename the temporary file. This will not unlink the target file if
         * it exists, because we reference it with "oldfd". */
1224
        if (rename(tmpfile,server.aof_filename) == -1) {
1225
            redisLog(REDIS_WARNING,
A
antirez 已提交
1226
                "Error trying to rename the temporary AOF file: %s", strerror(errno));
1227
            close(newfd);
1228
            if (oldfd != -1) close(oldfd);
1229 1230
            goto cleanup;
        }
H
Henry Rawas 已提交
1231
#endif
1232

A
antirez 已提交
1233
        if (server.aof_fd == -1) {
1234 1235
            /* AOF disabled, we don't need to set the AOF file descriptor
             * to this new file, so we can close it. */
1236 1237
            close(newfd);
        } else {
1238
            /* AOF enabled, replace the old fd with the new one. */
A
antirez 已提交
1239 1240
            oldfd = server.aof_fd;
            server.aof_fd = newfd;
1241
            if (server.aof_fsync == AOF_FSYNC_ALWAYS)
1242
                aof_fsync(newfd);
1243
            else if (server.aof_fsync == AOF_FSYNC_EVERYSEC)
1244
                aof_background_fsync(newfd);
A
antirez 已提交
1245
            server.aof_selected_db = -1; /* Make sure SELECT is re-issued */
1246
            aofUpdateCurrentSize();
1247
            server.aof_rewrite_base_size = server.aof_current_size;
1248 1249 1250

            /* Clear regular AOF buffer since its contents was just written to
             * the new AOF from the background rewrite buffer. */
A
antirez 已提交
1251 1252
            sdsfree(server.aof_buf);
            server.aof_buf = sdsempty();
1253
        }
1254

1255 1256
        server.aof_lastbgrewrite_status = REDIS_OK;

A
antirez 已提交
1257
        redisLog(REDIS_NOTICE, "Background AOF rewrite finished successfully");
1258 1259 1260
        /* Change state from WAIT_REWRITE to ON if needed */
        if (server.aof_state == REDIS_AOF_WAIT_REWRITE)
            server.aof_state = REDIS_AOF_ON;
1261 1262

        /* Asynchronously close the overwritten AOF. */
1263
        if (oldfd != -1) bioCreateBackgroundJob(REDIS_BIO_CLOSE_FILE,(void*)(long)oldfd,NULL,NULL);
1264 1265 1266

        redisLog(REDIS_VERBOSE,
            "Background AOF rewrite signal handler took %lldus", ustime()-now);
1267
    } else if (!bysignal && exitcode != 0) {
1268 1269
        server.aof_lastbgrewrite_status = REDIS_ERR;

1270 1271
        redisLog(REDIS_WARNING,
            "Background AOF rewrite terminated with error");
1272
    } else {
1273 1274
        server.aof_lastbgrewrite_status = REDIS_ERR;

1275
        redisLog(REDIS_WARNING,
1276
            "Background AOF rewrite terminated by signal %d", bysignal);
1277
    }
1278

1279
cleanup:
1280
    aofRewriteBufferReset();
A
antirez 已提交
1281 1282
    aofRemoveTempFile(server.aof_child_pid);
    server.aof_child_pid = -1;
1283 1284
    server.aof_rewrite_time_last = time(NULL)-server.aof_rewrite_time_start;
    server.aof_rewrite_time_start = -1;
1285 1286
    /* Schedule a new rewrite if we are waiting for it to switch the AOF ON. */
    if (server.aof_state == REDIS_AOF_WAIT_REWRITE)
1287
        server.aof_rewrite_scheduled = 1;
1288
}