diff --git a/redis.conf b/redis.conf index 507527cc876403e0a667615bc762f049e6a582e8..7c550a532c3aec4041d69e42bde681d7cf10ea13 100644 --- a/redis.conf +++ b/redis.conf @@ -1211,6 +1211,12 @@ hz 10 # big latency spikes. aof-rewrite-incremental-fsync yes +# When redis saves RDB file, if the following option is enabled +# the file will be fsync-ed every 32 MB of data generated. This is useful +# in order to commit the file to the disk more incrementally and avoid +# big latency spikes. +rdb-save-incremental-fsync yes + # Redis LFU eviction (see maxmemory setting) can be tuned. However it is a good # idea to start with the default settings and only change them after investigating # how to improve the performances and how the keys LFU change over time, which diff --git a/src/aof.c b/src/aof.c index 0aa081f0027507487c3df4052ee0944d645990ff..ab50897aa6883f512e245fdc6a13a192d3409ae0 100644 --- a/src/aof.c +++ b/src/aof.c @@ -228,7 +228,7 @@ static void killAppendOnlyChild(void) { void stopAppendOnly(void) { serverAssert(server.aof_state != AOF_OFF); flushAppendOnlyFile(1); - aof_fsync(server.aof_fd); + redis_fsync(server.aof_fd); close(server.aof_fd); server.aof_fd = -1; @@ -476,10 +476,10 @@ void flushAppendOnlyFile(int force) { /* Perform the fsync if needed. */ if (server.aof_fsync == AOF_FSYNC_ALWAYS) { - /* aof_fsync is defined as fdatasync() for Linux in order to avoid + /* redis_fsync is defined as fdatasync() for Linux in order to avoid * flushing metadata. */ latencyStartMonitor(latency); - aof_fsync(server.aof_fd); /* Let's try to get this data on the disk */ + redis_fsync(server.aof_fd); /* Let's try to get this data on the disk */ latencyEndMonitor(latency); latencyAddSampleIfNeeded("aof-fsync-always",latency); server.aof_last_fsync = server.unixtime; @@ -1322,7 +1322,7 @@ int rewriteAppendOnlyFile(char *filename) { rioInitWithFile(&aof,fp); if (server.aof_rewrite_incremental_fsync) - rioSetAutoSync(&aof,AOF_AUTOSYNC_BYTES); + rioSetAutoSync(&aof,REDIS_AUTOSYNC_BYTES); if (server.aof_use_rdb_preamble) { int error; @@ -1690,7 +1690,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { oldfd = server.aof_fd; server.aof_fd = newfd; if (server.aof_fsync == AOF_FSYNC_ALWAYS) - aof_fsync(newfd); + redis_fsync(newfd); else if (server.aof_fsync == AOF_FSYNC_EVERYSEC) aof_background_fsync(newfd); server.aof_selected_db = -1; /* Make sure SELECT is re-issued */ diff --git a/src/bio.c b/src/bio.c index da11f7b868bb4f82136f2be05dd1163047f0a805..0c92d053bcd254189bddb3083778ffeaabf909d6 100644 --- a/src/bio.c +++ b/src/bio.c @@ -187,7 +187,7 @@ void *bioProcessBackgroundJobs(void *arg) { if (type == BIO_CLOSE_FILE) { close((long)job->arg1); } else if (type == BIO_AOF_FSYNC) { - aof_fsync((long)job->arg1); + redis_fsync((long)job->arg1); } else if (type == BIO_LAZY_FREE) { /* What we free changes depending on what arguments are set: * arg1 -> free the object at pointer. diff --git a/src/config.c b/src/config.c index a6e45e0335b3e8c25e7157ed6f864a428a3994d0..2efed153c33bfe5b03c4e8380fa762af019284e2 100644 --- a/src/config.c +++ b/src/config.c @@ -483,6 +483,13 @@ void loadServerConfigFromString(char *config) { yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } + } else if (!strcasecmp(argv[0],"rdb-save-incremental-fsync") && + argc == 2) + { + if ((server.rdb_save_incremental_fsync = + yesnotoi(argv[1])) == -1) { + err = "argument must be 'yes' or 'no'"; goto loaderr; + } } else if (!strcasecmp(argv[0],"aof-load-truncated") && argc == 2) { if ((server.aof_load_truncated = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; @@ -1021,6 +1028,8 @@ void configSetCommand(client *c) { "cluster-slave-no-failover",server.cluster_slave_no_failover) { } config_set_bool_field( "aof-rewrite-incremental-fsync",server.aof_rewrite_incremental_fsync) { + } config_set_bool_field( + "rdb-save-incremental-fsync",server.rdb_save_incremental_fsync) { } config_set_bool_field( "aof-load-truncated",server.aof_load_truncated) { } config_set_bool_field( @@ -1347,6 +1356,8 @@ void configGetCommand(client *c) { server.repl_diskless_sync); config_get_bool_field("aof-rewrite-incremental-fsync", server.aof_rewrite_incremental_fsync); + config_get_bool_field("rdb-save-incremental-fsync", + server.rdb_save_incremental_fsync); config_get_bool_field("aof-load-truncated", server.aof_load_truncated); config_get_bool_field("aof-use-rdb-preamble", @@ -2084,6 +2095,7 @@ int rewriteConfig(char *path) { rewriteConfigClientoutputbufferlimitOption(state); rewriteConfigNumericalOption(state,"hz",server.hz,CONFIG_DEFAULT_HZ); rewriteConfigYesNoOption(state,"aof-rewrite-incremental-fsync",server.aof_rewrite_incremental_fsync,CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC); + rewriteConfigYesNoOption(state,"rdb-save-incremental-fsync",server.rdb_save_incremental_fsync,CONFIG_DEFAULT_RDB_SAVE_INCREMENTAL_FSYNC); rewriteConfigYesNoOption(state,"aof-load-truncated",server.aof_load_truncated,CONFIG_DEFAULT_AOF_LOAD_TRUNCATED); rewriteConfigYesNoOption(state,"aof-use-rdb-preamble",server.aof_use_rdb_preamble,CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE); rewriteConfigEnumOption(state,"supervised",server.supervised_mode,supervised_mode_enum,SUPERVISED_NONE); diff --git a/src/config.h b/src/config.h index c23f1c789385b573b2030af5df1f9be25453be7d..ee3ad508eb6c3e87e1eceec02fe8d78677f884b0 100644 --- a/src/config.h +++ b/src/config.h @@ -87,11 +87,11 @@ #endif #endif -/* Define aof_fsync to fdatasync() in Linux and fsync() for all the rest */ +/* Define redis_fsync to fdatasync() in Linux and fsync() for all the rest */ #ifdef __linux__ -#define aof_fsync fdatasync +#define redis_fsync fdatasync #else -#define aof_fsync fsync +#define redis_fsync fsync #endif /* Define rdb_fsync_range to sync_file_range() on Linux, otherwise we use diff --git a/src/rdb.c b/src/rdb.c index 10566fc55b8e1658ec045259a9f0026ca749e4b3..e37e2f94bac3d8286cb01f1b8de213e9f97eb95f 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1241,6 +1241,10 @@ int rdbSave(char *filename, rdbSaveInfo *rsi) { } rioInitWithFile(&rdb,fp); + + if (server.rdb_save_incremental_fsync) + rioSetAutoSync(&rdb,REDIS_AUTOSYNC_BYTES); + if (rdbSaveRio(&rdb,&error,RDB_SAVE_NONE,rsi) == C_ERR) { errno = error; goto werr; diff --git a/src/rio.c b/src/rio.c index 23b9070605d2acfd43248c4b6d25131d5dd91dbc..c9c76b8f2cef50a3d0ed9e732c51aaa66503f03e 100644 --- a/src/rio.c +++ b/src/rio.c @@ -116,7 +116,7 @@ static size_t rioFileWrite(rio *r, const void *buf, size_t len) { r->io.file.buffered >= r->io.file.autosync) { fflush(r->io.file.fp); - aof_fsync(fileno(r->io.file.fp)); + redis_fsync(fileno(r->io.file.fp)); r->io.file.buffered = 0; } return retval; diff --git a/src/server.c b/src/server.c index 16c807f6107ce28229704e8255bc963e6b4163a4..e7d3c1a6740d95f8698e843b13ebfa5b01923d2d 100644 --- a/src/server.c +++ b/src/server.c @@ -1457,6 +1457,7 @@ void initServerConfig(void) { server.aof_selected_db = -1; /* Make sure the first time will not match */ server.aof_flush_postponed_start = 0; server.aof_rewrite_incremental_fsync = CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC; + server.rdb_save_incremental_fsync = CONFIG_DEFAULT_RDB_SAVE_INCREMENTAL_FSYNC; server.aof_load_truncated = CONFIG_DEFAULT_AOF_LOAD_TRUNCATED; server.aof_use_rdb_preamble = CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE; server.pidfile = NULL; @@ -2638,7 +2639,7 @@ int prepareForShutdown(int flags) { /* Append only file: flush buffers and fsync() the AOF at exit */ serverLog(LL_NOTICE,"Calling fsync() on the AOF file."); flushAppendOnlyFile(1); - aof_fsync(server.aof_fd); + redis_fsync(server.aof_fd); } /* Create a new RDB file before exiting. */ diff --git a/src/server.h b/src/server.h index f5089438d2f077db7e3785c6044f004970884fc0..264380d8e2947d7e8b149f2967c976bc2012f905 100644 --- a/src/server.h +++ b/src/server.h @@ -142,6 +142,7 @@ typedef long long mstime_t; /* millisecond time type. */ #define CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE 1 #define CONFIG_DEFAULT_ACTIVE_REHASHING 1 #define CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC 1 +#define CONFIG_DEFAULT_RDB_SAVE_INCREMENTAL_FSYNC 1 #define CONFIG_DEFAULT_MIN_SLAVES_TO_WRITE 0 #define CONFIG_DEFAULT_MIN_SLAVES_MAX_LAG 10 #define NET_IP_STR_LEN 46 /* INET6_ADDRSTRLEN is 46, but we need to be sure */ @@ -183,7 +184,7 @@ typedef long long mstime_t; /* millisecond time type. */ #define PROTO_INLINE_MAX_SIZE (1024*64) /* Max size of inline reads */ #define PROTO_MBULK_BIG_ARG (1024*32) #define LONG_STR_SIZE 21 /* Bytes needed for long -> str + '\0' */ -#define AOF_AUTOSYNC_BYTES (1024*1024*32) /* fdatasync every 32MB */ +#define REDIS_AUTOSYNC_BYTES (1024*1024*32) /* fdatasync every 32MB */ /* When configuring the server eventloop, we setup it so that the total number * of file descriptors we can handle are server.maxclients + RESERVED_FDS + @@ -1046,7 +1047,8 @@ struct redisServer { time_t aof_rewrite_time_start; /* Current AOF rewrite start time. */ int aof_lastbgrewrite_status; /* C_OK or C_ERR */ unsigned long aof_delayed_fsync; /* delayed AOF fsync() counter */ - int aof_rewrite_incremental_fsync;/* fsync incrementally while rewriting? */ + int aof_rewrite_incremental_fsync;/* fsync incrementally while aof rewriting? */ + int rdb_save_incremental_fsync; /* fsync incrementally while rdb saving? */ int aof_last_write_status; /* C_OK or C_ERR */ int aof_last_write_errno; /* Valid if aof_last_write_status is ERR */ int aof_load_truncated; /* Don't stop on unexpected AOF EOF. */