Max limit to 10k clients removed, this implements feature request on issue #194

src/ae.c

@@ -52,12 +52,22 @@
     #endif
 #endif
 
-aeEventLoop *aeCreateEventLoop(void) {
+aeEventLoop *aeCreateEventLoop(int setsize) {
     aeEventLoop *eventLoop;
     int i;
 
-    eventLoop = zmalloc(sizeof(*eventLoop));
-    if (!eventLoop) return NULL;
+    if ((eventLoop = zmalloc(sizeof(*eventLoop))) == NULL) return NULL;
+    eventLoop->events = NULL;
+    eventLoop->fired = NULL;
+    eventLoop->events = zmalloc(sizeof(aeFileEvent)*setsize);
+    eventLoop->fired = zmalloc(sizeof(aeFiredEvent)*setsize);
+    if (eventLoop->events == NULL || eventLoop->fired == NULL) {
+        zfree(eventLoop->events);
+        zfree(eventLoop->fired);
+        zfree(eventLoop);
+        return NULL;
+    }
+    eventLoop->setsize = setsize;
     eventLoop->timeEventHead = NULL;
     eventLoop->timeEventNextId = 0;
     eventLoop->stop = 0;
@@ -69,7 +79,7 @@ aeEventLoop *aeCreateEventLoop(void) {
     }
     /* Events with mask == AE_NONE are not set. So let's initialize the
      * vector with it. */
-    for (i = 0; i < AE_SETSIZE; i++)
+    for (i = 0; i < setsize; i++)
         eventLoop->events[i].mask = AE_NONE;
     return eventLoop;
 }
@@ -86,7 +96,7 @@ void aeStop(aeEventLoop *eventLoop) {
 int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask,
         aeFileProc *proc, void *clientData)
 {
-    if (fd >= AE_SETSIZE) return AE_ERR;
+    if (fd >= eventLoop->setsize) return AE_ERR;
     aeFileEvent *fe = &eventLoop->events[fd];
 
     if (aeApiAddEvent(eventLoop, fd, mask) == -1)
@@ -102,7 +112,7 @@ int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask,
 
 void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask)
 {
-    if (fd >= AE_SETSIZE) return;
+    if (fd >= eventLoop->setsize) return;
     aeFileEvent *fe = &eventLoop->events[fd];
 
     if (fe->mask == AE_NONE) return;
@@ -119,7 +129,7 @@ void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask)
 }
 
 int aeGetFileEvents(aeEventLoop *eventLoop, int fd) {
-    if (fd >= AE_SETSIZE) return 0;
+    if (fd >= eventLoop->setsize) return 0;
     aeFileEvent *fe = &eventLoop->events[fd];
 
     return fe->mask;

src/ae.h

@@ -33,8 +33,6 @@
 #ifndef __AE_H__
 #define __AE_H__
 
-#define AE_SETSIZE (1024*10)    /* Max number of fd supported */
-
 #define AE_OK 0
 #define AE_ERR -1
 
@@ -87,10 +85,11 @@ typedef struct aeFiredEvent {
 
 /* State of an event based program */
 typedef struct aeEventLoop {
-    int maxfd;
+    int maxfd;   /* highest file descriptor currently registered */
+    int setsize; /* max number of file descriptors tracked */
     long long timeEventNextId;
-    aeFileEvent events[AE_SETSIZE]; /* Registered events */
-    aeFiredEvent fired[AE_SETSIZE]; /* Fired events */
+    aeFileEvent *events; /* Registered events */
+    aeFiredEvent *fired; /* Fired events */
     aeTimeEvent *timeEventHead;
     int stop;
     void *apidata; /* This is used for polling API specific data */
@@ -98,7 +97,7 @@ typedef struct aeEventLoop {
 } aeEventLoop;
 
 /* Prototypes */
-aeEventLoop *aeCreateEventLoop(void);
+aeEventLoop *aeCreateEventLoop(int setsize);
 void aeDeleteEventLoop(aeEventLoop *eventLoop);
 void aeStop(aeEventLoop *eventLoop);
 int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask,

src/ae_epoll.c

@@ -6,15 +6,21 @@
 
 typedef struct aeApiState {
     int epfd;
-    struct epoll_event events[AE_SETSIZE];
+    struct epoll_event *events;
 } aeApiState;
 
 static int aeApiCreate(aeEventLoop *eventLoop) {
     aeApiState *state = zmalloc(sizeof(aeApiState));
 
     if (!state) return -1;
+    state->events = zmalloc(sizeof(epoll_event)*eventLoop->setsize);
+    if (!state->events) {
+        zfree(state);
+        return -1;
+    }
     state->epfd = epoll_create(1024); /* 1024 is just an hint for the kernel */
     if (state->epfd == -1) {
+        zfree(state->events);
         zfree(state);
         return -1;
     }
@@ -26,6 +32,7 @@ static void aeApiFree(aeEventLoop *eventLoop) {
     aeApiState *state = eventLoop->apidata;
 
     close(state->epfd);
+    zfree(state->events);
     zfree(state);
 }
 
@@ -70,7 +77,7 @@ static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
     aeApiState *state = eventLoop->apidata;
     int retval, numevents = 0;
 
-    retval = epoll_wait(state->epfd,state->events,AE_SETSIZE,
+    retval = epoll_wait(state->epfd,state->events,eventLoop->setsize,
             tvp ? (tvp->tv_sec*1000 + tvp->tv_usec/1000) : -1);
     if (retval > 0) {
         int j;

src/ae_kqueue.c

@@ -8,15 +8,21 @@
 
 typedef struct aeApiState {
     int kqfd;
-    struct kevent events[AE_SETSIZE];
+    struct kevent *events;
 } aeApiState;
 
 static int aeApiCreate(aeEventLoop *eventLoop) {
     aeApiState *state = zmalloc(sizeof(aeApiState));
 
     if (!state) return -1;
+    state->events = zmalloc(sizeof(struct kevent)*eventLoop->setsize);
+    if (!state->events) {
+        zfree(state);
+        return -1;
+    }
     state->kqfd = kqueue();
     if (state->kqfd == -1) {
+        zfree(state->events);
         zfree(state);
         return -1;
     }
@@ -29,6 +35,7 @@ static void aeApiFree(aeEventLoop *eventLoop) {
     aeApiState *state = eventLoop->apidata;
 
     close(state->kqfd);
+    zfree(state->events);
     zfree(state);
 }
 
@@ -69,10 +76,12 @@ static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
         struct timespec timeout;
         timeout.tv_sec = tvp->tv_sec;
         timeout.tv_nsec = tvp->tv_usec * 1000;
-        retval = kevent(state->kqfd, NULL, 0, state->events, AE_SETSIZE, &timeout);
+        retval = kevent(state->kqfd, NULL, 0, state->events, eventLoop->setsize,
+                        &timeout);
     } else {
-        retval = kevent(state->kqfd, NULL, 0, state->events, AE_SETSIZE, NULL);
-    }    
+        retval = kevent(state->kqfd, NULL, 0, state->events, eventLoop->setsize,
+                        NULL);
+    }
 
     if (retval > 0) {
         int j;

src/redis-benchmark.c

@@ -490,7 +490,7 @@ int main(int argc, const char **argv) {
     config.numclients = 50;
     config.requests = 10000;
     config.liveclients = 0;
-    config.el = aeCreateEventLoop();
+    config.el = aeCreateEventLoop(1024*10);
     aeCreateTimeEvent(config.el,1,showThroughput,NULL,NULL);
     config.keepalive = 1;
     config.datasize = 3;

src/redis.c

@@ -954,6 +954,43 @@ void initServerConfig() {
     server.bug_report_start = 0;
 }
 
+/* This function will try to raise the max number of open files accordingly to
+ * the configured max number of clients. It will also account for 32 additional
+ * file descriptors as we need a few more for persistence, listening
+ * sockets, log files and so forth.
+ *
+ * If it will not be possible to set the limit accordingly to the configured
+ * max number of clients, the function will do the reverse setting
+ * server.maxclients to the value that we can actually handle. */
+void adjustOpenFilesLimit(void) {
+    rlim_t maxfiles = server.maxclients+32;
+    struct rlimit limit;
+
+    if (maxfiles < 1024) maxfiles = 1024;
+    if (getrlimit(RLIMIT_NOFILE,&limit) == -1) {
+        redisLog(REDIS_WARNING,"Unable to obtain the current NOFILE limit (%s), assuming 1024 and setting the max clients configuration accordingly.",
+            strerror(errno));
+        server.maxclients = 1024-32;
+    } else {
+        rlim_t oldlimit = limit.rlim_cur;
+
+        /* Set the max number of files if the current limit is not enough
+         * for our needs. */
+        if (oldlimit < maxfiles) {
+            limit.rlim_cur = maxfiles;
+            limit.rlim_max = maxfiles;
+            if (setrlimit(RLIMIT_NOFILE,&limit) == -1) {
+                server.maxclients = oldlimit-32;
+                redisLog(REDIS_WARNING,"Unable to set the max number of files limit to %d (%s), setting the max clients configuration to %d.",
+                    (int) maxfiles, strerror(errno), (int) server.maxclients);
+            } else {
+                redisLog(REDIS_NOTICE,"Max number of open files set to %d",
+                    (int) maxfiles);
+            }
+        }
+    }
+}
+
 void initServer() {
     int j;
 
@@ -972,7 +1009,8 @@ void initServer() {
     server.unblocked_clients = listCreate();
 
     createSharedObjects();
-    server.el = aeCreateEventLoop();
+    adjustOpenFilesLimit();
+    server.el = aeCreateEventLoop(server.maxclients+1024);
     server.db = zmalloc(sizeof(redisDb)*server.dbnum);
 
     if (server.port != 0) {
@@ -1045,38 +1083,6 @@ void initServer() {
     bioInit();
     srand(time(NULL)^getpid());
 
-    /* Try to raise the max number of open files accordingly to the
-     * configured max number of clients. Also account for 32 additional
-     * file descriptors as we need a few more for persistence, listening
-     * sockets, log files and so forth. */
-    {
-        rlim_t maxfiles = server.maxclients+32;
-        struct rlimit limit;
-
-        if (maxfiles < 1024) maxfiles = 1024;
-        if (getrlimit(RLIMIT_NOFILE,&limit) == -1) {
-            redisLog(REDIS_WARNING,"Unable to obtain the current NOFILE limit (%s), assuming 1024 and setting the max clients configuration accordingly.",
-                strerror(errno));
-            server.maxclients = 1024-32;
-        } else {
-            rlim_t oldlimit = limit.rlim_cur;
-
-            /* Set the max number of files if the current limit is not enough
-             * for our needs. */
-            if (oldlimit < maxfiles) {
-                limit.rlim_cur = maxfiles;
-                limit.rlim_max = maxfiles;
-                if (setrlimit(RLIMIT_NOFILE,&limit) == -1) {
-                    server.maxclients = oldlimit-32;
-                    redisLog(REDIS_WARNING,"Unable to set the max number of files limit to %d (%s), setting the max clients configuration to %d.",
-                        (int) maxfiles, strerror(errno), (int) server.maxclients);
-                } else {
-                    redisLog(REDIS_NOTICE,"Max number of open files set to %d",
-                        (int) maxfiles);
-                }
-            }
-        }
-    }
 }
 
 /* Populates the Redis Command Table starting from the hard coded list