netpoll: optimize epoll implementation

diff --git a/src/netpoll.c b/src/netpoll.c
index 93b7cb1..37f93b5 100644 (file)
--- a/src/netpoll.c
+++ b/src/netpoll.c
@@ -1,22 +1,21 @@
 #include "netpoll.h"
+
+/* For epoll we have to guarantee stability of reference, so the table
+ * contains pointers. net_poll_init initializes these pointers. The generic
+ * functions always swap elements in and out of the allocated portion of the
+ * table. */
+#if defined(NETPOLL_USE_EPOLL)
+#define REF(x) (*(x))
+typedef struct net_pollfd *net_pollfd_tabent_t;
+#else
+#define REF(x) (x)
+typedef struct net_pollfd net_pollfd_tabent_t;
+#endif
+
 #define NETPOLL_MAXFDS 1024
-static struct net_pollfd fds[NETPOLL_MAXFDS] = {0};
+static net_pollfd_tabent_t fds[NETPOLL_MAXFDS] = {0};
 static int nfds = 0;
 
-static int search(SOCKET fd) {
-       int l = 0, r = nfds;
-       while (l != r) {
-               int m = l + (r-l)/2;
-               if (fds[m].fd < fd)
-                       l = m+1;
-               else if (fd < fds[m].fd)
-                       r = m;
-               else
-                       l = r = m;
-       }
-       return l;
-}
-
 #if defined(NETPOLL_USE_SELECT)
 static fd_set readfds, writefds, xreadfds, xwritefds;
 static int lastptr = 0;
@@ -137,8 +136,11 @@ static int epollfd = -1;
 static struct epoll_event events[256] = {0};
 static int nevents = 0;
 static int lastptr = 0;
+static struct net_pollfd fdsbuf[NETPOLL_MAXFDS] = {0};
 void net_poll_init(void)
 {
+       for (int i = 0; i < NETPOLL_MAXFDS; i++)
+               fds[i] = &fdsbuf[i];
        epollfd = epoll_create1(EPOLL_CLOEXEC);
        if (epollfd == -1) {
                perror("epoll_create");
@@ -152,24 +154,33 @@ void net_poll_fini(void)
 }
 void net_poll_add_impl(int i)
 {
-       struct epoll_event e = { .data.fd = fds[i].fd };
-       if (epoll_ctl(epollfd, EPOLL_CTL_ADD, fds[i].fd, &e) == -1) {
+       struct epoll_event e = { .data.ptr = fds[i] };
+       if (epoll_ctl(epollfd, EPOLL_CTL_ADD, fds[i]->fd, &e) == -1) {
                perror("epoll_ctl(EPOLL_CTL_ADD)");
                abort();
        }
 }
 void net_poll_del_impl(int i)
 {
-       if (epoll_ctl(epollfd, EPOLL_CTL_DEL, fds[i].fd, 0) == -1) {
+       /* prevent leaked references */
+       struct epoll_event *p, *q;
+       p = q = events+lastptr;
+       for (int i = lastptr; i < nevents; i++) {
+               if (p->data.ptr != fds[i])
+                       *q++ = *p;
+               p++;
+       }
+       nevents = q-events;
+       if (epoll_ctl(epollfd, EPOLL_CTL_DEL, fds[i]->fd, 0) == -1) {
                perror("epoll_ctl(EPOLL_CTL_DEL)");
                abort();
        }
 }
 void net_poll_reg_impl(int i, int ev)
 {
-       struct epoll_event e = { .data.fd = fds[i].fd };
+       struct epoll_event e = { .data.ptr = fds[i] };
        e.events = (ev & NETPOLL_IN ? EPOLLIN : 0) | (ev & NETPOLL_OUT ? EPOLLOUT : 0);
-       if (epoll_ctl(epollfd, EPOLL_CTL_MOD, fds[i].fd, &e) == -1) {
+       if (epoll_ctl(epollfd, EPOLL_CTL_MOD, fds[i]->fd, &e) == -1) {
                perror("epoll_ctl(EPOLL_CTL_MOD)");
                abort();
        }
@@ -194,13 +205,28 @@ int net_poll_get(struct net_pollfd *out)
                        ev |= NETPOLL_OUT;
                lastptr++;
        }
-       /* FIXME: this is stupid */
-       *out = fds[search(events[lastptr-1].data.fd)];
+       *out = *(struct net_pollfd *)events[lastptr-1].data.ptr;
        return ev;
 }
 #else
 #error pick a netpoll implementation
 #endif
+
+static int search(SOCKET fd) {
+       int l = 0, r = nfds;
+       while (l != r) {
+               int m = l + (r-l)/2;
+               if (REF(fds[m]).fd < fd)
+                       l = m+1;
+               else if (fd < REF(fds[m]).fd)
+                       r = m;
+               else
+                       l = r = m;
+       }
+       return l;
+}
+#define IS_FOUND(i, f) ((i) < nfds && REF(fds[i]).fd == (f))
+
 void net_poll_add(SOCKET fd, int type, void *assoc)
 {
        if (fd == INVALID_SOCKET) {
@@ -208,7 +234,7 @@ void net_poll_add(SOCKET fd, int type, void *assoc)
                abort();
        }
        int i = search(fd);
-       if (i < nfds && fds[i].fd == fd) {
+       if (IS_FOUND(i, fd)) {
                fprintf(stderr, "net_poll_add: socket already added\n");
                abort();
        }
@@ -216,11 +242,11 @@ void net_poll_add(SOCKET fd, int type, void *assoc)
                fprintf(stderr, "net_poll_add: NETPOLL_MAXFDS limit\n");
                abort();
        }
+       net_pollfd_tabent_t t = fds[nfds];
        memmove(fds+i+1, fds+i, sizeof(fds[0])*(nfds-i));
        nfds++;
-       fds[i].fd = fd;
-       fds[i].type = type;
-       fds[i].assoc = assoc;
+       fds[i] = t;
+       REF(fds[i]) = (struct net_pollfd){ .fd = fd, .type = type, .assoc = assoc };
        net_poll_add_impl(i);
 }
 void net_poll_del(SOCKET fd)
@@ -230,14 +256,16 @@ void net_poll_del(SOCKET fd)
                abort();
        }
        int i = search(fd);
-       if (i >= nfds || fds[i].fd != fd) {
+       if (!IS_FOUND(i, fd)) {
                fprintf(stderr, "net_poll_del: socket not added\n");
                abort();
        }
        net_poll_reg_impl(i, 0);
        net_poll_del_impl(i);
+       net_pollfd_tabent_t t = fds[i];
        memmove(fds+i, fds+i+1, sizeof(fds[0])*(nfds-(i+1)));
        nfds--;
+       fds[nfds] = t;
 }
 void net_poll_reg(SOCKET fd, int ev)
 {
@@ -246,7 +274,7 @@ void net_poll_reg(SOCKET fd, int ev)
                abort();
        }
        int i = search(fd);
-       if (i >= nfds || fds[i].fd != fd) {
+       if (!IS_FOUND(i, fd)) {
                fprintf(stderr, "net_poll_reg: socket not added\n");
                abort();
        }