From 0b56f29e04e0d2ac8b5e5d43ef2583db0960360d Mon Sep 17 00:00:00 2001
From: Egor <egor@patchysicp.xyz>
Date: Sat, 21 Oct 2023 20:01:49 +0300
Subject: [PATCH] netpoll: optimize epoll implementation

---
 src/netpoll.c | 84 ++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 56 insertions(+), 28 deletions(-)

diff --git a/src/netpoll.c b/src/netpoll.c
index 93b7cb1..37f93b5 100644
--- a/src/netpoll.c
+++ b/src/netpoll.c
@@ -1,22 +1,21 @@
 #include "netpoll.h"
+
+/* For epoll we have to guarantee stability of reference, so the table
+ * contains pointers. net_poll_init initializes these pointers. The generic
+ * functions always swap elements in and out of the allocated portion of the
+ * table. */
+#if defined(NETPOLL_USE_EPOLL)
+#define REF(x) (*(x))
+typedef struct net_pollfd *net_pollfd_tabent_t;
+#else
+#define REF(x) (x)
+typedef struct net_pollfd net_pollfd_tabent_t;
+#endif
+
 #define NETPOLL_MAXFDS 1024
-static struct net_pollfd fds[NETPOLL_MAXFDS] = {0};
+static net_pollfd_tabent_t fds[NETPOLL_MAXFDS] = {0};
 static int nfds = 0;
 
-static int search(SOCKET fd) {
-	int l = 0, r = nfds;
-	while (l != r) {
-		int m = l + (r-l)/2;
-		if (fds[m].fd < fd)
-			l = m+1;
-		else if (fd < fds[m].fd)
-			r = m;
-		else
-			l = r = m;
-	}
-	return l;
-}
-
 #if defined(NETPOLL_USE_SELECT)
 static fd_set readfds, writefds, xreadfds, xwritefds;
 static int lastptr = 0;
@@ -137,8 +136,11 @@ static int epollfd = -1;
 static struct epoll_event events[256] = {0};
 static int nevents = 0;
 static int lastptr = 0;
+static struct net_pollfd fdsbuf[NETPOLL_MAXFDS] = {0};
 void net_poll_init(void)
 {
+	for (int i = 0; i < NETPOLL_MAXFDS; i++)
+		fds[i] = &fdsbuf[i];
 	epollfd = epoll_create1(EPOLL_CLOEXEC);
 	if (epollfd == -1) {
 		perror("epoll_create");
@@ -152,24 +154,33 @@ void net_poll_fini(void)
 }
 void net_poll_add_impl(int i)
 {
-	struct epoll_event e = { .data.fd = fds[i].fd };
-	if (epoll_ctl(epollfd, EPOLL_CTL_ADD, fds[i].fd, &e) == -1) {
+	struct epoll_event e = { .data.ptr = fds[i] };
+	if (epoll_ctl(epollfd, EPOLL_CTL_ADD, fds[i]->fd, &e) == -1) {
 		perror("epoll_ctl(EPOLL_CTL_ADD)");
 		abort();
 	}
 }
 void net_poll_del_impl(int i)
 {
-	if (epoll_ctl(epollfd, EPOLL_CTL_DEL, fds[i].fd, 0) == -1) {
+	/* prevent leaked references */
+	struct epoll_event *p, *q;
+	p = q = events+lastptr;
+	for (int i = lastptr; i < nevents; i++) {
+		if (p->data.ptr != fds[i])
+			*q++ = *p;
+		p++;
+	}
+	nevents = q-events;
+	if (epoll_ctl(epollfd, EPOLL_CTL_DEL, fds[i]->fd, 0) == -1) {
 		perror("epoll_ctl(EPOLL_CTL_DEL)");
 		abort();
 	}
 }
 void net_poll_reg_impl(int i, int ev)
 {
-	struct epoll_event e = { .data.fd = fds[i].fd };
+	struct epoll_event e = { .data.ptr = fds[i] };
 	e.events = (ev & NETPOLL_IN ? EPOLLIN : 0) | (ev & NETPOLL_OUT ? EPOLLOUT : 0);
-	if (epoll_ctl(epollfd, EPOLL_CTL_MOD, fds[i].fd, &e) == -1) {
+	if (epoll_ctl(epollfd, EPOLL_CTL_MOD, fds[i]->fd, &e) == -1) {
 		perror("epoll_ctl(EPOLL_CTL_MOD)");
 		abort();
 	}
@@ -194,13 +205,28 @@ int net_poll_get(struct net_pollfd *out)
 			ev |= NETPOLL_OUT;
 		lastptr++;
 	}
-	/* FIXME: this is stupid */
-	*out = fds[search(events[lastptr-1].data.fd)];
+	*out = *(struct net_pollfd *)events[lastptr-1].data.ptr;
 	return ev;
 }
 #else
 #error pick a netpoll implementation
 #endif
+
+static int search(SOCKET fd) {
+	int l = 0, r = nfds;
+	while (l != r) {
+		int m = l + (r-l)/2;
+		if (REF(fds[m]).fd < fd)
+			l = m+1;
+		else if (fd < REF(fds[m]).fd)
+			r = m;
+		else
+			l = r = m;
+	}
+	return l;
+}
+#define IS_FOUND(i, f) ((i) < nfds && REF(fds[i]).fd == (f))
+
 void net_poll_add(SOCKET fd, int type, void *assoc)
 {
 	if (fd == INVALID_SOCKET) {
@@ -208,7 +234,7 @@ void net_poll_add(SOCKET fd, int type, void *assoc)
 		abort();
 	}
 	int i = search(fd);
-	if (i < nfds && fds[i].fd == fd) {
+	if (IS_FOUND(i, fd)) {
 		fprintf(stderr, "net_poll_add: socket already added\n");
 		abort();
 	}
@@ -216,11 +242,11 @@ void net_poll_add(SOCKET fd, int type, void *assoc)
 		fprintf(stderr, "net_poll_add: NETPOLL_MAXFDS limit\n");
 		abort();
 	}
+	net_pollfd_tabent_t t = fds[nfds];
 	memmove(fds+i+1, fds+i, sizeof(fds[0])*(nfds-i));
 	nfds++;
-	fds[i].fd = fd;
-	fds[i].type = type;
-	fds[i].assoc = assoc;
+	fds[i] = t;
+	REF(fds[i]) = (struct net_pollfd){ .fd = fd, .type = type, .assoc = assoc };
 	net_poll_add_impl(i);
 }
 void net_poll_del(SOCKET fd)
@@ -230,14 +256,16 @@ void net_poll_del(SOCKET fd)
 		abort();
 	}
 	int i = search(fd);
-	if (i >= nfds || fds[i].fd != fd) {
+	if (!IS_FOUND(i, fd)) {
 		fprintf(stderr, "net_poll_del: socket not added\n");
 		abort();
 	}
 	net_poll_reg_impl(i, 0);
 	net_poll_del_impl(i);
+	net_pollfd_tabent_t t = fds[i];
 	memmove(fds+i, fds+i+1, sizeof(fds[0])*(nfds-(i+1)));
 	nfds--;
+	fds[nfds] = t;
 }
 void net_poll_reg(SOCKET fd, int ev)
 {
@@ -246,7 +274,7 @@ void net_poll_reg(SOCKET fd, int ev)
 		abort();
 	}
 	int i = search(fd);
-	if (i >= nfds || fds[i].fd != fd) {
+	if (!IS_FOUND(i, fd)) {
 		fprintf(stderr, "net_poll_reg: socket not added\n");
 		abort();
 	}
-- 
2.56.0