I have written a working user-space tunnel protocol, which opens tun0.
void peer_tun_open(struct peer *peer) {
struct ifreq ifr;
int err;
proc_assert_master();
if ((peer->tun_fd = open("/dev/net/tun", O_RDWR)) == -1) {
LOG_ERROR_1("%s", "open /dev/net/tun");
exit(1);
}
memset(&ifr, 0, sizeof(ifr));
ifr.ifr_flags = IFF_TUN | IFF_NO_PI;
if ((err = ioctl(peer->tun_fd, TUNSETIFF, (void *) &ifr)) == -1) {
LOG_ERROR_1("%s", "ioctl TUNSETIFF");
close(peer->tun_fd);
exit(1);
}
...
So long as I use tun0 from only one process' epoll, everything works fine.
(I am able to use UDP sockets from multiple processes, without problems, with SO_REUSEPORT, etc. UDP ports are opened by the children, but the tun file descriptor is inherited via fork()
.)
When I add the tun0 fd to multiple process' epolls, random delays of 100-3000ms affect all packets arriving on tun0.
static void proc_epoll_callback(const char *key, void *ob, size_t index, void *extra) {
struct peer *peer = (struct peer *)ob;
int epoll_fd = *(int *) extra;
struct epoll_event event = {0};
if (peer->worker_id != state.proc_worker_id) return; // only one process must listen to a each tun interface, otherwise 100-3000ms delays occur
event.events = EPOLLIN;
event.data.fd = peer->tun_fd;
if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, event.data.fd, &event)) {
LOG_ERROR_1("failed to add fd: %d to epoll fd: %d for peer %s", event.data.fd, epoll_fd, peer->name);
} else {
LOG_DEBUG_1("added fd: %d to epoll fd: %d for peer %s", event.data.fd, epoll_fd, peer->name);
}
}
What causes these delays?
How can I use tun0 from multiple worker processes, without these delays?