[PATCH 6 of 6] QUIC: client sockets
Roman Arutyunyan
arut at nginx.com
Fri Dec 9 09:38:52 UTC 2022
# HG changeset patch
# User Roman Arutyunyan <arut at nginx.com>
# Date 1670509141 0
# Thu Dec 08 14:19:01 2022 +0000
# Branch quic
# Node ID 779035cf3e1c7886d06e5d5295943f425f4aac60
# Parent afbac4ba4c75023e10e68bae39df5b1a0fdbd17b
QUIC: client sockets.
In client sockets mode, a new socket is created for each client. This socket
is bound to server address and connected to client address. This allows for
seamless configuration reload and binary upgrade.
This mode is enabled by default and can be disabled by
"--without-quic_client_sockets" configure option.
With this approach, it is possible that some new connection packets will arrive
not to the listen socket, but to a client socket due to a race between bind()
and connect(). Such packets initiate new connections in the workers they end
up in.
diff --git a/auto/modules b/auto/modules
--- a/auto/modules
+++ b/auto/modules
@@ -1378,6 +1378,10 @@ if [ $USE_OPENSSL_QUIC = YES ]; then
have=NGX_QUIC_BPF . auto/have
fi
+
+ if [ $QUIC_CLIENT_SOCKETS = YES ]; then
+ have=NGX_QUIC_CLIENT_SOCKETS . auto/have
+ fi
fi
diff --git a/auto/options b/auto/options
--- a/auto/options
+++ b/auto/options
@@ -45,6 +45,7 @@ USE_THREADS=NO
NGX_FILE_AIO=NO
+QUIC_CLIENT_SOCKETS=YES
QUIC_BPF=NO
HTTP=YES
@@ -216,6 +217,7 @@ do
--with-file-aio) NGX_FILE_AIO=YES ;;
+ --without-quic_client_sockets) QUIC_CLIENT_SOCKETS=NONE ;;
--without-quic_bpf_module) QUIC_BPF=NONE ;;
--with-ipv6)
@@ -452,6 +454,7 @@ cat << END
--with-file-aio enable file AIO support
+ --without-quic_client_sockets disable QUIC client sockets
--without-quic_bpf_module disable ngx_quic_bpf_module
--with-http_ssl_module enable ngx_http_ssl_module
diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c
--- a/src/core/ngx_connection.c
+++ b/src/core/ngx_connection.c
@@ -513,6 +513,33 @@ ngx_open_listening_sockets(ngx_cycle_t *
#if (NGX_HAVE_REUSEPORT)
+#if (NGX_QUIC_CLIENT_SOCKETS && defined SO_REUSEPORT_LB)
+
+ if (ls[i].quic) {
+ int reuseport;
+
+ reuseport = 1;
+
+ if (setsockopt(s, SOL_SOCKET, SO_REUSEPORT,
+ (const void *) &reuseport, sizeof(int))
+ == -1)
+ {
+ ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno,
+ "setsockopt(SO_REUSEPORT) %V failed",
+ &ls[i].addr_text);
+
+ if (ngx_close_socket(s) == -1) {
+ ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno,
+ ngx_close_socket_n " %V failed",
+ &ls[i].addr_text);
+ }
+
+ return NGX_ERROR;
+ }
+ }
+
+#endif
+
if (ls[i].reuseport && !ngx_test_config) {
int reuseport;
diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c
--- a/src/event/quic/ngx_event_quic.c
+++ b/src/event/quic/ngx_event_quic.c
@@ -10,11 +10,15 @@
#include <ngx_event_quic_connection.h>
+#define NGX_QUIC_LINGERING_TIMEOUT 50
+
+
static ngx_quic_connection_t *ngx_quic_new_connection(ngx_connection_t *c,
ngx_quic_conf_t *conf, ngx_quic_header_t *pkt);
static ngx_int_t ngx_quic_handle_stateless_reset(ngx_connection_t *c,
ngx_quic_header_t *pkt);
static void ngx_quic_input_handler(ngx_event_t *rev);
+static void ngx_quic_lingering_handler(ngx_event_t *rev);
static void ngx_quic_close_handler(ngx_event_t *ev);
static ngx_int_t ngx_quic_handle_packet(ngx_connection_t *c,
@@ -218,6 +222,13 @@ ngx_quic_run(ngx_connection_t *c, ngx_qu
c->read->handler = ngx_quic_input_handler;
+ if (!c->shared) {
+ if (ngx_add_event(c->read, NGX_READ_EVENT, 0) == NGX_ERROR) {
+ ngx_quic_close_connection(c, NGX_ERROR);
+ return;
+ }
+ }
+
return;
}
@@ -441,6 +452,10 @@ ngx_quic_input_handler(ngx_event_t *rev)
return;
}
+
+ if (!c->shared) {
+ ngx_quic_recvmsg(rev);
+ }
}
@@ -583,6 +598,15 @@ quic_done:
c->destroyed = 1;
+ if (c->read->ready) {
+ ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic lingering");
+
+ c->read->handler = ngx_quic_lingering_handler;
+ ngx_add_timer(c->read, NGX_QUIC_LINGERING_TIMEOUT);
+
+ return;
+ }
+
pool = c->pool;
ngx_close_connection(c);
@@ -591,6 +615,31 @@ quic_done:
}
+static void
+ngx_quic_lingering_handler(ngx_event_t *rev)
+{
+ ngx_pool_t *pool;
+ ngx_connection_t *c;
+
+ c = rev->data;
+
+ ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic lingering handler");
+
+ if (rev->ready && !rev->timedout) {
+ ngx_quic_recvmsg(rev);
+ }
+
+ if (!rev->ready || rev->timedout) {
+ pool = c->pool;
+
+ ngx_close_connection(c);
+ ngx_destroy_pool(pool);
+
+ return;
+ }
+}
+
+
void
ngx_quic_finalize_connection(ngx_connection_t *c, ngx_uint_t err,
const char *reason)
diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c
--- a/src/event/quic/ngx_event_quic_output.c
+++ b/src/event/quic/ngx_event_quic_output.c
@@ -437,8 +437,10 @@ ngx_quic_send_segments(ngx_connection_t
msg.msg_iov = &iov;
msg.msg_iovlen = 1;
- msg.msg_name = sockaddr;
- msg.msg_namelen = socklen;
+ if (c->shared) {
+ msg.msg_name = sockaddr;
+ msg.msg_namelen = socklen;
+ }
msg.msg_control = msg_control;
msg.msg_controllen = sizeof(msg_control);
@@ -455,7 +457,7 @@ ngx_quic_send_segments(ngx_connection_t
*valp = segment;
#if (NGX_HAVE_ADDRINFO_CMSG)
- if (c->listening && c->listening->wildcard && c->local_sockaddr) {
+ if (c->shared && c->listening->wildcard) {
cmsg = CMSG_NXTHDR(&msg, cmsg);
clen += ngx_set_srcaddr_cmsg(cmsg, c->local_sockaddr);
}
@@ -747,11 +749,13 @@ ngx_quic_send(ngx_connection_t *c, u_cha
msg.msg_iov = &iov;
msg.msg_iovlen = 1;
- msg.msg_name = sockaddr;
- msg.msg_namelen = socklen;
+ if (c->shared) {
+ msg.msg_name = sockaddr;
+ msg.msg_namelen = socklen;
+ }
#if (NGX_HAVE_ADDRINFO_CMSG)
- if (c->listening && c->listening->wildcard && c->local_sockaddr) {
+ if (c->shared && c->listening->wildcard) {
msg.msg_control = msg_control;
msg.msg_controllen = sizeof(msg_control);
diff --git a/src/event/quic/ngx_event_quic_udp.c b/src/event/quic/ngx_event_quic_udp.c
--- a/src/event/quic/ngx_event_quic_udp.c
+++ b/src/event/quic/ngx_event_quic_udp.c
@@ -11,6 +11,11 @@
#include <ngx_event_quic_connection.h>
+#if (NGX_QUIC_CLIENT_SOCKETS)
+static ngx_int_t ngx_quic_create_client_socket_connection(ngx_connection_t *lc,
+ ngx_connection_t **pc, struct sockaddr *sockaddr, socklen_t socklen,
+ struct sockaddr *local_sockaddr, socklen_t local_socklen);
+#endif
static void ngx_quic_close_accepted_connection(ngx_connection_t *c);
static ngx_connection_t *ngx_quic_lookup_connection(ngx_listening_t *ls,
ngx_str_t *key);
@@ -48,7 +53,6 @@ ngx_quic_recvmsg(ngx_event_t *ev)
lc = ev->data;
ls = lc->listening;
- ev->ready = 0;
ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
"quic recvmsg on %V, ready: %d",
@@ -65,8 +69,17 @@ ngx_quic_recvmsg(ngx_event_t *ev)
msg.msg_iov = iov;
msg.msg_iovlen = 1;
+ if (lc->local_sockaddr) {
+ local_sockaddr = lc->local_sockaddr;
+ local_socklen = lc->local_socklen;
+
+ } else {
+ local_sockaddr = ls->sockaddr;
+ local_socklen = ls->socklen;
+ }
+
#if (NGX_HAVE_ADDRINFO_CMSG)
- if (ls->wildcard) {
+ if (ngx_inet_wildcard(local_sockaddr)) {
msg.msg_control = &msg_control;
msg.msg_controllen = sizeof(msg_control);
@@ -82,6 +95,7 @@ ngx_quic_recvmsg(ngx_event_t *ev)
if (err == NGX_EAGAIN) {
ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, err,
"quic recvmsg() not ready");
+ ev->ready = 0;
return;
}
@@ -121,12 +135,9 @@ ngx_quic_recvmsg(ngx_event_t *ev)
#endif
- local_sockaddr = ls->sockaddr;
- local_socklen = ls->socklen;
-
#if (NGX_HAVE_ADDRINFO_CMSG)
- if (ls->wildcard) {
+ if (ngx_inet_wildcard(local_sockaddr)) {
struct cmsghdr *cmsg;
ngx_memcpy(&lsa, local_sockaddr, local_socklen);
@@ -201,6 +212,17 @@ ngx_quic_recvmsg(ngx_event_t *ev)
}
#endif
+#if (NGX_QUIC_CLIENT_SOCKETS)
+ if (c == NULL) {
+ if (ngx_quic_create_client_socket_connection(lc, &c,
+ sockaddr, socklen, local_sockaddr, local_socklen)
+ != NGX_OK)
+ {
+ return;
+ }
+ }
+#endif
+
if (c == NULL) {
c = ngx_get_connection(lc->fd, ev->log);
if (c == NULL) {
@@ -243,17 +265,13 @@ ngx_quic_recvmsg(ngx_event_t *ev)
c->pool->log = log;
c->listening = ls;
- if (local_sockaddr == &lsa.sockaddr) {
- local_sockaddr = ngx_palloc(c->pool, local_socklen);
- if (local_sockaddr == NULL) {
- ngx_quic_close_accepted_connection(c);
- return;
- }
-
- ngx_memcpy(local_sockaddr, &lsa, local_socklen);
+ c->local_sockaddr = ngx_palloc(c->pool, local_socklen);
+ if (c->local_sockaddr == NULL) {
+ ngx_quic_close_accepted_connection(c);
+ return;
}
- c->local_sockaddr = local_sockaddr;
+ ngx_memcpy(c->local_sockaddr, local_sockaddr, local_socklen);
c->local_socklen = local_socklen;
c->buffer = ngx_create_temp_buf(c->pool, n);
@@ -267,7 +285,7 @@ ngx_quic_recvmsg(ngx_event_t *ev)
rev = c->read;
wev = c->write;
- rev->active = 1;
+ rev->ready = (c->shared ? 0 : 1);
wev->ready = 1;
rev->log = log;
@@ -341,6 +359,94 @@ ngx_quic_recvmsg(ngx_event_t *ev)
}
+#if (NGX_QUIC_CLIENT_SOCKETS)
+
+static ngx_int_t
+ngx_quic_create_client_socket_connection(ngx_connection_t *lc,
+ ngx_connection_t **pc, struct sockaddr *sockaddr, socklen_t socklen,
+ struct sockaddr *local_sockaddr, socklen_t local_socklen)
+{
+ int value;
+ ngx_socket_t s;
+
+ s = ngx_socket(sockaddr->sa_family, SOCK_DGRAM, 0);
+ if (s == (ngx_socket_t) -1) {
+ ngx_log_error(NGX_LOG_ERR, lc->log, ngx_socket_errno,
+ ngx_socket_n " failed");
+ return NGX_ERROR;
+ }
+
+ if (ngx_nonblocking(s) == -1) {
+ ngx_log_error(NGX_LOG_EMERG, lc->log, ngx_socket_errno,
+ ngx_nonblocking_n " client socket failed");
+ goto failed;
+ }
+
+ value = 1;
+
+#if (NGX_HAVE_REUSEPORT && !NGX_LINUX)
+
+ if (setsockopt(s, SOL_SOCKET, SO_REUSEPORT,
+ (const void *) &value, sizeof(int))
+ == -1)
+ {
+ ngx_log_error(NGX_LOG_EMERG, lc->log, ngx_socket_errno,
+ "setsockopt(SO_REUSEPORT) client socket failed");
+ goto failed;
+ }
+
+#else
+
+ if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR,
+ (const void *) &value, sizeof(int))
+ == -1)
+ {
+ ngx_log_error(NGX_LOG_EMERG, lc->log, ngx_socket_errno,
+ "setsockopt(SO_REUSEADDR) client socket failed");
+ goto failed;
+ }
+
+#endif
+
+ if (bind(s, local_sockaddr, local_socklen) == -1) {
+ ngx_log_error(NGX_LOG_EMERG, lc->log, ngx_socket_errno,
+ "bind() for client socket failed");
+ goto failed;
+ }
+
+ if (connect(s, sockaddr, socklen) == -1) {
+ ngx_log_error(NGX_LOG_EMERG, lc->log, ngx_socket_errno,
+ "connect() to client failed");
+ goto failed;
+ }
+
+ *pc = ngx_get_connection(s, lc->log);
+ if (*pc == NULL) {
+ goto failed;
+ }
+
+ ngx_log_debug2(NGX_LOG_DEBUG_EVENT, lc->log, 0,
+ "quic client socket connection fd:%d r:%d",
+ s, lc->listening->connection != lc);
+
+ ngx_accept_disabled = ngx_cycle->connection_n / 8
+ - ngx_cycle->free_connection_n;
+
+ return NGX_OK;
+
+failed:
+
+ if (ngx_close_socket(s) == -1) {
+ ngx_log_error(NGX_LOG_EMERG, lc->log, ngx_socket_errno,
+ ngx_close_socket_n " client failed");
+ }
+
+ return NGX_ERROR;
+}
+
+#endif
+
+
static void
ngx_quic_close_accepted_connection(ngx_connection_t *c)
{
More information about the nginx-devel
mailing list