mirror/nginx
1
0
Fork 0
mirror of https://github.com/nginx/nginx.git synced 2025-06-06 00:42:40 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

The "reuseport" option of the "listen" directive.

Browse Source 
When configured, an individual listen socket on a given address is
created for each worker process.  This allows to reduce in-kernel lock
contention on configurations with high accept rates, resulting in better
performance.  As of now it works on Linux and DragonFly BSD.

Note that on Linux incoming connection requests are currently tied up
to a specific listen socket, and if some sockets are closed, connection
requests will be reset, see https://lwn.net/Articles/542629/.  With
nginx, this may happen if the number of worker processes is reduced.
There is no such problem on DragonFly BSD.

Based on previous work by Sepherosa Ziehau and Yingqi Lu.
This commit is contained in:
Maxim Dounin 2015-05-20 15:51:56 +03:00
parent d5c34785bc
commit f7f1607bf2
12 changed files with 206 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
auto/unix
View File

@ -308,6 +308,16 @@ ngx_feature_test="setsockopt(0, SOL_SOCKET, SO_SETFIB, NULL, 0)"
. auto/feature
ngx_feature="SO_REUSEPORT"
ngx_feature_name="NGX_HAVE_REUSEPORT"
ngx_feature_run=no
ngx_feature_incs="#include <sys/socket.h>"
ngx_feature_path=
ngx_feature_libs=
ngx_feature_test="setsockopt(0, SOL_SOCKET, SO_REUSEPORT, NULL, 0)"
. auto/feature
ngx_feature="SO_ACCEPTFILTER"
ngx_feature_name="NGX_HAVE_DEFERRED_ACCEPT"
ngx_feature_run=no

110
src/core/ngx_connection.c
View File

@ -90,6 +90,43 @@ ngx_create_listening(ngx_conf_t *cf, void *sockaddr, socklen_t socklen)
}
ngx_int_t
ngx_clone_listening(ngx_conf_t *cf, ngx_listening_t *ls)
{
#if (NGX_HAVE_REUSEPORT)
ngx_int_t n;
ngx_core_conf_t *ccf;
ngx_listening_t ols;
if (!ls->reuseport) {
return NGX_OK;
}
ols = *ls;
ccf = (ngx_core_conf_t *) ngx_get_conf(cf->cycle->conf_ctx,
ngx_core_module);
for (n = 1; n < ccf->worker_processes; n++) {
/* create a socket for each worker process */
ls = ngx_array_push(&cf->cycle->listening);
if (ls == NULL) {
return NGX_ERROR;
}
*ls = ols;
ls->worker = n;
}
#endif
return NGX_OK;
}
ngx_int_t
ngx_set_inherited_sockets(ngx_cycle_t *cycle)
{
@ -106,6 +143,9 @@ ngx_set_inherited_sockets(ngx_cycle_t *cycle)
#if (NGX_HAVE_DEFERRED_ACCEPT && defined TCP_DEFER_ACCEPT)
int timeout;
#endif
#if (NGX_HAVE_REUSEPORT)
int reuseport;
#endif
ls = cycle->listening.elts;
for (i = 0; i < cycle->listening.nelts; i++) {
@ -215,6 +255,25 @@ ngx_set_inherited_sockets(ngx_cycle_t *cycle)
#endif
#endif
#if (NGX_HAVE_REUSEPORT)
reuseport = 0;
olen = sizeof(int);
if (getsockopt(ls[i].fd, SOL_SOCKET, SO_REUSEPORT,
(void *) &reuseport, &olen)
== -1)
{
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno,
"getsockopt(SO_REUSEPORT) %V failed, ignored",
&ls[i].addr_text);
} else {
ls[i].reuseport = reuseport ? 1 : 0;
}
#endif
#if (NGX_HAVE_TCP_FASTOPEN)
olen = sizeof(int);
@ -332,6 +391,31 @@ ngx_open_listening_sockets(ngx_cycle_t *cycle)
continue;
}
#if (NGX_HAVE_REUSEPORT)
if (ls[i].add_reuseport) {
/*
* to allow transition from a socket without SO_REUSEPORT
* to multiple sockets with SO_REUSEPORT, we have to set
* SO_REUSEPORT on the old socket before opening new ones
*/
int reuseport = 1;
if (setsockopt(ls[i].fd, SOL_SOCKET, SO_REUSEPORT,
(const void *) &reuseport, sizeof(int))
== -1)
{
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno,
"setsockopt(SO_REUSEPORT) %V failed, ignored",
&ls[i].addr_text);
}
ls[i].add_reuseport = 0;
}
#endif
if (ls[i].fd != (ngx_socket_t) -1) {
continue;
}
@ -370,6 +454,32 @@ ngx_open_listening_sockets(ngx_cycle_t *cycle)
return NGX_ERROR;
}
#if (NGX_HAVE_REUSEPORT)
if (ls[i].reuseport) {
int reuseport;
reuseport = 1;
if (setsockopt(s, SOL_SOCKET, SO_REUSEPORT,
(const void *) &reuseport, sizeof(int))
== -1)
{
ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno,
"setsockopt(SO_REUSEPORT) %V failed, ignored",
&ls[i].addr_text);
if (ngx_close_socket(s) == -1) {
ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno,
ngx_close_socket_n " %V failed",
&ls[i].addr_text);
}
return NGX_ERROR;
}
}
#endif
#if (NGX_HAVE_INET6 && defined IPV6_V6ONLY)
if (ls[i].sockaddr->sa_family == AF_INET6) {

7
src/core/ngx_connection.h
View File

@ -51,6 +51,8 @@ struct ngx_listening_s {
ngx_listening_t *previous;
ngx_connection_t *connection;
ngx_uint_t worker;
unsigned open:1;
unsigned remain:1;
unsigned ignore:1;
@ -65,6 +67,10 @@ struct ngx_listening_s {
#if (NGX_HAVE_INET6 && defined IPV6_V6ONLY)
unsigned ipv6only:1;
#endif
#if (NGX_HAVE_REUSEPORT)
unsigned reuseport:1;
unsigned add_reuseport:1;
#endif
unsigned keepalive:2;
@ -203,6 +209,7 @@ struct ngx_connection_s {
ngx_listening_t *ngx_create_listening(ngx_conf_t *cf, void *sockaddr,
socklen_t socklen);
ngx_int_t ngx_clone_listening(ngx_conf_t *cf, ngx_listening_t *ls);
ngx_int_t ngx_set_inherited_sockets(ngx_cycle_t *cycle);
ngx_int_t ngx_open_listening_sockets(ngx_cycle_t *cycle);
void ngx_configure_listening_sockets(ngx_cycle_t *cycle);

11
src/core/ngx_cycle.c
View File

@ -493,6 +493,10 @@ ngx_init_cycle(ngx_cycle_t *old_cycle)
continue;
}
if (ls[i].remain) {
continue;
}
if (ngx_cmp_sockaddr(nls[n].sockaddr, nls[n].socklen,
ls[i].sockaddr, ls[i].socklen, 1)
== NGX_OK)
@ -540,6 +544,13 @@ ngx_init_cycle(ngx_cycle_t *old_cycle)
nls[n].add_deferred = 1;
}
#endif
#if (NGX_HAVE_REUSEPORT)
if (nls[n].reuseport && !ls[i].reuseport) {
nls[n].add_reuseport = 1;
}
#endif
break;
}
}

6
src/event/ngx_event.c
View File

@ -725,6 +725,12 @@ ngx_event_process_init(ngx_cycle_t *cycle)
ls = cycle->listening.elts;
for (i = 0; i < cycle->listening.nelts; i++) {
#if (NGX_HAVE_REUSEPORT)
if (ls[i].reuseport && ls[i].worker != ngx_worker) {
continue;
}
#endif
c = ngx_get_connection(ls[i].fd, cycle->log);
if (c == NULL) {

25
src/event/ngx_event_accept.c
View File

@ -11,7 +11,7 @@
static ngx_int_t ngx_enable_accept_events(ngx_cycle_t *cycle);
static ngx_int_t ngx_disable_accept_events(ngx_cycle_t *cycle);
static ngx_int_t ngx_disable_accept_events(ngx_cycle_t *cycle, ngx_uint_t all);
static void ngx_close_accepted_connection(ngx_connection_t *c);
@ -109,7 +109,7 @@ ngx_event_accept(ngx_event_t *ev)
}
if (err == NGX_EMFILE || err == NGX_ENFILE) {
if (ngx_disable_accept_events((ngx_cycle_t *) ngx_cycle)
if (ngx_disable_accept_events((ngx_cycle_t *) ngx_cycle, 1)
!= NGX_OK)
{
return;
@ -390,7 +390,7 @@ ngx_trylock_accept_mutex(ngx_cycle_t *cycle)
"accept mutex lock failed: %ui", ngx_accept_mutex_held);
if (ngx_accept_mutex_held) {
if (ngx_disable_accept_events(cycle) == NGX_ERROR) {
if (ngx_disable_accept_events(cycle, 0) == NGX_ERROR) {
return NGX_ERROR;
}
@ -413,7 +413,7 @@ ngx_enable_accept_events(ngx_cycle_t *cycle)
c = ls[i].connection;
if (c->read->active) {
if (c == NULL || c->read->active) {
continue;
}
@ -427,7 +427,7 @@ ngx_enable_accept_events(ngx_cycle_t *cycle)
static ngx_int_t
ngx_disable_accept_events(ngx_cycle_t *cycle)
ngx_disable_accept_events(ngx_cycle_t *cycle, ngx_uint_t all)
{
ngx_uint_t i;
ngx_listening_t *ls;
@ -438,10 +438,23 @@ ngx_disable_accept_events(ngx_cycle_t *cycle)
c = ls[i].connection;
if (!c->read->active) {
if (c == NULL || !c->read->active) {
continue;
}
#if (NGX_HAVE_REUSEPORT)
/*
* do not disable accept on worker's own sockets
* when disabling accept events due to accept mutex
*/
if (ls[i].reuseport && !all) {
continue;
}
#endif
if (ngx_del_event(c->read, NGX_READ_EVENT, NGX_DISABLE_EVENT)
== NGX_ERROR)
{

8
src/http/ngx_http.c
View File

@ -1737,6 +1737,10 @@ ngx_http_init_listening(ngx_conf_t *cf, ngx_http_conf_port_t *port)
break;
}
if (ngx_clone_listening(cf, ls) != NGX_OK) {
return NGX_ERROR;
}
addr++;
last--;
}
@ -1815,6 +1819,10 @@ ngx_http_add_listening(ngx_conf_t *cf, ngx_http_conf_addr_t *addr)
ls->fastopen = addr->opt.fastopen;
#endif
#if (NGX_HAVE_REUSEPORT)
ls->reuseport = addr->opt.reuseport;
#endif
return ls;
}

13
src/http/ngx_http_core_module.c
View File

@ -4166,6 +4166,19 @@ ngx_http_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
#endif
}
if (ngx_strcmp(value[n].data, "reuseport") == 0) {
#if (NGX_HAVE_REUSEPORT)
lsopt.reuseport = 1;
lsopt.set = 1;
lsopt.bind = 1;
#else
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
"reuseport is not supported "
"on this platform, ignored");
#endif
continue;
}
if (ngx_strcmp(value[n].data, "ssl") == 0) {
#if (NGX_HTTP_SSL)
lsopt.ssl = 1;

3
src/http/ngx_http_core_module.h
View File

@ -84,6 +84,9 @@ typedef struct {
#endif
#if (NGX_HAVE_INET6 && defined IPV6_V6ONLY)
unsigned ipv6only:1;
#endif
#if (NGX_HAVE_REUSEPORT)
unsigned reuseport:1;
#endif
unsigned so_keepalive:2;
unsigned proxy_protocol:1;

4
src/stream/ngx_stream.c
View File

@ -410,6 +410,10 @@ ngx_stream_optimize_servers(ngx_conf_t *cf, ngx_array_t *ports)
break;
}
if (ngx_clone_listening(cf, ls) != NGX_OK) {
return NGX_CONF_ERROR;
}
addr++;
last--;
}

3
src/stream/ngx_stream.h
View File

@ -44,6 +44,9 @@ typedef struct {
#endif
#if (NGX_HAVE_INET6 && defined IPV6_V6ONLY)
unsigned ipv6only:1;
#endif
#if (NGX_HAVE_REUSEPORT)
unsigned reuseport:1;
#endif
unsigned so_keepalive:2;
#if (NGX_HAVE_KEEPALIVE_TUNABLE)

12
src/stream/ngx_stream_core_module.c
View File

@ -384,6 +384,18 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
#endif
}
if (ngx_strcmp(value[i].data, "reuseport") == 0) {
#if (NGX_HAVE_REUSEPORT)
ls->reuseport = 1;
ls->bind = 1;
#else
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
"reuseport is not supported "
"on this platform, ignored");
#endif
continue;
}
if (ngx_strcmp(value[i].data, "ssl") == 0) {
#if (NGX_STREAM_SSL)
ls->ssl = 1;