From 50ff8b3c3a3eba0984ce55c63ab8ac07dcb65265 Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Tue, 11 Aug 2015 16:28:55 +0300 Subject: [PATCH] Core: idle connections now closed only once on exiting. Iterating through all connections takes a lot of CPU time, especially with large number of worker connections configured. As a result nginx processes used to consume CPU time during graceful shutdown. To mitigate this we now only do a full scan for idle connections when shutdown signal is received. Transitions of connections to idle ones are now expected to be avoided if the ngx_exiting flag is set. The upstream keepalive module was modified to follow this. --- src/core/ngx_connection.c | 20 +++++++++++++++++ src/core/ngx_connection.h | 1 + .../ngx_http_upstream_keepalive_module.c | 4 ++++ src/os/unix/ngx_process_cycle.c | 19 ++-------------- src/os/win32/ngx_process_cycle.c | 22 ++++--------------- 5 files changed, 31 insertions(+), 35 deletions(-) diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c index 00c5a08e6..9f2675b88 100644 --- a/src/core/ngx_connection.c +++ b/src/core/ngx_connection.c @@ -1163,6 +1163,26 @@ ngx_drain_connections(void) } +void +ngx_close_idle_connections(ngx_cycle_t *cycle) +{ + ngx_uint_t i; + ngx_connection_t *c; + + c = cycle->connections; + + for (i = 0; i < cycle->connection_n; i++) { + + /* THREAD: lock */ + + if (c[i].fd != -1 && c[i].idle) { + c[i].close = 1; + c[i].read->handler(c[i].read); + } + } +} + + ngx_int_t ngx_connection_local_sockaddr(ngx_connection_t *c, ngx_str_t *s, ngx_uint_t port) diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h index a49aa9579..c3aca7f47 100644 --- a/src/core/ngx_connection.h +++ b/src/core/ngx_connection.h @@ -215,6 +215,7 @@ ngx_int_t ngx_open_listening_sockets(ngx_cycle_t *cycle); void ngx_configure_listening_sockets(ngx_cycle_t *cycle); void ngx_close_listening_sockets(ngx_cycle_t *cycle); void ngx_close_connection(ngx_connection_t *c); +void ngx_close_idle_connections(ngx_cycle_t *cycle); ngx_int_t ngx_connection_local_sockaddr(ngx_connection_t *c, ngx_str_t *s, ngx_uint_t port); ngx_int_t ngx_connection_error(ngx_connection_t *c, ngx_err_t err, char *text); diff --git a/src/http/modules/ngx_http_upstream_keepalive_module.c b/src/http/modules/ngx_http_upstream_keepalive_module.c index 768881ee4..51887b4e8 100644 --- a/src/http/modules/ngx_http_upstream_keepalive_module.c +++ b/src/http/modules/ngx_http_upstream_keepalive_module.c @@ -302,6 +302,10 @@ ngx_http_upstream_free_keepalive_peer(ngx_peer_connection_t *pc, void *data, goto invalid; } + if (ngx_terminate || ngx_exiting) { + goto invalid; + } + if (ngx_handle_read_event(c->read, 0) != NGX_OK) { goto invalid; } diff --git a/src/os/unix/ngx_process_cycle.c b/src/os/unix/ngx_process_cycle.c index c69932ecd..5da09116f 100644 --- a/src/os/unix/ngx_process_cycle.c +++ b/src/os/unix/ngx_process_cycle.c @@ -728,9 +728,6 @@ ngx_worker_process_cycle(ngx_cycle_t *cycle, void *data) { ngx_int_t worker = (intptr_t) data; - ngx_uint_t i; - ngx_connection_t *c; - ngx_process = NGX_PROCESS_WORKER; ngx_worker = worker; @@ -741,19 +738,6 @@ ngx_worker_process_cycle(ngx_cycle_t *cycle, void *data) for ( ;; ) { if (ngx_exiting) { - - c = cycle->connections; - - for (i = 0; i < cycle->connection_n; i++) { - - /* THREAD: lock */ - - if (c[i].fd != -1 && c[i].idle) { - c[i].close = 1; - c[i].read->handler(c[i].read); - } - } - ngx_event_cancel_timers(); if (ngx_event_timer_rbtree.root == ngx_event_timer_rbtree.sentinel) @@ -781,8 +765,9 @@ ngx_worker_process_cycle(ngx_cycle_t *cycle, void *data) ngx_setproctitle("worker process is shutting down"); if (!ngx_exiting) { - ngx_close_listening_sockets(cycle); ngx_exiting = 1; + ngx_close_listening_sockets(cycle); + ngx_close_idle_connections(cycle); } } diff --git a/src/os/win32/ngx_process_cycle.c b/src/os/win32/ngx_process_cycle.c index 203bece3f..06ff8b50e 100644 --- a/src/os/win32/ngx_process_cycle.c +++ b/src/os/win32/ngx_process_cycle.c @@ -761,10 +761,8 @@ failed: static ngx_thread_value_t __stdcall ngx_worker_thread(void *data) { - ngx_int_t n; - ngx_uint_t i; - ngx_cycle_t *cycle; - ngx_connection_t *c; + ngx_int_t n; + ngx_cycle_t *cycle; cycle = (ngx_cycle_t *) ngx_cycle; @@ -780,19 +778,6 @@ ngx_worker_thread(void *data) while (!ngx_quit) { if (ngx_exiting) { - - c = cycle->connections; - - for (i = 0; i < cycle->connection_n; i++) { - - /* THREAD: lock */ - - if (c[i].fd != (ngx_socket_t) -1 && c[i].idle) { - c[i].close = 1; - c[i].read->handler(c[i].read); - } - } - ngx_event_cancel_timers(); if (ngx_event_timer_rbtree.root @@ -814,8 +799,9 @@ ngx_worker_thread(void *data) ngx_quit = 0; if (!ngx_exiting) { - ngx_close_listening_sockets(cycle); ngx_exiting = 1; + ngx_close_listening_sockets(cycle); + ngx_close_idle_connections(cycle); } }