Core: idle connections now closed only once on exiting.

Iterating through all connections takes a lot of CPU time, especially
with large number of worker connections configured.  As a result
nginx processes used to consume CPU time during graceful shutdown.
To mitigate this we now only do a full scan for idle connections when
shutdown signal is received.

Transitions of connections to idle ones are now expected to be
avoided if the ngx_exiting flag is set.  The upstream keepalive module
was modified to follow this.

src/core/ngx_connection.c

@@ -1163,6 +1163,26 @@ ngx_drain_connections(void)
 }
 
 
+void
+ngx_close_idle_connections(ngx_cycle_t *cycle)
+{
+    ngx_uint_t         i;
+    ngx_connection_t  *c;
+
+    c = cycle->connections;
+
+    for (i = 0; i < cycle->connection_n; i++) {
+
+        /* THREAD: lock */
+
+        if (c[i].fd != -1 && c[i].idle) {
+            c[i].close = 1;
+            c[i].read->handler(c[i].read);
+        }
+    }
+}
+
+
 ngx_int_t
 ngx_connection_local_sockaddr(ngx_connection_t *c, ngx_str_t *s,
     ngx_uint_t port)

src/core/ngx_connection.h

@@ -215,6 +215,7 @@ ngx_int_t ngx_open_listening_sockets(ngx_cycle_t *cycle);
 void ngx_configure_listening_sockets(ngx_cycle_t *cycle);
 void ngx_close_listening_sockets(ngx_cycle_t *cycle);
 void ngx_close_connection(ngx_connection_t *c);
+void ngx_close_idle_connections(ngx_cycle_t *cycle);
 ngx_int_t ngx_connection_local_sockaddr(ngx_connection_t *c, ngx_str_t *s,
     ngx_uint_t port);
 ngx_int_t ngx_connection_error(ngx_connection_t *c, ngx_err_t err, char *text);

src/http/modules/ngx_http_upstream_keepalive_module.c

@@ -302,6 +302,10 @@ ngx_http_upstream_free_keepalive_peer(ngx_peer_connection_t *pc, void *data,
         goto invalid;
     }
 
+    if (ngx_terminate || ngx_exiting) {
+        goto invalid;
+    }
+
     if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
         goto invalid;
     }

src/os/unix/ngx_process_cycle.c

@@ -728,9 +728,6 @@ ngx_worker_process_cycle(ngx_cycle_t *cycle, void *data)
 {
     ngx_int_t worker = (intptr_t) data;
 
-    ngx_uint_t         i;
-    ngx_connection_t  *c;
-
     ngx_process = NGX_PROCESS_WORKER;
     ngx_worker = worker;
 
@@ -741,19 +738,6 @@ ngx_worker_process_cycle(ngx_cycle_t *cycle, void *data)
     for ( ;; ) {
 
         if (ngx_exiting) {
-
-            c = cycle->connections;
-
-            for (i = 0; i < cycle->connection_n; i++) {
-
-                /* THREAD: lock */
-
-                if (c[i].fd != -1 && c[i].idle) {
-                    c[i].close = 1;
-                    c[i].read->handler(c[i].read);
-                }
-            }
-
             ngx_event_cancel_timers();
 
             if (ngx_event_timer_rbtree.root == ngx_event_timer_rbtree.sentinel)
@@ -781,8 +765,9 @@ ngx_worker_process_cycle(ngx_cycle_t *cycle, void *data)
             ngx_setproctitle("worker process is shutting down");
 
             if (!ngx_exiting) {
-                ngx_close_listening_sockets(cycle);
                 ngx_exiting = 1;
+                ngx_close_listening_sockets(cycle);
+                ngx_close_idle_connections(cycle);
             }
         }
 

src/os/win32/ngx_process_cycle.c

@@ -762,9 +762,7 @@ static ngx_thread_value_t __stdcall
 ngx_worker_thread(void *data)
 {
     ngx_int_t     n;
-    ngx_uint_t         i;
     ngx_cycle_t  *cycle;
-    ngx_connection_t  *c;
 
     cycle = (ngx_cycle_t *) ngx_cycle;
 
@@ -780,19 +778,6 @@ ngx_worker_thread(void *data)
     while (!ngx_quit) {
 
         if (ngx_exiting) {
-
-            c = cycle->connections;
-
-            for (i = 0; i < cycle->connection_n; i++) {
-
-                /* THREAD: lock */
-
-                if (c[i].fd != (ngx_socket_t) -1 && c[i].idle) {
-                    c[i].close = 1;
-                    c[i].read->handler(c[i].read);
-                }
-            }
-
             ngx_event_cancel_timers();
 
             if (ngx_event_timer_rbtree.root
@@ -814,8 +799,9 @@ ngx_worker_thread(void *data)
             ngx_quit = 0;
 
             if (!ngx_exiting) {
-                ngx_close_listening_sockets(cycle);
                 ngx_exiting = 1;
+                ngx_close_listening_sockets(cycle);
+                ngx_close_idle_connections(cycle);
             }
         }