reuse keepalive connections if there are no free worker connections

patch by Maxim Dounin

src/core/ngx_connection.c

@@ -12,6 +12,9 @@
 ngx_os_io_t  ngx_io;
 
 
+static void ngx_drain_connections(void);
+
+
 ngx_listening_t *
 ngx_create_listening(ngx_conf_t *cf, void *sockaddr, socklen_t socklen)
 {
@@ -718,6 +721,11 @@ ngx_get_connection(ngx_socket_t s, ngx_log_t *log)
 
     c = ngx_cycle->free_connections;
 
+    if (c == NULL) {
+        ngx_drain_connections();
+        c = ngx_cycle->free_connections;
+    }
+
     if (c == NULL) {
         ngx_log_error(NGX_LOG_ALERT, log, 0,
                       "%ui worker_connections are not enough",
@@ -861,6 +869,8 @@ ngx_close_connection(ngx_connection_t *c)
 
 #endif
 
+    ngx_reusable_connection(c, 0);
+
     log_error = c->log_error;
 
     ngx_free_connection(c);
@@ -900,6 +910,51 @@ ngx_close_connection(ngx_connection_t *c)
 }
 
 
+void
+ngx_reusable_connection(ngx_connection_t *c, ngx_uint_t reusable)
+{
+    ngx_log_debug1(NGX_LOG_DEBUG_CORE, c->log, 0,
+                   "reusable connection: %ui", reusable);
+
+    if (c->reusable) {
+        ngx_queue_remove(&c->queue);
+    }
+
+    c->reusable = reusable;
+
+    if (reusable) {
+        /* need cast as ngx_cycle is volatile */
+
+        ngx_queue_insert_head(
+            (ngx_queue_t *) &ngx_cycle->reusable_connections_queue, &c->queue);
+    }
+}
+
+
+static void
+ngx_drain_connections(void)
+{
+    ngx_int_t          i;
+    ngx_queue_t       *q;
+    ngx_connection_t  *c;
+
+    for (i = 0; i < 32; i++) {
+        if (ngx_queue_empty(&ngx_cycle->reusable_connections_queue)) {
+            break;
+        }
+
+        q = ngx_queue_last(&ngx_cycle->reusable_connections_queue);
+        c = ngx_queue_data(q, ngx_connection_t, queue);
+
+        ngx_log_debug0(NGX_LOG_DEBUG_CORE, c->log, 0,
+                       "reusing connection");
+
+        c->close = 1;
+        c->read->handler(c->read);
+    }
+}
+
+
 ngx_int_t
 ngx_connection_local_sockaddr(ngx_connection_t *c, ngx_str_t *s,
     ngx_uint_t port)

src/core/ngx_connection.h

@@ -135,6 +135,8 @@ struct ngx_connection_s {
 
     ngx_buf_t          *buffer;
 
+    ngx_queue_t         queue;
+
     ngx_atomic_uint_t   number;
 
     ngx_uint_t          requests;
@@ -150,6 +152,7 @@ struct ngx_connection_s {
     unsigned            destroyed:1;
 
     unsigned            idle:1;
+    unsigned            reusable:1;
     unsigned            close:1;
 
     unsigned            sendfile:1;
@@ -186,5 +189,6 @@ ngx_int_t ngx_connection_error(ngx_connection_t *c, ngx_err_t err, char *text);
 ngx_connection_t *ngx_get_connection(ngx_socket_t s, ngx_log_t *log);
 void ngx_free_connection(ngx_connection_t *c);
 
+void ngx_reusable_connection(ngx_connection_t *c, ngx_uint_t reusable);
 
 #endif /* _NGX_CONNECTION_H_INCLUDED_ */

src/core/ngx_cycle.c

@@ -181,6 +181,9 @@ ngx_init_cycle(ngx_cycle_t *old_cycle)
     cycle->listening.pool = pool;
 
 
+    ngx_queue_init(&cycle->reusable_connections_queue);
+
+
     cycle->conf_ctx = ngx_pcalloc(pool, ngx_max_module * sizeof(void *));
     if (cycle->conf_ctx == NULL) {
         ngx_destroy_pool(pool);

src/core/ngx_cycle.h

@@ -44,6 +44,8 @@ struct ngx_cycle_s {
     ngx_connection_t         *free_connections;
     ngx_uint_t                free_connection_n;
 
+    ngx_queue_t               reusable_connections_queue;
+
     ngx_array_t               listening;
     ngx_array_t               pathes;
     ngx_list_t                open_files;

src/http/ngx_http_request.c

@@ -2594,6 +2594,7 @@ ngx_http_set_keepalive(ngx_http_request_t *r)
 #endif
 
     c->idle = 1;
+    ngx_reusable_connection(c, 1);
 
     if (rev->ready) {
         ngx_post_event(rev, &ngx_posted_events);
@@ -2703,6 +2704,7 @@ ngx_http_keepalive_handler(ngx_event_t *rev)
     c->log->action = "reading client request line";
 
     c->idle = 0;
+    ngx_reusable_connection(c, 0);
 
     ngx_http_init_request(rev);
 }