Accept moderation in case of EMFILE/ENFILE.

In case of EMFILE/ENFILE returned from accept() we disable accept events,
and (in case of no accept mutex used) arm timer to re-enable them later.
With accept mutex we just drop it, and rely on normal accept mutex handling
to re-enable accept events once it's acquired again.

As we now handle errors in question, logging level was changed to "crit"
(instead of "alert" used for unknown errors).

Note: the code might call ngx_enable_accept_events() multiple times if
there are many listen sockets.  The ngx_enable_accept_events() function was
modified to check if connection is already active (via c->read->active) and
skip it then, thus making multiple calls safe.

src/event/ngx_event_accept.c

@@ -21,6 +21,7 @@ ngx_event_accept(ngx_event_t *ev)
     socklen_t          socklen;
     ngx_err_t          err;
     ngx_log_t         *log;
+    ngx_uint_t         level;
     ngx_socket_t       s;
     ngx_event_t       *rev, *wev;
     ngx_listening_t   *ls;
@@ -31,6 +32,14 @@ ngx_event_accept(ngx_event_t *ev)
     static ngx_uint_t  use_accept4 = 1;
 #endif
 
+    if (ev->timedout) {
+        if (ngx_enable_accept_events((ngx_cycle_t *) ngx_cycle) != NGX_OK) {
+            return;
+        }
+
+        ev->timedout = 0;
+    }
+
     ecf = ngx_event_get_conf(ngx_cycle->conf_ctx, ngx_event_core_module);
 
     if (ngx_event_flags & NGX_USE_RTSIG_EVENT) {
@@ -70,10 +79,17 @@ ngx_event_accept(ngx_event_t *ev)
                 return;
             }
 
+            level = NGX_LOG_ALERT;
+
+            if (err == NGX_ECONNABORTED) {
+                level = NGX_LOG_ERR;
+
+            } else if (err == NGX_EMFILE || err == NGX_ENFILE) {
+                level = NGX_LOG_CRIT;
+            }
+
 #if (NGX_HAVE_ACCEPT4)
-            ngx_log_error((ngx_uint_t) ((err == NGX_ECONNABORTED) ?
+            ngx_log_error(level, ev->log, err,
-                                             NGX_LOG_ERR : NGX_LOG_ALERT),
-                          ev->log, err,
                           use_accept4 ? "accept4() failed" : "accept() failed");
 
             if (use_accept4 && err == NGX_ENOSYS) {
@@ -82,9 +98,7 @@ ngx_event_accept(ngx_event_t *ev)
                 continue;
             }
 #else
-            ngx_log_error((ngx_uint_t) ((err == NGX_ECONNABORTED) ?
+            ngx_log_error(level, ev->log, err, "accept() failed");
-                                             NGX_LOG_ERR : NGX_LOG_ALERT),
-                          ev->log, err, "accept() failed");
 #endif
 
             if (err == NGX_ECONNABORTED) {
@@ -97,6 +111,26 @@ ngx_event_accept(ngx_event_t *ev)
                 }
             }
 
+            if (err == NGX_EMFILE || err == NGX_ENFILE) {
+                if (ngx_disable_accept_events((ngx_cycle_t *) ngx_cycle)
+                    != NGX_OK)
+                {
+                    return;
+                }
+
+                if (ngx_use_accept_mutex) {
+                    if (ngx_accept_mutex_held) {
+                        ngx_shmtx_unlock(&ngx_accept_mutex);
+                        ngx_accept_mutex_held = 0;
+                    }
+
+                    ngx_accept_disabled = 1;
+
+                } else {
+                    ngx_add_timer(ev, ecf->accept_mutex_delay);
+                }
+            }
+
             return;
         }
 
@@ -383,6 +417,10 @@ ngx_enable_accept_events(ngx_cycle_t *cycle)
 
         c = ls[i].connection;
 
+        if (c->read->active) {
+            continue;
+        }
+
         if (ngx_event_flags & NGX_USE_RTSIG_EVENT) {
 
             if (ngx_add_conn(c) == NGX_ERROR) {

src/os/unix/ngx_errno.h

@@ -29,6 +29,8 @@ typedef int               ngx_err_t;
 #define NGX_ENOTDIR       ENOTDIR
 #define NGX_EISDIR        EISDIR
 #define NGX_EINVAL        EINVAL
+#define NGX_ENFILE        ENFILE
+#define NGX_EMFILE        EMFILE
 #define NGX_ENOSPC        ENOSPC
 #define NGX_EPIPE         EPIPE
 #define NGX_EINPROGRESS   EINPROGRESS

src/os/win32/ngx_errno.h

@@ -54,6 +54,8 @@ typedef DWORD                      ngx_err_t;
 
 #define NGX_EALREADY               WSAEALREADY
 #define NGX_EINVAL                 WSAEINVAL
+#define NGX_EMFILE                 WSAEMFILE
+#define NGX_ENFILE                 WSAEMFILE
 
 
 u_char *ngx_strerror(ngx_err_t err, u_char *errstr, size_t size);