From 931ce7f02a9c55987ef9a7128882097b89a36540 Mon Sep 17 00:00:00 2001
From: Maxim Dounin <mdounin@mdounin.ru>
Date: Fri, 18 Mar 2016 05:04:45 +0300
Subject: [PATCH] Fixed timeouts with threaded sendfile() and subrequests.

If a write event happens after sendfile() but before we've got the
sendfile results in the main thread, this write event will be ignored.
And if no more events will happen, the connection will hang.

Removing the events works in the simple cases, but not always, as
in some cases events are added back by an unrelated code.  E.g.,
the upstream module adds write event in the ngx_http_upstream_init()
to track client aborts.

Fix is to use wev->complete instead.  It is now set to 0 before
a sendfile() task is posted, and it is set to 1 once a write event
happens.  If on completion of the sendfile() task wev->complete is 1,
we know that an event happened while we were executing sendfile(), and
the socket is still ready for writing even if sendfile() did not sent
all the data or returned EAGAIN.
---
 src/event/modules/ngx_epoll_module.c   |  3 +++
 src/os/unix/ngx_linux_sendfile_chain.c | 21 +++++++++++----------
 2 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/src/event/modules/ngx_epoll_module.c b/src/event/modules/ngx_epoll_module.c
index d7f915de9..081b0e550 100644
--- a/src/event/modules/ngx_epoll_module.c
+++ b/src/event/modules/ngx_epoll_module.c
@@ -840,6 +840,9 @@ ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags)
             }
 
             wev->ready = 1;
+#if (NGX_THREADS)
+            wev->complete = 1;
+#endif
 
             if (flags & NGX_POST_EVENTS) {
                 ngx_post_event(wev, &ngx_posted_events);
diff --git a/src/os/unix/ngx_linux_sendfile_chain.c b/src/os/unix/ngx_linux_sendfile_chain.c
index 8d01e10d0..50a9cea2e 100644
--- a/src/os/unix/ngx_linux_sendfile_chain.c
+++ b/src/os/unix/ngx_linux_sendfile_chain.c
@@ -328,7 +328,6 @@ static ngx_int_t
 ngx_linux_sendfile_thread(ngx_connection_t *c, ngx_buf_t *file, size_t size,
     size_t *sent)
 {
-    ngx_uint_t                 flags;
     ngx_event_t               *wev;
     ngx_thread_task_t         *task;
     ngx_linux_sendfile_ctx_t  *ctx;
@@ -358,6 +357,11 @@ ngx_linux_sendfile_thread(ngx_connection_t *c, ngx_buf_t *file, size_t size,
 
         if (ctx->err == NGX_EAGAIN) {
             *sent = 0;
+
+            if (wev->complete) {
+                return NGX_DONE;
+            }
+
             return NGX_AGAIN;
         }
 
@@ -382,7 +386,11 @@ ngx_linux_sendfile_thread(ngx_connection_t *c, ngx_buf_t *file, size_t size,
 
         *sent = ctx->sent;
 
-        return (ctx->sent == ctx->size) ? NGX_DONE : NGX_AGAIN;
+        if (ctx->sent == ctx->size || wev->complete) {
+            return NGX_DONE;
+        }
+
+        return NGX_AGAIN;
     }
 
     if (task->event.active && ctx->file == file) {
@@ -400,14 +408,7 @@ ngx_linux_sendfile_thread(ngx_connection_t *c, ngx_buf_t *file, size_t size,
     ctx->socket = c->fd;
     ctx->size = size;
 
-    if (wev->active) {
-        flags = (ngx_event_flags & NGX_USE_CLEAR_EVENT) ? NGX_CLEAR_EVENT
-                                                        : NGX_LEVEL_EVENT;
-
-        if (ngx_del_event(wev, NGX_WRITE_EVENT, flags) == NGX_ERROR) {
-            return NGX_ERROR;
-        }
-    }
+    wev->complete = 0;
 
     if (file->file->thread_handler(task, file->file) != NGX_OK) {
         return NGX_ERROR;