aio sendfile

auto/os/freebsd

@@ -43,6 +43,12 @@ if [ $osreldate -gt 300007 ]; then
     CORE_SRCS="$CORE_SRCS $FREEBSD_SENDFILE_SRCS"
 fi
 
+if [ $osreldate -gt 502103 ]; then
+    echo " + sendfile()'s SF_NODISKIO found"
+
+    have=NGX_HAVE_AIO_SENDFILE . auto/have
+fi
+
 
 # kqueue
 

src/core/ngx_connection.h

@@ -159,6 +159,11 @@ struct ngx_connection_s {
     unsigned            accept_context_updated:1;
 #endif
 
+#if (NGX_HAVE_AIO_SENDFILE)
+    unsigned            aio_sendfile:1;
+    ngx_buf_t          *busy_sendfile;
+#endif
+
 #if (NGX_THREADS)
     ngx_atomic_t        lock;
 #endif

src/event/ngx_event.h

@@ -209,6 +209,10 @@ struct ngx_event_aio_s {
     size_t                     nbytes;
 #endif
 
+#if (NGX_HAVE_AIO_SENDFILE)
+    off_t                      last_offset;
+#endif
+
     ngx_aiocb_t                aiocb;
     ngx_event_t                event;
 };

src/http/ngx_http_copy_filter_module.c

@@ -18,6 +18,9 @@ typedef struct {
 static void ngx_http_copy_aio_handler(ngx_output_chain_ctx_t *ctx,
     ngx_file_t *file);
 static void ngx_http_copy_aio_event_handler(ngx_event_t *ev);
+#if (NGX_HAVE_AIO_SENDFILE)
+static void ngx_http_copy_aio_sendfile_event_handler(ngx_event_t *ev);
+#endif
 #endif
 
 static void *ngx_http_copy_filter_create_conf(ngx_conf_t *cf);
@@ -121,6 +124,9 @@ ngx_http_copy_filter(ngx_http_request_t *r, ngx_chain_t *in)
 #if (NGX_HAVE_FILE_AIO)
         if (clcf->aio) {
             ctx->aio = ngx_http_copy_aio_handler;
+#if (NGX_HAVE_AIO_SENDFILE)
+            c->aio_sendfile = (clcf->aio == NGX_HTTP_AIO_SENDFILE);
+#endif
         }
 #endif
 
@@ -139,6 +145,42 @@ ngx_http_copy_filter(ngx_http_request_t *r, ngx_chain_t *in)
     ngx_log_debug3(NGX_LOG_DEBUG_HTTP, c->log, 0,
                    "http copy filter: %i \"%V?%V\"", rc, &r->uri, &r->args);
 
+#if (NGX_HAVE_AIO_SENDFILE)
+
+    if (c->busy_sendfile) {
+        off_t                  offset;
+        ngx_file_t            *file;
+        ngx_http_ephemeral_t  *e;
+
+        file = c->busy_sendfile->file;
+        offset = c->busy_sendfile->file_pos;
+
+        if (file->aio) {
+            c->aio_sendfile = (offset != file->aio->last_offset);
+            file->aio->last_offset = offset;
+
+            if (c->aio_sendfile == 0) {
+                ngx_log_error(NGX_LOG_ALERT, c->log, 0,
+                              "sendfile(%V) returned busy again", &file->name);
+            }
+        }
+
+        c->busy_sendfile = NULL;
+        e = (ngx_http_ephemeral_t *) &r->uri_start;
+
+        (void) ngx_file_aio_read(file, e->preload, 4, offset, r->pool);
+
+        if (file->aio) {
+            file->aio->data = r;
+            file->aio->handler = ngx_http_copy_aio_sendfile_event_handler;
+
+            r->main->blocked++;
+            r->aio = 1;
+        }
+    }
+
+#endif
+
     return rc;
 }
 
@@ -175,6 +217,26 @@ ngx_http_copy_aio_event_handler(ngx_event_t *ev)
     r->connection->write->handler(r->connection->write);
 }
 
+
+#if (NGX_HAVE_AIO_SENDFILE)
+
+static void
+ngx_http_copy_aio_sendfile_event_handler(ngx_event_t *ev)
+{
+    ngx_event_aio_t     *aio;
+    ngx_http_request_t  *r;
+
+    aio = ev->data;
+    r = aio->data;
+
+    r->main->blocked--;
+    r->aio = 0;
+    ev->complete = 0;
+
+    r->connection->write->handler(r->connection->write);
+}
+
+#endif
 #endif
 
 

src/http/ngx_http_core_module.c

@@ -104,6 +104,20 @@ static ngx_conf_enum_t  ngx_http_core_request_body_in_file[] = {
 };
 
 
+#if (NGX_HAVE_FILE_AIO)
+
+static ngx_conf_enum_t  ngx_http_core_aio[] = {
+    { ngx_string("off"), NGX_HTTP_AIO_OFF  },
+    { ngx_string("on"), NGX_HTTP_AIO_ON },
+#if (NGX_HAVE_AIO_SENDFILE)
+    { ngx_string("sendfile"), NGX_HTTP_AIO_SENDFILE },
+#endif
+    { ngx_null_string, 0 }
+};
+
+#endif
+
+
 static ngx_conf_enum_t  ngx_http_core_satisfy[] = {
     { ngx_string("all"), NGX_HTTP_SATISFY_ALL },
     { ngx_string("any"), NGX_HTTP_SATISFY_ANY },
@@ -386,11 +400,11 @@ static ngx_command_t  ngx_http_core_commands[] = {
 #if (NGX_HAVE_FILE_AIO)
 
     { ngx_string("aio"),
-      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG,
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
-      ngx_conf_set_flag_slot,
+      ngx_conf_set_enum_slot,
       NGX_HTTP_LOC_CONF_OFFSET,
       offsetof(ngx_http_core_loc_conf_t, aio),
-      NULL },
+      &ngx_http_core_aio },
 
 #endif
 

src/http/ngx_http_core_module.h

@@ -24,6 +24,11 @@
 #define NGX_HTTP_GZIP_PROXIED_ANY       0x0200
 
 
+#define NGX_HTTP_AIO_OFF                0
+#define NGX_HTTP_AIO_ON                 1
+#define NGX_HTTP_AIO_SENDFILE           2
+
+
 #define NGX_HTTP_SATISFY_ALL            0
 #define NGX_HTTP_SATISFY_ANY            1
 

src/http/ngx_http_request.h

@@ -546,6 +546,9 @@ struct ngx_http_request_s {
 
 typedef struct {
     ngx_http_posted_request_t         terminal_posted_request;
+#if (NGX_HAVE_AIO_SENDFILE)
+    u_char                            preload[4];
+#endif
 } ngx_http_ephemeral_t;
 
 

src/os/unix/ngx_file_aio_read.c

@@ -15,7 +15,8 @@
  *    if an asked data are already in VM cache, then aio_error() returns 0,
  *    and the data are already copied in buffer;
  *
- *    aio_read() preread in VM cache as minimum 32K;
+ *    aio_read() preread in VM cache as minimum 16K (probably BKVASIZE);
+ *    the first AIO preload may be up to 128K;
  *
  *    aio_read/aio_error() may return EINPROGRESS for just written data;
  *
@@ -60,6 +61,9 @@ ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset,
         aio->event.data = aio;
         aio->event.ready = 1;
         aio->event.log = file->log;
+#if (NGX_HAVE_AIO_SENDFILE)
+        aio->last_offset = -1;
+#endif
         file->aio = aio;
     }
 

src/os/unix/ngx_freebsd_sendfile_chain.c

@@ -40,7 +40,7 @@
 ngx_chain_t *
 ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
 {
-    int              rc;
+    int              rc, flags;
     u_char          *prev;
     off_t            size, send, prev_send, aligned, sent, fprev;
     size_t           header_size, file_size;
@@ -78,6 +78,7 @@ ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
 
     send = 0;
     eagain = 0;
+    flags = 0;
 
     header.elts = headers;
     header.size = sizeof(struct iovec);
@@ -261,8 +262,12 @@ ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
 
             sent = 0;
 
+#if (NGX_HAVE_AIO_SENDFILE)
+            flags = c->aio_sendfile ? SF_NODISKIO : 0;
+#endif
+
             rc = sendfile(file->file->fd, c->fd, file->file_pos,
-                          file_size + header_size, &hdtr, &sent, 0);
+                          file_size + header_size, &hdtr, &sent, flags);
 
             if (rc == -1) {
                 err = ngx_errno;
@@ -276,6 +281,12 @@ ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
                     eintr = 1;
                     break;
 
+#if (NGX_HAVE_AIO_SENDFILE)
+                case NGX_EBUSY:
+                    c->busy_sendfile = file;
+                    break;
+#endif
+
                 default:
                     wev->error = 1;
                     (void) ngx_connection_error(c, err, "sendfile() failed");
@@ -383,6 +394,12 @@ ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
             break;
         }
 
+#if (NGX_HAVE_AIO_SENDFILE)
+        if (c->busy_sendfile) {
+            return cl;
+        }
+#endif
+
         if (eagain) {
 
             /*