From 8165597cf6b52d0129057185e19e37a83621c51e Mon Sep 17 00:00:00 2001
From: Roman Arutyunyan <arut@nginx.com>
Date: Tue, 14 Sep 2021 12:12:02 +0300
Subject: [PATCH 01/27] Version bump.

---
 src/core/nginx.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/core/nginx.h b/src/core/nginx.h
index 6b134945e..bc4af23af 100644
--- a/src/core/nginx.h
+++ b/src/core/nginx.h
@@ -9,8 +9,8 @@
 #define _NGINX_H_INCLUDED_
 
 
-#define nginx_version      1021003
-#define NGINX_VERSION      "1.21.3"
+#define nginx_version      1021004
+#define NGINX_VERSION      "1.21.4"
 #define NGINX_VER          "nginx/" NGINX_VERSION
 
 #ifdef NGX_BUILD

From adbd28b7a9843ebce895f9903f19c5a740d61db4 Mon Sep 17 00:00:00 2001
From: Roman Arutyunyan <arut@nginx.com>
Date: Fri, 10 Sep 2021 12:59:22 +0300
Subject: [PATCH 02/27] Request body: do not create temp file if there's
 nothing to write.

Do this only when the entire request body is empty and
r->request_body_in_file_only is set.

The issue manifested itself with missing warning "a client request body is
buffered to a temporary file" when the entire rb->buf is full and all buffers
are delayed by a filter.
---
 src/http/ngx_http_request_body.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/http/ngx_http_request_body.c b/src/http/ngx_http_request_body.c
index 89a4c7492..ad3549f98 100644
--- a/src/http/ngx_http_request_body.c
+++ b/src/http/ngx_http_request_body.c
@@ -1309,7 +1309,7 @@ ngx_http_request_body_save_filter(ngx_http_request_t *r, ngx_chain_t *in)
 
     if (rb->rest > 0) {
 
-        if (rb->buf && rb->buf->last == rb->buf->end
+        if (rb->bufs && rb->buf && rb->buf->last == rb->buf->end
             && ngx_http_write_request_body(r) != NGX_OK)
         {
             return NGX_HTTP_INTERNAL_SERVER_ERROR;

From e56ba23158b8466d108fd4d571bd7d9a88f2a473 Mon Sep 17 00:00:00 2001
From: Vladimir Homutov <vl@nginx.com>
Date: Wed, 22 Sep 2021 10:20:00 +0300
Subject: [PATCH 03/27] Stream: added half-close support.

The "proxy_half_close" directive enables handling of TCP half close.  If
enabled, connection to proxied server is kept open until both read ends get
EOF.  Write end shutdown is properly transmitted via proxy.
---
 src/stream/ngx_stream_proxy_module.c | 36 ++++++++++++++++++++++++++++
 src/stream/ngx_stream_upstream.h     |  1 +
 2 files changed, 37 insertions(+)

diff --git a/src/stream/ngx_stream_proxy_module.c b/src/stream/ngx_stream_proxy_module.c
index 1275cf225..934e7d8f2 100644
--- a/src/stream/ngx_stream_proxy_module.c
+++ b/src/stream/ngx_stream_proxy_module.c
@@ -31,6 +31,7 @@ typedef struct {
     ngx_uint_t                       next_upstream_tries;
     ngx_flag_t                       next_upstream;
     ngx_flag_t                       proxy_protocol;
+    ngx_flag_t                       half_close;
     ngx_stream_upstream_local_t     *local;
     ngx_flag_t                       socket_keepalive;
 
@@ -245,6 +246,13 @@ static ngx_command_t  ngx_stream_proxy_commands[] = {
       offsetof(ngx_stream_proxy_srv_conf_t, proxy_protocol),
       NULL },
 
+    { ngx_string("proxy_half_close"),
+      NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_FLAG,
+      ngx_conf_set_flag_slot,
+      NGX_STREAM_SRV_CONF_OFFSET,
+      offsetof(ngx_stream_proxy_srv_conf_t, half_close),
+      NULL },
+
 #if (NGX_STREAM_SSL)
 
     { ngx_string("proxy_ssl"),
@@ -1755,6 +1763,24 @@ ngx_stream_proxy_process(ngx_stream_session_t *s, ngx_uint_t from_upstream,
     }
 
     if (dst) {
+
+        if (dst->type == SOCK_STREAM && pscf->half_close
+            && src->read->eof && !u->half_closed && !dst->buffered)
+        {
+            if (ngx_shutdown_socket(dst->fd, NGX_WRITE_SHUTDOWN) == -1) {
+                ngx_connection_error(c, ngx_socket_errno,
+                                     ngx_shutdown_socket_n " failed");
+
+                ngx_stream_proxy_finalize(s, NGX_STREAM_INTERNAL_SERVER_ERROR);
+                return;
+            }
+
+            u->half_closed = 1;
+            ngx_log_debug1(NGX_LOG_DEBUG_STREAM, s->connection->log, 0,
+                           "stream proxy %s socket shutdown",
+                           from_upstream ? "client" : "upstream");
+        }
+
         if (ngx_handle_write_event(dst->write, 0) != NGX_OK) {
             ngx_stream_proxy_finalize(s, NGX_STREAM_INTERNAL_SERVER_ERROR);
             return;
@@ -1833,6 +1859,13 @@ ngx_stream_proxy_test_finalize(ngx_stream_session_t *s,
         return NGX_DECLINED;
     }
 
+    if (pscf->half_close) {
+        /* avoid closing live connections until both read ends get EOF */
+        if (!(c->read->eof && pc->read->eof && !c->buffered && !pc->buffered)) {
+             return NGX_DECLINED;
+        }
+    }
+
     handler = c->log->handler;
     c->log->handler = NULL;
 
@@ -2052,6 +2085,7 @@ ngx_stream_proxy_create_srv_conf(ngx_conf_t *cf)
     conf->proxy_protocol = NGX_CONF_UNSET;
     conf->local = NGX_CONF_UNSET_PTR;
     conf->socket_keepalive = NGX_CONF_UNSET;
+    conf->half_close = NGX_CONF_UNSET;
 
 #if (NGX_STREAM_SSL)
     conf->ssl_enable = NGX_CONF_UNSET;
@@ -2110,6 +2144,8 @@ ngx_stream_proxy_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child)
     ngx_conf_merge_value(conf->socket_keepalive,
                               prev->socket_keepalive, 0);
 
+    ngx_conf_merge_value(conf->half_close, prev->half_close, 0);
+
 #if (NGX_STREAM_SSL)
 
     ngx_conf_merge_value(conf->ssl_enable, prev->ssl_enable, 0);
diff --git a/src/stream/ngx_stream_upstream.h b/src/stream/ngx_stream_upstream.h
index 9857e0b75..f5617794f 100644
--- a/src/stream/ngx_stream_upstream.h
+++ b/src/stream/ngx_stream_upstream.h
@@ -142,6 +142,7 @@ typedef struct {
     ngx_stream_upstream_state_t       *state;
     unsigned                           connected:1;
     unsigned                           proxy_protocol:1;
+    unsigned                           half_closed:1;
 } ngx_stream_upstream_t;
 
 

From c9c3f2f005c43fd885bd78f86e1044261e639bea Mon Sep 17 00:00:00 2001
From: Maxim Dounin <mdounin@mdounin.ru>
Date: Wed, 6 Oct 2021 18:01:42 +0300
Subject: [PATCH 04/27] Fixed $content_length cacheability with chunked (ticket
 #2252).

---
 src/http/ngx_http_variables.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/http/ngx_http_variables.c b/src/http/ngx_http_variables.c
index c25d80ccf..942dacd70 100644
--- a/src/http/ngx_http_variables.c
+++ b/src/http/ngx_http_variables.c
@@ -1179,6 +1179,10 @@ ngx_http_variable_content_length(ngx_http_request_t *r,
         v->no_cacheable = 0;
         v->not_found = 0;
 
+    } else if (r->headers_in.chunked) {
+        v->not_found = 1;
+        v->no_cacheable = 1;
+
     } else {
         v->not_found = 1;
     }

From 2a88047ac1a88fbe6038a822e0f1b67af847c998 Mon Sep 17 00:00:00 2001
From: Awdhesh Mathpal <mathpal@amazon.com>
Date: Thu, 7 Oct 2021 19:23:11 -0700
Subject: [PATCH 05/27] Proxy: disabled keepalive on extra data in non-buffered
 mode.

The u->keepalive flag is initialized early if the response has no body
(or an empty body), and needs to be reset if there are any extra data,
similarly to how it is done in ngx_http_proxy_copy_filter().  Missed
in 83c4622053b0.
---
 src/http/modules/ngx_http_proxy_module.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c
index 084462746..a5cc27b1e 100644
--- a/src/http/modules/ngx_http_proxy_module.c
+++ b/src/http/modules/ngx_http_proxy_module.c
@@ -2337,6 +2337,7 @@ ngx_http_proxy_non_buffered_copy_filter(void *data, ssize_t bytes)
         ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
                       "upstream sent more data than specified in "
                       "\"Content-Length\" header");
+        u->keepalive = 0;
         return NGX_OK;
     }
 

From 6119609cae6591320e5fc4b7d8e4546f59d40632 Mon Sep 17 00:00:00 2001
From: Maxim Dounin <mdounin@mdounin.ru>
Date: Tue, 12 Oct 2021 23:18:18 +0300
Subject: [PATCH 06/27] Synced ngx_http_subrequest() argument names (ticket
 #2255).

---
 src/http/ngx_http_core_module.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/http/ngx_http_core_module.h b/src/http/ngx_http_core_module.h
index 002d4193a..004a98eef 100644
--- a/src/http/ngx_http_core_module.h
+++ b/src/http/ngx_http_core_module.h
@@ -502,8 +502,8 @@ ngx_int_t ngx_http_gzip_ok(ngx_http_request_t *r);
 
 
 ngx_int_t ngx_http_subrequest(ngx_http_request_t *r,
-    ngx_str_t *uri, ngx_str_t *args, ngx_http_request_t **sr,
-    ngx_http_post_subrequest_t *psr, ngx_uint_t flags);
+    ngx_str_t *uri, ngx_str_t *args, ngx_http_request_t **psr,
+    ngx_http_post_subrequest_t *ps, ngx_uint_t flags);
 ngx_int_t ngx_http_internal_redirect(ngx_http_request_t *r,
     ngx_str_t *uri, ngx_str_t *args);
 ngx_int_t ngx_http_named_location(ngx_http_request_t *r, ngx_str_t *name);

From dde319ee0c9de26a6f104feb062cfffaa32c16c0 Mon Sep 17 00:00:00 2001
From: Maxim Dounin <mdounin@mdounin.ru>
Date: Mon, 18 Oct 2021 16:46:59 +0300
Subject: [PATCH 07/27] Upstream: fixed logging level of upstream invalid
 header errors.

In b87b7092cedb (nginx 1.21.1), logging level of "upstream sent invalid
header" errors was accidentally changed to "info".  This change restores
the "error" level, which is a proper logging level for upstream-side
errors.
---
 src/http/modules/ngx_http_fastcgi_module.c | 2 +-
 src/http/modules/ngx_http_proxy_module.c   | 2 +-
 src/http/modules/ngx_http_scgi_module.c    | 2 +-
 src/http/modules/ngx_http_uwsgi_module.c   | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/http/modules/ngx_http_fastcgi_module.c b/src/http/modules/ngx_http_fastcgi_module.c
index 69ac0f72c..4a8dc338e 100644
--- a/src/http/modules/ngx_http_fastcgi_module.c
+++ b/src/http/modules/ngx_http_fastcgi_module.c
@@ -2021,7 +2021,7 @@ ngx_http_fastcgi_process_header(ngx_http_request_t *r)
 
             /* rc == NGX_HTTP_PARSE_INVALID_HEADER */
 
-            ngx_log_error(NGX_LOG_INFO, r->connection->log, 0,
+            ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
                           "upstream sent invalid header: \"%*s\\x%02xd...\"",
                           r->header_end - r->header_name_start,
                           r->header_name_start, *r->header_end);
diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c
index a5cc27b1e..7c4061c02 100644
--- a/src/http/modules/ngx_http_proxy_module.c
+++ b/src/http/modules/ngx_http_proxy_module.c
@@ -2021,7 +2021,7 @@ ngx_http_proxy_process_header(ngx_http_request_t *r)
 
         /* rc == NGX_HTTP_PARSE_INVALID_HEADER */
 
-        ngx_log_error(NGX_LOG_INFO, r->connection->log, 0,
+        ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
                       "upstream sent invalid header: \"%*s\\x%02xd...\"",
                       r->header_end - r->header_name_start,
                       r->header_name_start, *r->header_end);
diff --git a/src/http/modules/ngx_http_scgi_module.c b/src/http/modules/ngx_http_scgi_module.c
index 570713df9..e5d31ae91 100644
--- a/src/http/modules/ngx_http_scgi_module.c
+++ b/src/http/modules/ngx_http_scgi_module.c
@@ -1142,7 +1142,7 @@ ngx_http_scgi_process_header(ngx_http_request_t *r)
 
         /* rc == NGX_HTTP_PARSE_INVALID_HEADER */
 
-        ngx_log_error(NGX_LOG_INFO, r->connection->log, 0,
+        ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
                       "upstream sent invalid header: \"%*s\\x%02xd...\"",
                       r->header_end - r->header_name_start,
                       r->header_name_start, *r->header_end);
diff --git a/src/http/modules/ngx_http_uwsgi_module.c b/src/http/modules/ngx_http_uwsgi_module.c
index 4f9c349c2..d46741a00 100644
--- a/src/http/modules/ngx_http_uwsgi_module.c
+++ b/src/http/modules/ngx_http_uwsgi_module.c
@@ -1363,7 +1363,7 @@ ngx_http_uwsgi_process_header(ngx_http_request_t *r)
 
         /* rc == NGX_HTTP_PARSE_INVALID_HEADER */
 
-        ngx_log_error(NGX_LOG_INFO, r->connection->log, 0,
+        ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
                       "upstream sent invalid header: \"%*s\\x%02xd...\"",
                       r->header_end - r->header_name_start,
                       r->header_name_start, *r->header_end);

From 1db517fb71aed6d6fffc8347086f89eb29b83dea Mon Sep 17 00:00:00 2001
From: Vladimir Homutov <vl@nginx.com>
Date: Fri, 15 Oct 2021 10:02:15 +0300
Subject: [PATCH 08/27] HTTP/2: removed support for NPN.

NPN was replaced with ALPN, published as RFC 7301 in July 2014.
It used to negotiate SPDY (and, in transition, HTTP/2).

NPN supported appeared in OpenSSL 1.0.1. It does not work with TLSv1.3 [1].
ALPN is supported since OpenSSL 1.0.2.

The NPN support was dropped in Firefox 53 [2] and Chrome 51 [3].

[1] https://github.com/openssl/openssl/issues/3665.
[2] https://bugzilla.mozilla.org/show_bug.cgi?id=1248198
[3] https://www.chromestatus.com/feature/5767920709795840
---
 src/http/modules/ngx_http_ssl_module.c | 59 +++-----------------------
 src/http/ngx_http.c                    |  5 +--
 src/http/ngx_http_request.c            | 14 +-----
 src/http/v2/ngx_http_v2.h              |  3 +-
 4 files changed, 9 insertions(+), 72 deletions(-)

diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c
index 1a744fff1..ed399f0cd 100644
--- a/src/http/modules/ngx_http_ssl_module.c
+++ b/src/http/modules/ngx_http_ssl_module.c
@@ -17,7 +17,7 @@ typedef ngx_int_t (*ngx_ssl_variable_handler_pt)(ngx_connection_t *c,
 #define NGX_DEFAULT_CIPHERS     "HIGH:!aNULL:!MD5"
 #define NGX_DEFAULT_ECDH_CURVE  "auto"
 
-#define NGX_HTTP_NPN_ADVERTISE  "\x08http/1.1"
+#define NGX_HTTP_ALPN_PROTO     "\x08http/1.1"
 
 
 #ifdef TLSEXT_TYPE_application_layer_protocol_negotiation
@@ -26,11 +26,6 @@ static int ngx_http_ssl_alpn_select(ngx_ssl_conn_t *ssl_conn,
     const unsigned char *in, unsigned int inlen, void *arg);
 #endif
 
-#ifdef TLSEXT_TYPE_next_proto_neg
-static int ngx_http_ssl_npn_advertised(ngx_ssl_conn_t *ssl_conn,
-    const unsigned char **out, unsigned int *outlen, void *arg);
-#endif
-
 static ngx_int_t ngx_http_ssl_static_variable(ngx_http_request_t *r,
     ngx_http_variable_value_t *v, uintptr_t data);
 static ngx_int_t ngx_http_ssl_variable(ngx_http_request_t *r,
@@ -444,15 +439,14 @@ ngx_http_ssl_alpn_select(ngx_ssl_conn_t *ssl_conn, const unsigned char **out,
     hc = c->data;
 
     if (hc->addr_conf->http2) {
-        srv =
-           (unsigned char *) NGX_HTTP_V2_ALPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE;
-        srvlen = sizeof(NGX_HTTP_V2_ALPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE) - 1;
+        srv = (unsigned char *) NGX_HTTP_V2_ALPN_PROTO NGX_HTTP_ALPN_PROTO;
+        srvlen = sizeof(NGX_HTTP_V2_ALPN_PROTO NGX_HTTP_ALPN_PROTO) - 1;
 
     } else
 #endif
     {
-        srv = (unsigned char *) NGX_HTTP_NPN_ADVERTISE;
-        srvlen = sizeof(NGX_HTTP_NPN_ADVERTISE) - 1;
+        srv = (unsigned char *) NGX_HTTP_ALPN_PROTO;
+        srvlen = sizeof(NGX_HTTP_ALPN_PROTO) - 1;
     }
 
     if (SSL_select_next_proto((unsigned char **) out, outlen, srv, srvlen,
@@ -471,44 +465,6 @@ ngx_http_ssl_alpn_select(ngx_ssl_conn_t *ssl_conn, const unsigned char **out,
 #endif
 
 
-#ifdef TLSEXT_TYPE_next_proto_neg
-
-static int
-ngx_http_ssl_npn_advertised(ngx_ssl_conn_t *ssl_conn,
-    const unsigned char **out, unsigned int *outlen, void *arg)
-{
-#if (NGX_HTTP_V2 || NGX_DEBUG)
-    ngx_connection_t  *c;
-
-    c = ngx_ssl_get_connection(ssl_conn);
-    ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "SSL NPN advertised");
-#endif
-
-#if (NGX_HTTP_V2)
-    {
-    ngx_http_connection_t  *hc;
-
-    hc = c->data;
-
-    if (hc->addr_conf->http2) {
-        *out =
-            (unsigned char *) NGX_HTTP_V2_NPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE;
-        *outlen = sizeof(NGX_HTTP_V2_NPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE) - 1;
-
-        return SSL_TLSEXT_ERR_OK;
-    }
-    }
-#endif
-
-    *out = (unsigned char *) NGX_HTTP_NPN_ADVERTISE;
-    *outlen = sizeof(NGX_HTTP_NPN_ADVERTISE) - 1;
-
-    return SSL_TLSEXT_ERR_OK;
-}
-
-#endif
-
-
 static ngx_int_t
 ngx_http_ssl_static_variable(ngx_http_request_t *r,
     ngx_http_variable_value_t *v, uintptr_t data)
@@ -792,11 +748,6 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child)
     SSL_CTX_set_alpn_select_cb(conf->ssl.ctx, ngx_http_ssl_alpn_select, NULL);
 #endif
 
-#ifdef TLSEXT_TYPE_next_proto_neg
-    SSL_CTX_set_next_protos_advertised_cb(conf->ssl.ctx,
-                                          ngx_http_ssl_npn_advertised, NULL);
-#endif
-
     if (ngx_ssl_ciphers(cf, &conf->ssl, &conf->ciphers,
                         conf->prefer_server_ciphers)
         != NGX_OK)
diff --git a/src/http/ngx_http.c b/src/http/ngx_http.c
index 3d5c4ac16..73c08d593 100644
--- a/src/http/ngx_http.c
+++ b/src/http/ngx_http.c
@@ -1338,13 +1338,12 @@ ngx_http_add_address(ngx_conf_t *cf, ngx_http_core_srv_conf_t *cscf,
     }
 
 #if (NGX_HTTP_V2 && NGX_HTTP_SSL                                              \
-     && !defined TLSEXT_TYPE_application_layer_protocol_negotiation           \
-     && !defined TLSEXT_TYPE_next_proto_neg)
+     && !defined TLSEXT_TYPE_application_layer_protocol_negotiation)
 
     if (lsopt->http2 && lsopt->ssl) {
         ngx_conf_log_error(NGX_LOG_WARN, cf, 0,
                            "nginx was built with OpenSSL that lacks ALPN "
-                           "and NPN support, HTTP/2 is not enabled for %V",
+                           "support, HTTP/2 is not enabled for %V",
                            &lsopt->addr_text);
     }
 
diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c
index bf931bf35..76bbcd67c 100644
--- a/src/http/ngx_http_request.c
+++ b/src/http/ngx_http_request.c
@@ -806,8 +806,7 @@ ngx_http_ssl_handshake_handler(ngx_connection_t *c)
         c->ssl->no_wait_shutdown = 1;
 
 #if (NGX_HTTP_V2                                                              \
-     && (defined TLSEXT_TYPE_application_layer_protocol_negotiation           \
-         || defined TLSEXT_TYPE_next_proto_neg))
+     && defined TLSEXT_TYPE_application_layer_protocol_negotiation)
         {
         unsigned int            len;
         const unsigned char    *data;
@@ -817,19 +816,8 @@ ngx_http_ssl_handshake_handler(ngx_connection_t *c)
 
         if (hc->addr_conf->http2) {
 
-#ifdef TLSEXT_TYPE_application_layer_protocol_negotiation
             SSL_get0_alpn_selected(c->ssl->connection, &data, &len);
 
-#ifdef TLSEXT_TYPE_next_proto_neg
-            if (len == 0) {
-                SSL_get0_next_proto_negotiated(c->ssl->connection, &data, &len);
-            }
-#endif
-
-#else /* TLSEXT_TYPE_next_proto_neg */
-            SSL_get0_next_proto_negotiated(c->ssl->connection, &data, &len);
-#endif
-
             if (len == 2 && data[0] == 'h' && data[1] == '2') {
                 ngx_http_v2_init(c->read);
                 return;
diff --git a/src/http/v2/ngx_http_v2.h b/src/http/v2/ngx_http_v2.h
index 349229711..0eceae3d5 100644
--- a/src/http/v2/ngx_http_v2.h
+++ b/src/http/v2/ngx_http_v2.h
@@ -13,8 +13,7 @@
 #include <ngx_http.h>
 
 
-#define NGX_HTTP_V2_ALPN_ADVERTISE       "\x02h2"
-#define NGX_HTTP_V2_NPN_ADVERTISE        NGX_HTTP_V2_ALPN_ADVERTISE
+#define NGX_HTTP_V2_ALPN_PROTO           "\x02h2"
 
 #define NGX_HTTP_V2_STATE_BUFFER_SIZE    16
 

From a9f4f25b72c39653795dfb4b1f13b55625fb9fbc Mon Sep 17 00:00:00 2001
From: Vladimir Homutov <vl@nginx.com>
Date: Thu, 14 Oct 2021 11:46:23 +0300
Subject: [PATCH 09/27] SSL: added $ssl_alpn_protocol variable.

The variable contains protocol selected by ALPN during handshake and
is empty otherwise.
---
 src/event/ngx_event_openssl.c          | 30 ++++++++++++++++++++++++++
 src/event/ngx_event_openssl.h          |  2 ++
 src/http/modules/ngx_http_ssl_module.c |  3 +++
 src/stream/ngx_stream_ssl_module.c     |  3 +++
 4 files changed, 38 insertions(+)

diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c
index c087884ce..52efea3c6 100644
--- a/src/event/ngx_event_openssl.c
+++ b/src/event/ngx_event_openssl.c
@@ -4698,6 +4698,36 @@ ngx_ssl_get_server_name(ngx_connection_t *c, ngx_pool_t *pool, ngx_str_t *s)
 }
 
 
+ngx_int_t
+ngx_ssl_get_alpn_protocol(ngx_connection_t *c, ngx_pool_t *pool, ngx_str_t *s)
+{
+#ifdef TLSEXT_TYPE_application_layer_protocol_negotiation
+
+    unsigned int          len;
+    const unsigned char  *data;
+
+    SSL_get0_alpn_selected(c->ssl->connection, &data, &len);
+
+    if (len > 0) {
+
+        s->data = ngx_pnalloc(pool, len);
+        if (s->data == NULL) {
+            return NGX_ERROR;
+        }
+
+        ngx_memcpy(s->data, data, len);
+        s->len = len;
+
+        return NGX_OK;
+    }
+
+#endif
+
+    s->len = 0;
+    return NGX_OK;
+}
+
+
 ngx_int_t
 ngx_ssl_get_raw_certificate(ngx_connection_t *c, ngx_pool_t *pool, ngx_str_t *s)
 {
diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h
index 68e970b89..e7cb06e70 100644
--- a/src/event/ngx_event_openssl.h
+++ b/src/event/ngx_event_openssl.h
@@ -265,6 +265,8 @@ ngx_int_t ngx_ssl_get_early_data(ngx_connection_t *c, ngx_pool_t *pool,
     ngx_str_t *s);
 ngx_int_t ngx_ssl_get_server_name(ngx_connection_t *c, ngx_pool_t *pool,
     ngx_str_t *s);
+ngx_int_t ngx_ssl_get_alpn_protocol(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
 ngx_int_t ngx_ssl_get_raw_certificate(ngx_connection_t *c, ngx_pool_t *pool,
     ngx_str_t *s);
 ngx_int_t ngx_ssl_get_certificate(ngx_connection_t *c, ngx_pool_t *pool,
diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c
index ed399f0cd..87f038898 100644
--- a/src/http/modules/ngx_http_ssl_module.c
+++ b/src/http/modules/ngx_http_ssl_module.c
@@ -358,6 +358,9 @@ static ngx_http_variable_t  ngx_http_ssl_vars[] = {
     { ngx_string("ssl_server_name"), NULL, ngx_http_ssl_variable,
       (uintptr_t) ngx_ssl_get_server_name, NGX_HTTP_VAR_CHANGEABLE, 0 },
 
+    { ngx_string("ssl_alpn_protocol"), NULL, ngx_http_ssl_variable,
+      (uintptr_t) ngx_ssl_get_alpn_protocol, NGX_HTTP_VAR_CHANGEABLE, 0 },
+
     { ngx_string("ssl_client_cert"), NULL, ngx_http_ssl_variable,
       (uintptr_t) ngx_ssl_get_certificate, NGX_HTTP_VAR_CHANGEABLE, 0 },
 
diff --git a/src/stream/ngx_stream_ssl_module.c b/src/stream/ngx_stream_ssl_module.c
index b7350002c..0e42e080a 100644
--- a/src/stream/ngx_stream_ssl_module.c
+++ b/src/stream/ngx_stream_ssl_module.c
@@ -266,6 +266,9 @@ static ngx_stream_variable_t  ngx_stream_ssl_vars[] = {
     { ngx_string("ssl_server_name"), NULL, ngx_stream_ssl_variable,
       (uintptr_t) ngx_ssl_get_server_name, NGX_STREAM_VAR_CHANGEABLE, 0 },
 
+    { ngx_string("ssl_alpn_protocol"), NULL, ngx_stream_ssl_variable,
+      (uintptr_t) ngx_ssl_get_alpn_protocol, NGX_STREAM_VAR_CHANGEABLE, 0 },
+
     { ngx_string("ssl_client_cert"), NULL, ngx_stream_ssl_variable,
       (uintptr_t) ngx_ssl_get_certificate, NGX_STREAM_VAR_CHANGEABLE, 0 },
 

From df472eecc043700275ecae2655206163c786f758 Mon Sep 17 00:00:00 2001
From: Vladimir Homutov <vl@nginx.com>
Date: Tue, 19 Oct 2021 12:19:59 +0300
Subject: [PATCH 10/27] Stream: the "ssl_alpn" directive.

The directive sets the server list of supported application protocols
and requires one of this protocols to be negotiated if client is using
ALPN.
---
 src/event/ngx_event_openssl.c      |   3 +
 src/stream/ngx_stream_ssl_module.c | 117 +++++++++++++++++++++++++++++
 src/stream/ngx_stream_ssl_module.h |   1 +
 3 files changed, 121 insertions(+)

diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c
index 52efea3c6..fb0823017 100644
--- a/src/event/ngx_event_openssl.c
+++ b/src/event/ngx_event_openssl.c
@@ -3133,6 +3133,9 @@ ngx_ssl_connection_error(ngx_connection_t *c, int sslerr, ngx_err_t err,
 #endif
 #ifdef SSL_R_CALLBACK_FAILED
             || n == SSL_R_CALLBACK_FAILED                            /*  234 */
+#endif
+#ifdef SSL_R_NO_APPLICATION_PROTOCOL
+            || n == SSL_R_NO_APPLICATION_PROTOCOL                    /*  235 */
 #endif
             || n == SSL_R_UNEXPECTED_MESSAGE                         /*  244 */
             || n == SSL_R_UNEXPECTED_RECORD                          /*  245 */
diff --git a/src/stream/ngx_stream_ssl_module.c b/src/stream/ngx_stream_ssl_module.c
index 0e42e080a..4b6926ab8 100644
--- a/src/stream/ngx_stream_ssl_module.c
+++ b/src/stream/ngx_stream_ssl_module.c
@@ -25,6 +25,11 @@ static void ngx_stream_ssl_handshake_handler(ngx_connection_t *c);
 #ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME
 int ngx_stream_ssl_servername(ngx_ssl_conn_t *ssl_conn, int *ad, void *arg);
 #endif
+#ifdef TLSEXT_TYPE_application_layer_protocol_negotiation
+static int ngx_stream_ssl_alpn_select(ngx_ssl_conn_t *ssl_conn,
+    const unsigned char **out, unsigned char *outlen,
+    const unsigned char *in, unsigned int inlen, void *arg);
+#endif
 #ifdef SSL_R_CERT_CB_ERROR
 static int ngx_stream_ssl_certificate(ngx_ssl_conn_t *ssl_conn, void *arg);
 #endif
@@ -45,6 +50,8 @@ static char *ngx_stream_ssl_password_file(ngx_conf_t *cf, ngx_command_t *cmd,
     void *conf);
 static char *ngx_stream_ssl_session_cache(ngx_conf_t *cf, ngx_command_t *cmd,
     void *conf);
+static char *ngx_stream_ssl_alpn(ngx_conf_t *cf, ngx_command_t *cmd,
+    void *conf);
 
 static char *ngx_stream_ssl_conf_command_check(ngx_conf_t *cf, void *post,
     void *data);
@@ -211,6 +218,13 @@ static ngx_command_t  ngx_stream_ssl_commands[] = {
       offsetof(ngx_stream_ssl_conf_t, conf_commands),
       &ngx_stream_ssl_conf_command_post },
 
+    { ngx_string("ssl_alpn"),
+      NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_1MORE,
+      ngx_stream_ssl_alpn,
+      NGX_STREAM_SRV_CONF_OFFSET,
+      0,
+      NULL },
+
       ngx_null_command
 };
 
@@ -446,6 +460,46 @@ ngx_stream_ssl_servername(ngx_ssl_conn_t *ssl_conn, int *ad, void *arg)
 #endif
 
 
+#ifdef TLSEXT_TYPE_application_layer_protocol_negotiation
+
+static int
+ngx_stream_ssl_alpn_select(ngx_ssl_conn_t *ssl_conn, const unsigned char **out,
+    unsigned char *outlen, const unsigned char *in, unsigned int inlen,
+    void *arg)
+{
+    ngx_str_t         *alpn;
+#if (NGX_DEBUG)
+    unsigned int       i;
+    ngx_connection_t  *c;
+
+    c = ngx_ssl_get_connection(ssl_conn);
+
+    for (i = 0; i < inlen; i += in[i] + 1) {
+        ngx_log_debug2(NGX_LOG_DEBUG_STREAM, c->log, 0,
+                       "SSL ALPN supported by client: %*s",
+                       (size_t) in[i], &in[i + 1]);
+    }
+
+#endif
+
+    alpn = arg;
+
+    if (SSL_select_next_proto((unsigned char **) out, outlen, alpn->data,
+                              alpn->len, in, inlen)
+        != OPENSSL_NPN_NEGOTIATED)
+    {
+        return SSL_TLSEXT_ERR_ALERT_FATAL;
+    }
+
+    ngx_log_debug2(NGX_LOG_DEBUG_STREAM, c->log, 0,
+                   "SSL ALPN selected: %*s", (size_t) *outlen, *out);
+
+    return SSL_TLSEXT_ERR_OK;
+}
+
+#endif
+
+
 #ifdef SSL_R_CERT_CB_ERROR
 
 int
@@ -605,6 +659,7 @@ ngx_stream_ssl_create_conf(ngx_conf_t *cf)
      *     scf->client_certificate = { 0, NULL };
      *     scf->trusted_certificate = { 0, NULL };
      *     scf->crl = { 0, NULL };
+     *     scf->alpn = { 0, NULL };
      *     scf->ciphers = { 0, NULL };
      *     scf->shm_zone = NULL;
      */
@@ -663,6 +718,7 @@ ngx_stream_ssl_merge_conf(ngx_conf_t *cf, void *parent, void *child)
     ngx_conf_merge_str_value(conf->trusted_certificate,
                          prev->trusted_certificate, "");
     ngx_conf_merge_str_value(conf->crl, prev->crl, "");
+    ngx_conf_merge_str_value(conf->alpn, prev->alpn, "");
 
     ngx_conf_merge_str_value(conf->ecdh_curve, prev->ecdh_curve,
                          NGX_DEFAULT_ECDH_CURVE);
@@ -723,6 +779,13 @@ ngx_stream_ssl_merge_conf(ngx_conf_t *cf, void *parent, void *child)
                                            ngx_stream_ssl_servername);
 #endif
 
+#ifdef TLSEXT_TYPE_application_layer_protocol_negotiation
+    if (conf->alpn.len) {
+        SSL_CTX_set_alpn_select_cb(conf->ssl.ctx, ngx_stream_ssl_alpn_select,
+                                   &conf->alpn);
+    }
+#endif
+
     if (ngx_ssl_ciphers(cf, &conf->ssl, &conf->ciphers,
                         conf->prefer_server_ciphers)
         != NGX_OK)
@@ -1059,6 +1122,60 @@ invalid:
 }
 
 
+static char *
+ngx_stream_ssl_alpn(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
+{
+#ifdef TLSEXT_TYPE_application_layer_protocol_negotiation
+
+    ngx_stream_ssl_conf_t  *scf = conf;
+
+    u_char      *p;
+    size_t       len;
+    ngx_str_t   *value;
+    ngx_uint_t   i;
+
+    if (scf->alpn.len) {
+        return "is duplicate";
+    }
+
+    value = cf->args->elts;
+
+    len = 0;
+
+    for (i = 1; i < cf->args->nelts; i++) {
+
+        if (value[i].len > 255) {
+            return "protocol too long";
+        }
+
+        len += value[i].len + 1;
+    }
+
+    scf->alpn.data = ngx_pnalloc(cf->pool, len);
+    if (scf->alpn.data == NULL) {
+        return NGX_CONF_ERROR;
+    }
+
+    p = scf->alpn.data;
+
+    for (i = 1; i < cf->args->nelts; i++) {
+        *p++ = value[i].len;
+        p = ngx_cpymem(p, value[i].data, value[i].len);
+    }
+
+    scf->alpn.len = len;
+
+    return NGX_CONF_OK;
+
+#else
+    ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+                       "the \"ssl_alpn\" directive requires OpenSSL "
+                       "with ALPN support");
+    return NGX_CONF_ERROR;
+#endif
+}
+
+
 static char *
 ngx_stream_ssl_conf_command_check(ngx_conf_t *cf, void *post, void *data)
 {
diff --git a/src/stream/ngx_stream_ssl_module.h b/src/stream/ngx_stream_ssl_module.h
index c6e24bef3..e7c825e9b 100644
--- a/src/stream/ngx_stream_ssl_module.h
+++ b/src/stream/ngx_stream_ssl_module.h
@@ -42,6 +42,7 @@ typedef struct {
     ngx_str_t        client_certificate;
     ngx_str_t        trusted_certificate;
     ngx_str_t        crl;
+    ngx_str_t        alpn;
 
     ngx_str_t        ciphers;
 

From ebb6f7d6563f51ae8325e3c0f10e9c5a91004fda Mon Sep 17 00:00:00 2001
From: Vladimir Homutov <vl@nginx.com>
Date: Wed, 20 Oct 2021 09:50:02 +0300
Subject: [PATCH 11/27] HTTP: connections with wrong ALPN protocols are now
 rejected.

This is a recommended behavior by RFC 7301 and is useful
for mitigation of protocol confusion attacks [1].

To avoid possible negative effects, list of supported protocols
was extended to include all possible HTTP protocol ALPN IDs
registered by IANA [2], i.e. "http/1.0" and "http/0.9".

[1] https://alpaca-attack.com/
[2] https://www.iana.org/assignments/tls-extensiontype-values/
---
 src/http/modules/ngx_http_ssl_module.c | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c
index 87f038898..c633f3464 100644
--- a/src/http/modules/ngx_http_ssl_module.c
+++ b/src/http/modules/ngx_http_ssl_module.c
@@ -17,7 +17,7 @@ typedef ngx_int_t (*ngx_ssl_variable_handler_pt)(ngx_connection_t *c,
 #define NGX_DEFAULT_CIPHERS     "HIGH:!aNULL:!MD5"
 #define NGX_DEFAULT_ECDH_CURVE  "auto"
 
-#define NGX_HTTP_ALPN_PROTO     "\x08http/1.1"
+#define NGX_HTTP_ALPN_PROTOS    "\x08http/1.1\x08http/1.0\x08http/0.9"
 
 
 #ifdef TLSEXT_TYPE_application_layer_protocol_negotiation
@@ -442,21 +442,20 @@ ngx_http_ssl_alpn_select(ngx_ssl_conn_t *ssl_conn, const unsigned char **out,
     hc = c->data;
 
     if (hc->addr_conf->http2) {
-        srv = (unsigned char *) NGX_HTTP_V2_ALPN_PROTO NGX_HTTP_ALPN_PROTO;
-        srvlen = sizeof(NGX_HTTP_V2_ALPN_PROTO NGX_HTTP_ALPN_PROTO) - 1;
-
+        srv = (unsigned char *) NGX_HTTP_V2_ALPN_PROTO NGX_HTTP_ALPN_PROTOS;
+        srvlen = sizeof(NGX_HTTP_V2_ALPN_PROTO NGX_HTTP_ALPN_PROTOS) - 1;
     } else
 #endif
     {
-        srv = (unsigned char *) NGX_HTTP_ALPN_PROTO;
-        srvlen = sizeof(NGX_HTTP_ALPN_PROTO) - 1;
+        srv = (unsigned char *) NGX_HTTP_ALPN_PROTOS;
+        srvlen = sizeof(NGX_HTTP_ALPN_PROTOS) - 1;
     }
 
     if (SSL_select_next_proto((unsigned char **) out, outlen, srv, srvlen,
                               in, inlen)
         != OPENSSL_NPN_NEGOTIATED)
     {
-        return SSL_TLSEXT_ERR_NOACK;
+        return SSL_TLSEXT_ERR_ALERT_FATAL;
     }
 
     ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0,

From 1fecec0cbf1554c0473d5cca0fb55f8dc006e4ba Mon Sep 17 00:00:00 2001
From: Vladimir Homutov <vl@nginx.com>
Date: Wed, 20 Oct 2021 09:45:34 +0300
Subject: [PATCH 12/27] Mail: connections with wrong ALPN protocols are now
 rejected.

This is a recommended behavior by RFC 7301 and is useful
for mitigation of protocol confusion attacks [1].

For POP3 and IMAP protocols IANA-assigned ALPN IDs are used [2].
For the SMTP protocol "smtp" is used.

[1] https://alpaca-attack.com/
[2] https://www.iana.org/assignments/tls-extensiontype-values/
---
 src/mail/ngx_mail.h             |  1 +
 src/mail/ngx_mail_imap_module.c |  1 +
 src/mail/ngx_mail_pop3_module.c |  1 +
 src/mail/ngx_mail_smtp_module.c |  1 +
 src/mail/ngx_mail_ssl_module.c  | 58 +++++++++++++++++++++++++++++++++
 5 files changed, 62 insertions(+)

diff --git a/src/mail/ngx_mail.h b/src/mail/ngx_mail.h
index 21178c3e2..e0c62b7ab 100644
--- a/src/mail/ngx_mail.h
+++ b/src/mail/ngx_mail.h
@@ -324,6 +324,7 @@ typedef ngx_int_t (*ngx_mail_parse_command_pt)(ngx_mail_session_t *s);
 
 struct ngx_mail_protocol_s {
     ngx_str_t                   name;
+    ngx_str_t                   alpn;
     in_port_t                   port[4];
     ngx_uint_t                  type;
 
diff --git a/src/mail/ngx_mail_imap_module.c b/src/mail/ngx_mail_imap_module.c
index 1f187fdee..02c684cd4 100644
--- a/src/mail/ngx_mail_imap_module.c
+++ b/src/mail/ngx_mail_imap_module.c
@@ -46,6 +46,7 @@ static ngx_str_t  ngx_mail_imap_auth_methods_names[] = {
 
 static ngx_mail_protocol_t  ngx_mail_imap_protocol = {
     ngx_string("imap"),
+    ngx_string("\x04imap"),
     { 143, 993, 0, 0 },
     NGX_MAIL_IMAP_PROTOCOL,
 
diff --git a/src/mail/ngx_mail_pop3_module.c b/src/mail/ngx_mail_pop3_module.c
index a67307073..a257b5a70 100644
--- a/src/mail/ngx_mail_pop3_module.c
+++ b/src/mail/ngx_mail_pop3_module.c
@@ -46,6 +46,7 @@ static ngx_str_t  ngx_mail_pop3_auth_methods_names[] = {
 
 static ngx_mail_protocol_t  ngx_mail_pop3_protocol = {
     ngx_string("pop3"),
+    ngx_string("\x04pop3"),
     { 110, 995, 0, 0 },
     NGX_MAIL_POP3_PROTOCOL,
 
diff --git a/src/mail/ngx_mail_smtp_module.c b/src/mail/ngx_mail_smtp_module.c
index 3b5a2d8f3..0e05fdc03 100644
--- a/src/mail/ngx_mail_smtp_module.c
+++ b/src/mail/ngx_mail_smtp_module.c
@@ -39,6 +39,7 @@ static ngx_str_t  ngx_mail_smtp_auth_methods_names[] = {
 
 static ngx_mail_protocol_t  ngx_mail_smtp_protocol = {
     ngx_string("smtp"),
+    ngx_string("\x04smtp"),
     { 25, 465, 587, 0 },
     NGX_MAIL_SMTP_PROTOCOL,
 
diff --git a/src/mail/ngx_mail_ssl_module.c b/src/mail/ngx_mail_ssl_module.c
index 09cc425d6..2a1043e66 100644
--- a/src/mail/ngx_mail_ssl_module.c
+++ b/src/mail/ngx_mail_ssl_module.c
@@ -14,6 +14,12 @@
 #define NGX_DEFAULT_ECDH_CURVE  "auto"
 
 
+#ifdef TLSEXT_TYPE_application_layer_protocol_negotiation
+static int ngx_mail_ssl_alpn_select(ngx_ssl_conn_t *ssl_conn,
+    const unsigned char **out, unsigned char *outlen,
+    const unsigned char *in, unsigned int inlen, void *arg);
+#endif
+
 static void *ngx_mail_ssl_create_conf(ngx_conf_t *cf);
 static char *ngx_mail_ssl_merge_conf(ngx_conf_t *cf, void *parent, void *child);
 
@@ -244,6 +250,54 @@ ngx_module_t  ngx_mail_ssl_module = {
 static ngx_str_t ngx_mail_ssl_sess_id_ctx = ngx_string("MAIL");
 
 
+#ifdef TLSEXT_TYPE_application_layer_protocol_negotiation
+
+static int
+ngx_mail_ssl_alpn_select(ngx_ssl_conn_t *ssl_conn, const unsigned char **out,
+    unsigned char *outlen, const unsigned char *in, unsigned int inlen,
+    void *arg)
+{
+    unsigned int               srvlen;
+    unsigned char             *srv;
+    ngx_connection_t          *c;
+    ngx_mail_session_t        *s;
+    ngx_mail_core_srv_conf_t  *cscf;
+#if (NGX_DEBUG)
+    unsigned int               i;
+#endif
+
+    c = ngx_ssl_get_connection(ssl_conn);
+    s = c->data;
+
+#if (NGX_DEBUG)
+    for (i = 0; i < inlen; i += in[i] + 1) {
+        ngx_log_debug2(NGX_LOG_DEBUG_MAIL, c->log, 0,
+                       "SSL ALPN supported by client: %*s",
+                       (size_t) in[i], &in[i + 1]);
+    }
+#endif
+
+    cscf = ngx_mail_get_module_srv_conf(s, ngx_mail_core_module);
+
+    srv = cscf->protocol->alpn.data;
+    srvlen = cscf->protocol->alpn.len;
+
+    if (SSL_select_next_proto((unsigned char **) out, outlen, srv, srvlen,
+                              in, inlen)
+        != OPENSSL_NPN_NEGOTIATED)
+    {
+        return SSL_TLSEXT_ERR_ALERT_FATAL;
+    }
+
+    ngx_log_debug2(NGX_LOG_DEBUG_MAIL, c->log, 0,
+                   "SSL ALPN selected: %*s", (size_t) *outlen, *out);
+
+    return SSL_TLSEXT_ERR_OK;
+}
+
+#endif
+
+
 static void *
 ngx_mail_ssl_create_conf(ngx_conf_t *cf)
 {
@@ -394,6 +448,10 @@ ngx_mail_ssl_merge_conf(ngx_conf_t *cf, void *parent, void *child)
     cln->handler = ngx_ssl_cleanup_ctx;
     cln->data = &conf->ssl;
 
+#ifdef TLSEXT_TYPE_application_layer_protocol_negotiation
+    SSL_CTX_set_alpn_select_cb(conf->ssl.ctx, ngx_mail_ssl_alpn_select, NULL);
+#endif
+
     if (ngx_ssl_ciphers(cf, &conf->ssl, &conf->ciphers,
                         conf->prefer_server_ciphers)
         != NGX_OK)

From f29d7ade54f8baa117f7c40fec58683952c46cdb Mon Sep 17 00:00:00 2001
From: Maxim Dounin <mdounin@mdounin.ru>
Date: Thu, 21 Oct 2021 18:38:38 +0300
Subject: [PATCH 13/27] Removed CLOCK_MONOTONIC_COARSE support.

While clock_gettime(CLOCK_MONOTONIC_COARSE) is faster than
clock_gettime(CLOCK_MONOTONIC), the latter is fast enough on Linux for
practical usage, and the difference is negligible compared to other costs
at each event loop iteration.  On the other hand, CLOCK_MONOTONIC_COARSE
causes various issues with typical CONFIG_HZ=250, notably very inaccurate
limit_rate handling in some edge cases (ticket #1678) and negative difference
between $request_time and $upstream_response_time (ticket #1965).
---
 src/core/ngx_times.c | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/core/ngx_times.c b/src/core/ngx_times.c
index 7964b008f..16788c98c 100644
--- a/src/core/ngx_times.c
+++ b/src/core/ngx_times.c
@@ -200,10 +200,6 @@ ngx_monotonic_time(time_t sec, ngx_uint_t msec)
 
 #if defined(CLOCK_MONOTONIC_FAST)
     clock_gettime(CLOCK_MONOTONIC_FAST, &ts);
-
-#elif defined(CLOCK_MONOTONIC_COARSE)
-    clock_gettime(CLOCK_MONOTONIC_COARSE, &ts);
-
 #else
     clock_gettime(CLOCK_MONOTONIC, &ts);
 #endif

From 3ab1b64463cfc1cea5209f159d8fe3797328be51 Mon Sep 17 00:00:00 2001
From: Maxim Dounin <mdounin@mdounin.ru>
Date: Thu, 21 Oct 2021 18:43:13 +0300
Subject: [PATCH 14/27] Style: added missing "static" specifiers.

Mostly found by gcc -Wtraditional, per "non-static declaration of ...
follows static declaration [-Wtraditional]" warnings.
---
 src/event/ngx_event_openssl.c      | 2 +-
 src/stream/ngx_stream_ssl_module.c | 7 ++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c
index fb0823017..d4ba1f7d5 100644
--- a/src/event/ngx_event_openssl.c
+++ b/src/event/ngx_event_openssl.c
@@ -2767,7 +2767,7 @@ ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size)
 
 #ifdef SSL_READ_EARLY_DATA_SUCCESS
 
-ssize_t
+static ssize_t
 ngx_ssl_write_early(ngx_connection_t *c, u_char *data, size_t size)
 {
     int        n, sslerr;
diff --git a/src/stream/ngx_stream_ssl_module.c b/src/stream/ngx_stream_ssl_module.c
index 4b6926ab8..530fe8b3d 100644
--- a/src/stream/ngx_stream_ssl_module.c
+++ b/src/stream/ngx_stream_ssl_module.c
@@ -23,7 +23,8 @@ static ngx_int_t ngx_stream_ssl_init_connection(ngx_ssl_t *ssl,
     ngx_connection_t *c);
 static void ngx_stream_ssl_handshake_handler(ngx_connection_t *c);
 #ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME
-int ngx_stream_ssl_servername(ngx_ssl_conn_t *ssl_conn, int *ad, void *arg);
+static int ngx_stream_ssl_servername(ngx_ssl_conn_t *ssl_conn, int *ad,
+    void *arg);
 #endif
 #ifdef TLSEXT_TYPE_application_layer_protocol_negotiation
 static int ngx_stream_ssl_alpn_select(ngx_ssl_conn_t *ssl_conn,
@@ -451,7 +452,7 @@ ngx_stream_ssl_handshake_handler(ngx_connection_t *c)
 
 #ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME
 
-int
+static int
 ngx_stream_ssl_servername(ngx_ssl_conn_t *ssl_conn, int *ad, void *arg)
 {
     return SSL_TLSEXT_ERR_OK;
@@ -502,7 +503,7 @@ ngx_stream_ssl_alpn_select(ngx_ssl_conn_t *ssl_conn, const unsigned char **out,
 
 #ifdef SSL_R_CERT_CB_ERROR
 
-int
+static int
 ngx_stream_ssl_certificate(ngx_ssl_conn_t *ssl_conn, void *arg)
 {
     ngx_str_t                    cert, key;

From 1fc61b7b1ff182e86078200a59d3c523419c7b3b Mon Sep 17 00:00:00 2001
From: Maxim Dounin <mdounin@mdounin.ru>
Date: Thu, 21 Oct 2021 18:44:07 +0300
Subject: [PATCH 15/27] SSL: SSL_sendfile() support with kernel TLS.

Requires OpenSSL 3.0 compiled with "enable-ktls" option.  Further, KTLS
needs to be enabled in kernel, and in OpenSSL, either via OpenSSL
configuration file or with "ssl_conf_command Options KTLS;" in nginx
configuration.

On FreeBSD, kernel TLS is available starting with FreeBSD 13.0, and
can be enabled with "sysctl kern.ipc.tls.enable=1" and "kldload ktls_ocf"
to load a software backend, see man ktls(4) for details.

On Linux, kernel TLS is available starting with kernel 4.13 (at least 5.2
is recommended), and needs kernel compiled with CONFIG_TLS=y (with
CONFIG_TLS=m, which is used at least on Ubuntu 21.04 by default,
the tls module needs to be loaded with "modprobe tls").
---
 src/event/ngx_event_openssl.c | 209 +++++++++++++++++++++++++++++++++-
 src/event/ngx_event_openssl.h |   1 +
 src/http/ngx_http_request.c   |   2 +-
 src/http/ngx_http_upstream.c  |   8 +-
 4 files changed, 211 insertions(+), 9 deletions(-)

diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c
index d4ba1f7d5..84afecd0b 100644
--- a/src/event/ngx_event_openssl.c
+++ b/src/event/ngx_event_openssl.c
@@ -47,6 +47,8 @@ static void ngx_ssl_write_handler(ngx_event_t *wev);
 static ssize_t ngx_ssl_write_early(ngx_connection_t *c, u_char *data,
     size_t size);
 #endif
+static ssize_t ngx_ssl_sendfile(ngx_connection_t *c, ngx_buf_t *file,
+    size_t size);
 static void ngx_ssl_read_handler(ngx_event_t *rev);
 static void ngx_ssl_shutdown_handler(ngx_event_t *ev);
 static void ngx_ssl_connection_error(ngx_connection_t *c, int sslerr,
@@ -1762,6 +1764,16 @@ ngx_ssl_handshake(ngx_connection_t *c)
 
 #endif
 #endif
+#endif
+
+#ifdef BIO_get_ktls_send
+
+        if (BIO_get_ktls_send(SSL_get_wbio(c->ssl->connection)) == 1) {
+            ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                           "BIO_get_ktls_send(): 1");
+            c->ssl->sendfile = 1;
+        }
+
 #endif
 
         rc = ngx_ssl_ocsp_validate(c);
@@ -1899,6 +1911,16 @@ ngx_ssl_try_early_data(ngx_connection_t *c)
         c->read->ready = 1;
         c->write->ready = 1;
 
+#ifdef BIO_get_ktls_send
+
+        if (BIO_get_ktls_send(SSL_get_wbio(c->ssl->connection)) == 1) {
+            ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                           "BIO_get_ktls_send(): 1");
+            c->ssl->sendfile = 1;
+        }
+
+#endif
+
         rc = ngx_ssl_ocsp_validate(c);
 
         if (rc == NGX_ERROR) {
@@ -2502,10 +2524,11 @@ ngx_ssl_write_handler(ngx_event_t *wev)
 ngx_chain_t *
 ngx_ssl_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
 {
-    int          n;
-    ngx_uint_t   flush;
-    ssize_t      send, size;
-    ngx_buf_t   *buf;
+    int           n;
+    ngx_uint_t    flush;
+    ssize_t       send, size, file_size;
+    ngx_buf_t    *buf;
+    ngx_chain_t  *cl;
 
     if (!c->ssl->buffer) {
 
@@ -2579,6 +2602,11 @@ ngx_ssl_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
                 continue;
             }
 
+            if (in->buf->in_file && c->ssl->sendfile) {
+                flush = 1;
+                break;
+            }
+
             size = in->buf->last - in->buf->pos;
 
             if (size > buf->end - buf->last) {
@@ -2610,8 +2638,35 @@ ngx_ssl_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
         size = buf->last - buf->pos;
 
         if (size == 0) {
+
+            if (in && in->buf->in_file && send < limit) {
+
+                /* coalesce the neighbouring file bufs */
+
+                cl = in;
+                file_size = (size_t) ngx_chain_coalesce_file(&cl, limit - send);
+
+                n = ngx_ssl_sendfile(c, in->buf, file_size);
+
+                if (n == NGX_ERROR) {
+                    return NGX_CHAIN_ERROR;
+                }
+
+                if (n == NGX_AGAIN) {
+                    break;
+                }
+
+                in = ngx_chain_update_sent(in, n);
+
+                send += n;
+                flush = 0;
+
+                continue;
+            }
+
             buf->flush = 0;
             c->buffered &= ~NGX_SSL_BUFFERED;
+
             return in;
         }
 
@@ -2636,7 +2691,7 @@ ngx_ssl_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
         buf->pos = buf->start;
         buf->last = buf->start;
 
-        if (in == NULL || send == limit) {
+        if (in == NULL || send >= limit) {
             break;
         }
     }
@@ -2882,6 +2937,150 @@ ngx_ssl_write_early(ngx_connection_t *c, u_char *data, size_t size)
 #endif
 
 
+static ssize_t
+ngx_ssl_sendfile(ngx_connection_t *c, ngx_buf_t *file, size_t size)
+{
+#ifdef BIO_get_ktls_send
+
+    int        sslerr;
+    ssize_t    n;
+    ngx_err_t  err;
+
+    ngx_ssl_clear_error(c->log);
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "SSL to sendfile: @%O %uz",
+                   file->file_pos, size);
+
+    ngx_set_errno(0);
+
+    n = SSL_sendfile(c->ssl->connection, file->file->fd, file->file_pos,
+                     size, 0);
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL_sendfile: %d", n);
+
+    if (n > 0) {
+
+        if (c->ssl->saved_read_handler) {
+
+            c->read->handler = c->ssl->saved_read_handler;
+            c->ssl->saved_read_handler = NULL;
+            c->read->ready = 1;
+
+            if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
+                return NGX_ERROR;
+            }
+
+            ngx_post_event(c->read, &ngx_posted_events);
+        }
+
+        c->sent += n;
+
+        return n;
+    }
+
+    if (n == 0) {
+
+        /*
+         * if sendfile returns zero, then someone has truncated the file,
+         * so the offset became beyond the end of the file
+         */
+
+        ngx_log_error(NGX_LOG_ALERT, c->log, 0,
+                      "SSL_sendfile() reported that \"%s\" was truncated at %O",
+                      file->file->name.data, file->file_pos);
+
+        return NGX_ERROR;
+    }
+
+    sslerr = SSL_get_error(c->ssl->connection, n);
+
+    if (sslerr == SSL_ERROR_ZERO_RETURN) {
+
+        /*
+         * OpenSSL fails to return SSL_ERROR_SYSCALL if an error
+         * happens during writing after close_notify alert from the
+         * peer, and returns SSL_ERROR_ZERO_RETURN instead
+         */
+
+        sslerr = SSL_ERROR_SYSCALL;
+    }
+
+    if (sslerr == SSL_ERROR_SSL
+        && ERR_GET_REASON(ERR_peek_error()) == SSL_R_UNINITIALIZED
+        && ngx_errno != 0)
+    {
+        /*
+         * OpenSSL fails to return SSL_ERROR_SYSCALL if an error
+         * happens in sendfile(), and returns SSL_ERROR_SSL with
+         * SSL_R_UNINITIALIZED reason instead
+         */
+
+        sslerr = SSL_ERROR_SYSCALL;
+    }
+
+    err = (sslerr == SSL_ERROR_SYSCALL) ? ngx_errno : 0;
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL_get_error: %d", sslerr);
+
+    if (sslerr == SSL_ERROR_WANT_WRITE) {
+
+        if (c->ssl->saved_read_handler) {
+
+            c->read->handler = c->ssl->saved_read_handler;
+            c->ssl->saved_read_handler = NULL;
+            c->read->ready = 1;
+
+            if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
+                return NGX_ERROR;
+            }
+
+            ngx_post_event(c->read, &ngx_posted_events);
+        }
+
+        c->write->ready = 0;
+        return NGX_AGAIN;
+    }
+
+    if (sslerr == SSL_ERROR_WANT_READ) {
+
+        ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                       "SSL_sendfile: want read");
+
+        c->read->ready = 0;
+
+        if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
+            return NGX_ERROR;
+        }
+
+        /*
+         * we do not set the timer because there is already
+         * the write event timer
+         */
+
+        if (c->ssl->saved_read_handler == NULL) {
+            c->ssl->saved_read_handler = c->read->handler;
+            c->read->handler = ngx_ssl_read_handler;
+        }
+
+        return NGX_AGAIN;
+    }
+
+    c->ssl->no_wait_shutdown = 1;
+    c->ssl->no_send_shutdown = 1;
+    c->write->error = 1;
+
+    ngx_ssl_connection_error(c, sslerr, err, "SSL_sendfile() failed");
+
+#else
+    ngx_log_error(NGX_LOG_ALERT, c->log, 0,
+                  "SSL_sendfile() not available");
+#endif
+
+    return NGX_ERROR;
+}
+
+
 static void
 ngx_ssl_read_handler(ngx_event_t *rev)
 {
diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h
index e7cb06e70..4afdfad48 100644
--- a/src/event/ngx_event_openssl.h
+++ b/src/event/ngx_event_openssl.h
@@ -109,6 +109,7 @@ struct ngx_ssl_connection_s {
     unsigned                    handshake_rejected:1;
     unsigned                    renegotiation:1;
     unsigned                    buffer:1;
+    unsigned                    sendfile:1;
     unsigned                    no_wait_shutdown:1;
     unsigned                    no_send_shutdown:1;
     unsigned                    shutdown_without_free:1;
diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c
index 76bbcd67c..013b7158e 100644
--- a/src/http/ngx_http_request.c
+++ b/src/http/ngx_http_request.c
@@ -607,7 +607,7 @@ ngx_http_alloc_request(ngx_connection_t *c)
     }
 
 #if (NGX_HTTP_SSL)
-    if (c->ssl) {
+    if (c->ssl && !c->ssl->sendfile) {
         r->main_filter_need_in_memory = 1;
     }
 #endif
diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c
index 01c7877e6..daa8d2201 100644
--- a/src/http/ngx_http_upstream.c
+++ b/src/http/ngx_http_upstream.c
@@ -1683,9 +1683,6 @@ ngx_http_upstream_ssl_init_connection(ngx_http_request_t *r,
         return;
     }
 
-    c->sendfile = 0;
-    u->output.sendfile = 0;
-
     if (u->conf->ssl_server_name || u->conf->ssl_verify) {
         if (ngx_http_upstream_ssl_name(r, u, c) != NGX_OK) {
             ngx_http_upstream_finalize_request(r, u,
@@ -1791,6 +1788,11 @@ ngx_http_upstream_ssl_handshake(ngx_http_request_t *r, ngx_http_upstream_t *u,
             }
         }
 
+        if (!c->ssl->sendfile) {
+            c->sendfile = 0;
+            u->output.sendfile = 0;
+        }
+
         c->write->handler = ngx_http_upstream_handler;
         c->read->handler = ngx_http_upstream_handler;
 

From 67d2a9541826ecd5db97d604f23460210fd3e517 Mon Sep 17 00:00:00 2001
From: Maxim Dounin <mdounin@mdounin.ru>
Date: Mon, 25 Oct 2021 20:49:15 +0300
Subject: [PATCH 16/27] MIME: added image/avif type.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Prodded by Ryo Hirafuji, André Rømcke, Artur Juraszek.
---
 conf/mime.types | 1 +
 1 file changed, 1 insertion(+)

diff --git a/conf/mime.types b/conf/mime.types
index b53f7f7e0..1c00d701a 100644
--- a/conf/mime.types
+++ b/conf/mime.types
@@ -15,6 +15,7 @@ types {
     text/vnd.wap.wml                                 wml;
     text/x-component                                 htc;
 
+    image/avif                                       avif;
     image/png                                        png;
     image/svg+xml                                    svg svgz;
     image/tiff                                       tif tiff;

From 3253b346fb8b067d68a79ae72e08a376f234b0b3 Mon Sep 17 00:00:00 2001
From: Alexey Radkov <alexey.radkov@gmail.com>
Date: Thu, 19 Aug 2021 20:51:27 +0300
Subject: [PATCH 17/27] Core: removed unnecessary restriction in hash
 initialization.

Hash initialization ignores elements with key.data set to NULL.
Nevertheless, the initial hash bucket size check didn't skip them,
resulting in unnecessary restrictions on, for example, variables with
long names and with the NGX_HTTP_VARIABLE_NOHASH flag.

Fix is to update the initial hash bucket size check to skip elements
with key.data set to NULL, similarly to how it is done in other parts
of the code.
---
 src/core/ngx_hash.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/core/ngx_hash.c b/src/core/ngx_hash.c
index d9c157c1d..8215c2717 100644
--- a/src/core/ngx_hash.c
+++ b/src/core/ngx_hash.c
@@ -274,6 +274,10 @@ ngx_hash_init(ngx_hash_init_t *hinit, ngx_hash_key_t *names, ngx_uint_t nelts)
     }
 
     for (n = 0; n < nelts; n++) {
+        if (names[n].key.data == NULL) {
+            continue;
+        }
+
         if (hinit->bucket_size < NGX_HASH_ELT_SIZE(&names[n]) + sizeof(void *))
         {
             ngx_log_error(NGX_LOG_EMERG, hinit->pool->log, 0,

From 27cdfe30f7505361c31e4d3b2d0d21adaae803c4 Mon Sep 17 00:00:00 2001
From: Roman Arutyunyan <arut@nginx.com>
Date: Thu, 28 Oct 2021 13:11:31 +0300
Subject: [PATCH 18/27] Mp4: added ngx_http_mp4_update_mdhd_atom() function.

The function updates the duration field of mdhd atom.  Previously it was
updated in ngx_http_mp4_read_mdhd_atom().  The change makes it possible to
alter track duration as a result of processing track frames.
---
 src/http/modules/ngx_http_mp4_module.c | 40 ++++++++++++++++++++------
 1 file changed, 32 insertions(+), 8 deletions(-)

diff --git a/src/http/modules/ngx_http_mp4_module.c b/src/http/modules/ngx_http_mp4_module.c
index 0e93fbd09..fab8e2ca2 100644
--- a/src/http/modules/ngx_http_mp4_module.c
+++ b/src/http/modules/ngx_http_mp4_module.c
@@ -70,6 +70,7 @@ typedef struct {
     ngx_uint_t            end_chunk_samples;
     uint64_t              start_chunk_samples_size;
     uint64_t              end_chunk_samples_size;
+    uint64_t              duration;
     off_t                 start_offset;
     off_t                 end_offset;
 
@@ -253,6 +254,8 @@ static void ngx_http_mp4_update_mdia_atom(ngx_http_mp4_file_t *mp4,
     ngx_http_mp4_trak_t *trak);
 static ngx_int_t ngx_http_mp4_read_mdhd_atom(ngx_http_mp4_file_t *mp4,
     uint64_t atom_data_size);
+static void ngx_http_mp4_update_mdhd_atom(ngx_http_mp4_file_t *mp4,
+    ngx_http_mp4_trak_t *trak);
 static ngx_int_t ngx_http_mp4_read_hdlr_atom(ngx_http_mp4_file_t *mp4,
     uint64_t atom_data_size);
 static ngx_int_t ngx_http_mp4_read_minf_atom(ngx_http_mp4_file_t *mp4,
@@ -822,7 +825,7 @@ ngx_http_mp4_process(ngx_http_mp4_file_t *mp4)
 
         ngx_http_mp4_update_stbl_atom(mp4, &trak[i]);
         ngx_http_mp4_update_minf_atom(mp4, &trak[i]);
-        trak[i].size += trak[i].mdhd_size;
+        ngx_http_mp4_update_mdhd_atom(mp4, &trak[i]);
         trak[i].size += trak[i].hdlr_size;
         ngx_http_mp4_update_mdia_atom(mp4, &trak[i]);
         trak[i].size += trak[i].tkhd_size;
@@ -1749,16 +1752,10 @@ ngx_http_mp4_read_mdhd_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size)
     trak = ngx_mp4_last_trak(mp4);
     trak->mdhd_size = atom_size;
     trak->timescale = timescale;
+    trak->duration = duration;
 
     ngx_mp4_set_32value(mdhd_atom->size, atom_size);
 
-    if (mdhd_atom->version[0] == 0) {
-        ngx_mp4_set_32value(mdhd_atom->duration, duration);
-
-    } else {
-        ngx_mp4_set_64value(mdhd64_atom->duration, duration);
-    }
-
     atom = &trak->mdhd_atom_buf;
     atom->temporary = 1;
     atom->pos = atom_header;
@@ -1772,6 +1769,33 @@ ngx_http_mp4_read_mdhd_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size)
 }
 
 
+static void
+ngx_http_mp4_update_mdhd_atom(ngx_http_mp4_file_t *mp4,
+            ngx_http_mp4_trak_t *trak)
+{
+    ngx_buf_t              *atom;
+    ngx_mp4_mdhd_atom_t    *mdhd_atom;
+    ngx_mp4_mdhd64_atom_t  *mdhd64_atom;
+
+    atom = trak->out[NGX_HTTP_MP4_MDHD_ATOM].buf;
+    if (atom == NULL) {
+        return;
+    }
+
+    mdhd_atom = (ngx_mp4_mdhd_atom_t *) atom->pos;
+    mdhd64_atom = (ngx_mp4_mdhd64_atom_t *) atom->pos;
+
+    if (mdhd_atom->version[0] == 0) {
+        ngx_mp4_set_32value(mdhd_atom->duration, trak->duration);
+
+    } else {
+        ngx_mp4_set_64value(mdhd64_atom->duration, trak->duration);
+    }
+
+    trak->size += trak->mdhd_size;
+}
+
+
 static ngx_int_t
 ngx_http_mp4_read_hdlr_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size)
 {

From 7927071ee26ff6313301b744a90240dccbc836db Mon Sep 17 00:00:00 2001
From: Roman Arutyunyan <arut@nginx.com>
Date: Thu, 28 Oct 2021 14:14:25 +0300
Subject: [PATCH 19/27] Mp4: mp4_start_key_frame directive.

The directive enables including all frames from start time to the most recent
key frame in the result.  Those frames are removed from presentation timeline
using mp4 edit lists.

Edit lists are currently supported by popular players and browsers such as
Chrome, Safari, QuickTime and ffmpeg.  Among those not supporting them properly
is Firefox[1].

Based on a patch by Tracey Jaquith, Internet Archive.

[1] https://bugzilla.mozilla.org/show_bug.cgi?id=1735300
---
 src/http/modules/ngx_http_mp4_module.c | 221 ++++++++++++++++++++++---
 1 file changed, 194 insertions(+), 27 deletions(-)

diff --git a/src/http/modules/ngx_http_mp4_module.c b/src/http/modules/ngx_http_mp4_module.c
index fab8e2ca2..9c3f627fe 100644
--- a/src/http/modules/ngx_http_mp4_module.c
+++ b/src/http/modules/ngx_http_mp4_module.c
@@ -11,31 +11,33 @@
 
 #define NGX_HTTP_MP4_TRAK_ATOM     0
 #define NGX_HTTP_MP4_TKHD_ATOM     1
-#define NGX_HTTP_MP4_MDIA_ATOM     2
-#define NGX_HTTP_MP4_MDHD_ATOM     3
-#define NGX_HTTP_MP4_HDLR_ATOM     4
-#define NGX_HTTP_MP4_MINF_ATOM     5
-#define NGX_HTTP_MP4_VMHD_ATOM     6
-#define NGX_HTTP_MP4_SMHD_ATOM     7
-#define NGX_HTTP_MP4_DINF_ATOM     8
-#define NGX_HTTP_MP4_STBL_ATOM     9
-#define NGX_HTTP_MP4_STSD_ATOM    10
-#define NGX_HTTP_MP4_STTS_ATOM    11
-#define NGX_HTTP_MP4_STTS_DATA    12
-#define NGX_HTTP_MP4_STSS_ATOM    13
-#define NGX_HTTP_MP4_STSS_DATA    14
-#define NGX_HTTP_MP4_CTTS_ATOM    15
-#define NGX_HTTP_MP4_CTTS_DATA    16
-#define NGX_HTTP_MP4_STSC_ATOM    17
-#define NGX_HTTP_MP4_STSC_START   18
-#define NGX_HTTP_MP4_STSC_DATA    19
-#define NGX_HTTP_MP4_STSC_END     20
-#define NGX_HTTP_MP4_STSZ_ATOM    21
-#define NGX_HTTP_MP4_STSZ_DATA    22
-#define NGX_HTTP_MP4_STCO_ATOM    23
-#define NGX_HTTP_MP4_STCO_DATA    24
-#define NGX_HTTP_MP4_CO64_ATOM    25
-#define NGX_HTTP_MP4_CO64_DATA    26
+#define NGX_HTTP_MP4_EDTS_ATOM     2
+#define NGX_HTTP_MP4_ELST_ATOM     3
+#define NGX_HTTP_MP4_MDIA_ATOM     4
+#define NGX_HTTP_MP4_MDHD_ATOM     5
+#define NGX_HTTP_MP4_HDLR_ATOM     6
+#define NGX_HTTP_MP4_MINF_ATOM     7
+#define NGX_HTTP_MP4_VMHD_ATOM     8
+#define NGX_HTTP_MP4_SMHD_ATOM     9
+#define NGX_HTTP_MP4_DINF_ATOM    10
+#define NGX_HTTP_MP4_STBL_ATOM    11
+#define NGX_HTTP_MP4_STSD_ATOM    12
+#define NGX_HTTP_MP4_STTS_ATOM    13
+#define NGX_HTTP_MP4_STTS_DATA    14
+#define NGX_HTTP_MP4_STSS_ATOM    15
+#define NGX_HTTP_MP4_STSS_DATA    16
+#define NGX_HTTP_MP4_CTTS_ATOM    17
+#define NGX_HTTP_MP4_CTTS_DATA    18
+#define NGX_HTTP_MP4_STSC_ATOM    19
+#define NGX_HTTP_MP4_STSC_START   20
+#define NGX_HTTP_MP4_STSC_DATA    21
+#define NGX_HTTP_MP4_STSC_END     22
+#define NGX_HTTP_MP4_STSZ_ATOM    23
+#define NGX_HTTP_MP4_STSZ_DATA    24
+#define NGX_HTTP_MP4_STCO_ATOM    25
+#define NGX_HTTP_MP4_STCO_DATA    26
+#define NGX_HTTP_MP4_CO64_ATOM    27
+#define NGX_HTTP_MP4_CO64_DATA    28
 
 #define NGX_HTTP_MP4_LAST_ATOM    NGX_HTTP_MP4_CO64_DATA
 
@@ -43,6 +45,7 @@
 typedef struct {
     size_t                buffer_size;
     size_t                max_buffer_size;
+    ngx_flag_t            start_key_frame;
 } ngx_http_mp4_conf_t;
 
 
@@ -53,6 +56,25 @@ typedef struct {
 } ngx_mp4_stsc_entry_t;
 
 
+typedef struct {
+    u_char                size[4];
+    u_char                name[4];
+} ngx_mp4_edts_atom_t;
+
+
+typedef struct {
+    u_char                size[4];
+    u_char                name[4];
+    u_char                version[1];
+    u_char                flags[3];
+    u_char                entries[4];
+    u_char                duration[8];
+    u_char                media_time[8];
+    u_char                media_rate[2];
+    u_char                reserved[2];
+} ngx_mp4_elst_atom_t;
+
+
 typedef struct {
     uint32_t              timescale;
     uint32_t              time_to_sample_entries;
@@ -71,6 +93,8 @@ typedef struct {
     uint64_t              start_chunk_samples_size;
     uint64_t              end_chunk_samples_size;
     uint64_t              duration;
+    uint64_t              prefix;
+    uint64_t              movie_duration;
     off_t                 start_offset;
     off_t                 end_offset;
 
@@ -86,6 +110,8 @@ typedef struct {
 
     ngx_buf_t             trak_atom_buf;
     ngx_buf_t             tkhd_atom_buf;
+    ngx_buf_t             edts_atom_buf;
+    ngx_buf_t             elst_atom_buf;
     ngx_buf_t             mdia_atom_buf;
     ngx_buf_t             mdhd_atom_buf;
     ngx_buf_t             hdlr_atom_buf;
@@ -112,6 +138,8 @@ typedef struct {
     ngx_buf_t             co64_atom_buf;
     ngx_buf_t             co64_data_buf;
 
+    ngx_mp4_edts_atom_t   edts_atom;
+    ngx_mp4_elst_atom_t   elst_atom;
     ngx_mp4_stsc_entry_t  stsc_start_chunk_entry;
     ngx_mp4_stsc_entry_t  stsc_end_chunk_entry;
 } ngx_http_mp4_trak_t;
@@ -187,6 +215,14 @@ typedef struct {
     ((u_char *) (p))[6] = n3;                                                 \
     ((u_char *) (p))[7] = n4
 
+#define ngx_mp4_get_16value(p)                                                \
+    ( ((uint16_t) ((u_char *) (p))[0] << 8)                                   \
+    + (           ((u_char *) (p))[1]) )
+
+#define ngx_mp4_set_16value(p, n)                                             \
+    ((u_char *) (p))[0] = (u_char) ((n) >> 8);                                \
+    ((u_char *) (p))[1] = (u_char)  (n)
+
 #define ngx_mp4_get_32value(p)                                                \
     ( ((uint32_t) ((u_char *) (p))[0] << 24)                                  \
     + (           ((u_char *) (p))[1] << 16)                                  \
@@ -270,6 +306,8 @@ static ngx_int_t ngx_http_mp4_read_smhd_atom(ngx_http_mp4_file_t *mp4,
     uint64_t atom_data_size);
 static ngx_int_t ngx_http_mp4_read_stbl_atom(ngx_http_mp4_file_t *mp4,
     uint64_t atom_data_size);
+static void ngx_http_mp4_update_edts_atom(ngx_http_mp4_file_t *mp4,
+    ngx_http_mp4_trak_t *trak);
 static void ngx_http_mp4_update_stbl_atom(ngx_http_mp4_file_t *mp4,
     ngx_http_mp4_trak_t *trak);
 static ngx_int_t ngx_http_mp4_read_stsd_atom(ngx_http_mp4_file_t *mp4,
@@ -280,6 +318,8 @@ static ngx_int_t ngx_http_mp4_update_stts_atom(ngx_http_mp4_file_t *mp4,
     ngx_http_mp4_trak_t *trak);
 static ngx_int_t ngx_http_mp4_crop_stts_data(ngx_http_mp4_file_t *mp4,
     ngx_http_mp4_trak_t *trak, ngx_uint_t start);
+static uint32_t ngx_http_mp4_seek_key_frame(ngx_http_mp4_file_t *mp4,
+    ngx_http_mp4_trak_t *trak, uint32_t start_sample);
 static ngx_int_t ngx_http_mp4_read_stss_atom(ngx_http_mp4_file_t *mp4,
     uint64_t atom_data_size);
 static ngx_int_t ngx_http_mp4_update_stss_atom(ngx_http_mp4_file_t *mp4,
@@ -343,6 +383,13 @@ static ngx_command_t  ngx_http_mp4_commands[] = {
       offsetof(ngx_http_mp4_conf_t, max_buffer_size),
       NULL },
 
+    { ngx_string("mp4_start_key_frame"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG,
+      ngx_conf_set_flag_slot,
+      NGX_HTTP_LOC_CONF_OFFSET,
+      offsetof(ngx_http_mp4_conf_t, start_key_frame),
+      NULL },
+
       ngx_null_command
 };
 
@@ -829,6 +876,7 @@ ngx_http_mp4_process(ngx_http_mp4_file_t *mp4)
         trak[i].size += trak[i].hdlr_size;
         ngx_http_mp4_update_mdia_atom(mp4, &trak[i]);
         trak[i].size += trak[i].tkhd_size;
+        ngx_http_mp4_update_edts_atom(mp4, &trak[i]);
         ngx_http_mp4_update_trak_atom(mp4, &trak[i]);
 
         mp4->moov_size += trak[i].size;
@@ -1590,6 +1638,7 @@ ngx_http_mp4_read_tkhd_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size)
 
     trak = ngx_mp4_last_trak(mp4);
     trak->tkhd_size = atom_size;
+    trak->movie_duration = duration;
 
     ngx_mp4_set_32value(tkhd_atom->size, atom_size);
 
@@ -1985,6 +2034,59 @@ ngx_http_mp4_read_stbl_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size)
 }
 
 
+static void
+ngx_http_mp4_update_edts_atom(ngx_http_mp4_file_t *mp4,
+    ngx_http_mp4_trak_t *trak)
+{
+    ngx_buf_t            *atom;
+    ngx_mp4_elst_atom_t  *elst_atom;
+    ngx_mp4_edts_atom_t  *edts_atom;
+
+    if (trak->prefix == 0) {
+        return;
+    }
+
+    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0,
+                   "mp4 edts atom update prefix:%uL", trak->prefix);
+
+    edts_atom = &trak->edts_atom;
+    ngx_mp4_set_32value(edts_atom->size, sizeof(ngx_mp4_edts_atom_t)
+                                         + sizeof(ngx_mp4_elst_atom_t));
+    ngx_mp4_set_atom_name(edts_atom, 'e', 'd', 't', 's');
+
+    atom = &trak->edts_atom_buf;
+    atom->temporary = 1;
+    atom->pos = (u_char *) edts_atom;
+    atom->last = (u_char *) edts_atom + sizeof(ngx_mp4_edts_atom_t);
+
+    trak->out[NGX_HTTP_MP4_EDTS_ATOM].buf = atom;
+
+    elst_atom = &trak->elst_atom;
+    ngx_mp4_set_32value(elst_atom->size, sizeof(ngx_mp4_elst_atom_t));
+    ngx_mp4_set_atom_name(elst_atom, 'e', 'l', 's', 't');
+
+    elst_atom->version[0] = 1;
+    elst_atom->flags[0] = 0;
+    elst_atom->flags[1] = 0;
+    elst_atom->flags[2] = 0;
+
+    ngx_mp4_set_32value(elst_atom->entries, 1);
+    ngx_mp4_set_64value(elst_atom->duration, trak->movie_duration);
+    ngx_mp4_set_64value(elst_atom->media_time, trak->prefix);
+    ngx_mp4_set_16value(elst_atom->media_rate, 1);
+    ngx_mp4_set_16value(elst_atom->reserved, 0);
+
+    atom = &trak->elst_atom_buf;
+    atom->temporary = 1;
+    atom->pos = (u_char *) elst_atom;
+    atom->last = (u_char *) elst_atom + sizeof(ngx_mp4_elst_atom_t);
+
+    trak->out[NGX_HTTP_MP4_ELST_ATOM].buf = atom;
+
+    trak->size += sizeof(ngx_mp4_edts_atom_t) + sizeof(ngx_mp4_elst_atom_t);
+}
+
+
 static void
 ngx_http_mp4_update_stbl_atom(ngx_http_mp4_file_t *mp4,
     ngx_http_mp4_trak_t *trak)
@@ -2183,7 +2285,7 @@ static ngx_int_t
 ngx_http_mp4_crop_stts_data(ngx_http_mp4_file_t *mp4,
     ngx_http_mp4_trak_t *trak, ngx_uint_t start)
 {
-    uint32_t               count, duration, rest;
+    uint32_t               count, duration, rest, key_prefix;
     uint64_t               start_time;
     ngx_buf_t             *data;
     ngx_uint_t             start_sample, entries, start_sec;
@@ -2207,7 +2309,7 @@ ngx_http_mp4_crop_stts_data(ngx_http_mp4_file_t *mp4,
 
     data = trak->out[NGX_HTTP_MP4_STTS_DATA].buf;
 
-    start_time = (uint64_t) start_sec * trak->timescale / 1000;
+    start_time = (uint64_t) start_sec * trak->timescale / 1000 + trak->prefix;
 
     entries = trak->time_to_sample_entries;
     start_sample = 0;
@@ -2253,6 +2355,26 @@ ngx_http_mp4_crop_stts_data(ngx_http_mp4_file_t *mp4,
 found:
 
     if (start) {
+        key_prefix = ngx_http_mp4_seek_key_frame(mp4, trak, start_sample);
+
+        start_sample -= key_prefix;
+
+        while (rest < key_prefix) {
+            trak->prefix += rest * duration;
+            key_prefix -= rest;
+
+            entry--;
+            entries++;
+
+            count = ngx_mp4_get_32value(entry->count);
+            duration = ngx_mp4_get_32value(entry->duration);
+            rest = count;
+        }
+
+        trak->prefix += key_prefix * duration;
+        trak->duration += trak->prefix;
+        rest -= key_prefix;
+
         ngx_mp4_set_32value(entry->count, count - rest);
         data->pos = (u_char *) entry;
         trak->time_to_sample_entries = entries;
@@ -2277,6 +2399,49 @@ found:
 }
 
 
+static uint32_t
+ngx_http_mp4_seek_key_frame(ngx_http_mp4_file_t *mp4, ngx_http_mp4_trak_t *trak,
+    uint32_t start_sample)
+{
+    uint32_t              key_prefix, sample, *entry, *end;
+    ngx_buf_t            *data;
+    ngx_http_mp4_conf_t  *conf;
+
+    conf = ngx_http_get_module_loc_conf(mp4->request, ngx_http_mp4_module);
+    if (!conf->start_key_frame) {
+        return 0;
+    }
+
+    data = trak->out[NGX_HTTP_MP4_STSS_DATA].buf;
+    if (data == NULL) {
+        return 0;
+    }
+
+    entry = (uint32_t *) data->pos;
+    end = (uint32_t *) data->last;
+
+    /* sync samples starts from 1 */
+    start_sample++;
+
+    key_prefix = 0;
+
+    while (entry < end) {
+        sample = ngx_mp4_get_32value(entry);
+        if (sample > start_sample) {
+            break;
+        }
+
+        key_prefix = start_sample - sample;
+        entry++;
+    }
+
+    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0,
+                   "mp4 key frame prefix:%uD", key_prefix);
+
+    return key_prefix;
+}
+
+
 typedef struct {
     u_char    size[4];
     u_char    name[4];
@@ -3614,6 +3779,7 @@ ngx_http_mp4_create_conf(ngx_conf_t *cf)
 
     conf->buffer_size = NGX_CONF_UNSET_SIZE;
     conf->max_buffer_size = NGX_CONF_UNSET_SIZE;
+    conf->start_key_frame = NGX_CONF_UNSET;
 
     return conf;
 }
@@ -3628,6 +3794,7 @@ ngx_http_mp4_merge_conf(ngx_conf_t *cf, void *parent, void *child)
     ngx_conf_merge_size_value(conf->buffer_size, prev->buffer_size, 512 * 1024);
     ngx_conf_merge_size_value(conf->max_buffer_size, prev->max_buffer_size,
                               10 * 1024 * 1024);
+    ngx_conf_merge_value(conf->start_key_frame, prev->start_key_frame, 0);
 
     return NGX_CONF_OK;
 }

From f3fdc4c96f3cd1c338ecf11a67acfae6d0a08dc7 Mon Sep 17 00:00:00 2001
From: Maxim Dounin <mdounin@mdounin.ru>
Date: Fri, 29 Oct 2021 20:21:43 +0300
Subject: [PATCH 20/27] Switched to using posted next events after
 sendfile_max_chunk.

Previously, 1 millisecond delay was used instead.  In certain edge cases
this might result in noticeable performance degradation though, notably on
Linux with typical CONFIG_HZ=250 (so 1ms delay becomes 4ms),
sendfile_max_chunk 2m, and link speed above 2.5 Gbps.

Using posted next events removes the artificial delay and makes processing
fast in all cases.
---
 src/http/ngx_http_write_filter_module.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/http/ngx_http_write_filter_module.c b/src/http/ngx_http_write_filter_module.c
index 6a5d957b1..bd3770457 100644
--- a/src/http/ngx_http_write_filter_module.c
+++ b/src/http/ngx_http_write_filter_module.c
@@ -331,8 +331,7 @@ ngx_http_write_filter(ngx_http_request_t *r, ngx_chain_t *in)
         && c->write->ready
         && c->sent - sent >= limit - (off_t) (2 * ngx_pagesize))
     {
-        c->write->delayed = 1;
-        ngx_add_timer(c->write, 1);
+        ngx_post_event(c->write, &ngx_posted_next_events);
     }
 
     for (cl = r->out; cl && cl != chain; /* void */) {

From 00f4be1ce498df08e097925b08cfb1f84a8fda29 Mon Sep 17 00:00:00 2001
From: Maxim Dounin <mdounin@mdounin.ru>
Date: Fri, 29 Oct 2021 20:21:48 +0300
Subject: [PATCH 21/27] Simplified sendfile_max_chunk handling.

Previously, it was checked that sendfile_max_chunk was enabled and
almost whole sendfile_max_chunk was sent (see e67ef50c3176), to avoid
delaying connections where sendfile_max_chunk wasn't reached (for example,
when sending responses smaller than sendfile_max_chunk).  Now we instead
check if there are unsent data, and the connection is still ready for writing.
Additionally we also check c->write->delayed to ignore connections already
delayed by limit_rate.

This approach is believed to be more robust, and correctly handles
not only sendfile_max_chunk, but also internal limits of c->send_chain(),
such as sendfile() maximum supported length (ticket #1870).
---
 src/http/ngx_http_write_filter_module.c | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/src/http/ngx_http_write_filter_module.c b/src/http/ngx_http_write_filter_module.c
index bd3770457..932f26dd3 100644
--- a/src/http/ngx_http_write_filter_module.c
+++ b/src/http/ngx_http_write_filter_module.c
@@ -321,16 +321,12 @@ ngx_http_write_filter(ngx_http_request_t *r, ngx_chain_t *in)
         delay = (ngx_msec_t) ((nsent - sent) * 1000 / r->limit_rate);
 
         if (delay > 0) {
-            limit = 0;
             c->write->delayed = 1;
             ngx_add_timer(c->write, delay);
         }
     }
 
-    if (limit
-        && c->write->ready
-        && c->sent - sent >= limit - (off_t) (2 * ngx_pagesize))
-    {
+    if (chain && c->write->ready && !c->write->delayed) {
         ngx_post_event(c->write, &ngx_posted_next_events);
     }
 

From 6b0d773fbd95675b64ceaa34aefd60ae92b2a96b Mon Sep 17 00:00:00 2001
From: Maxim Dounin <mdounin@mdounin.ru>
Date: Fri, 29 Oct 2021 20:21:51 +0300
Subject: [PATCH 22/27] Fixed sendfile() limit handling on Linux.

On Linux starting with 2.6.16, sendfile() silently limits all operations
to MAX_RW_COUNT, defined as (INT_MAX & PAGE_MASK).  This incorrectly
triggered the interrupt check, and resulted in 0-sized writev() on the
next loop iteration.

Fix is to make sure the limit is always checked, so we will return from
the loop if the limit is already reached even if number of bytes sent is
not exactly equal to the number of bytes we've tried to send.
---
 src/os/unix/ngx_linux_sendfile_chain.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/os/unix/ngx_linux_sendfile_chain.c b/src/os/unix/ngx_linux_sendfile_chain.c
index 5695839b0..91e7f1d93 100644
--- a/src/os/unix/ngx_linux_sendfile_chain.c
+++ b/src/os/unix/ngx_linux_sendfile_chain.c
@@ -38,6 +38,9 @@ static void ngx_linux_sendfile_thread_handler(void *data, ngx_log_t *log);
  * On Linux up to 2.6.16 sendfile() does not allow to pass the count parameter
  * more than 2G-1 bytes even on 64-bit platforms: it returns EINVAL,
  * so we limit it to 2G-1 bytes.
+ *
+ * On Linux 2.6.16 and later, sendfile() silently limits the count parameter
+ * to 2G minus the page size, even on 64-bit platforms.
  */
 
 #define NGX_SENDFILE_MAXSIZE  2147483647L
@@ -216,7 +219,6 @@ ngx_linux_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
              */
 
             send = prev_send + sent;
-            continue;
         }
 
         if (send >= limit || in == NULL) {

From b3b368184b1e3c82da6703e8d7367f38fdc98d1a Mon Sep 17 00:00:00 2001
From: Maxim Dounin <mdounin@mdounin.ru>
Date: Fri, 29 Oct 2021 20:21:54 +0300
Subject: [PATCH 23/27] Upstream: sendfile_max_chunk support.

Previously, connections to upstream servers used sendfile() if it was
enabled, but never honored sendfile_max_chunk.  This might result
in worker monopolization for a long time if large request bodies
are allowed.
---
 src/core/ngx_output_chain.c  | 4 ++++
 src/http/ngx_http_upstream.c | 9 ++++++---
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/core/ngx_output_chain.c b/src/core/ngx_output_chain.c
index 5c3dbe872..fd4603b19 100644
--- a/src/core/ngx_output_chain.c
+++ b/src/core/ngx_output_chain.c
@@ -803,6 +803,10 @@ ngx_chain_writer(void *data, ngx_chain_t *in)
         return NGX_ERROR;
     }
 
+    if (chain && c->write->ready) {
+        ngx_post_event(c->write, &ngx_posted_next_events);
+    }
+
     for (cl = ctx->out; cl && cl != chain; /* void */) {
         ln = cl;
         cl = cl->next;
diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c
index daa8d2201..002b3e65d 100644
--- a/src/http/ngx_http_upstream.c
+++ b/src/http/ngx_http_upstream.c
@@ -1511,8 +1511,9 @@ ngx_http_upstream_check_broken_connection(ngx_http_request_t *r,
 static void
 ngx_http_upstream_connect(ngx_http_request_t *r, ngx_http_upstream_t *u)
 {
-    ngx_int_t          rc;
-    ngx_connection_t  *c;
+    ngx_int_t                  rc;
+    ngx_connection_t          *c;
+    ngx_http_core_loc_conf_t  *clcf;
 
     r->connection->log->action = "connecting to upstream";
 
@@ -1599,10 +1600,12 @@ ngx_http_upstream_connect(ngx_http_request_t *r, ngx_http_upstream_t *u)
 
     /* init or reinit the ngx_output_chain() and ngx_chain_writer() contexts */
 
+    clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
+
     u->writer.out = NULL;
     u->writer.last = &u->writer.out;
     u->writer.connection = c;
-    u->writer.limit = 0;
+    u->writer.limit = clcf->sendfile_max_chunk;
 
     if (u->request_sent) {
         if (ngx_http_upstream_reinit(r, u) != NGX_OK) {

From 5636e7f7b491a4db287e3aa5312cd5955075f5ec Mon Sep 17 00:00:00 2001
From: Maxim Dounin <mdounin@mdounin.ru>
Date: Fri, 29 Oct 2021 20:21:57 +0300
Subject: [PATCH 24/27] Changed default value of sendfile_max_chunk to 2m.

The "sendfile_max_chunk" directive is important to prevent worker
monopolization by fast connections.  The 2m value implies maximum 200ms
delay with 100 Mbps links, 20ms delay with 1 Gbps links, and 2ms on
10 Gbps links.  It also seems to be a good value for disks.
---
 src/http/ngx_http_core_module.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c
index bad43ea5c..fe1da4576 100644
--- a/src/http/ngx_http_core_module.c
+++ b/src/http/ngx_http_core_module.c
@@ -3720,7 +3720,7 @@ ngx_http_core_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child)
     ngx_conf_merge_value(conf->internal, prev->internal, 0);
     ngx_conf_merge_value(conf->sendfile, prev->sendfile, 0);
     ngx_conf_merge_size_value(conf->sendfile_max_chunk,
-                              prev->sendfile_max_chunk, 0);
+                              prev->sendfile_max_chunk, 2 * 1024 * 1024);
     ngx_conf_merge_size_value(conf->subrequest_output_buffer_size,
                               prev->subrequest_output_buffer_size,
                               (size_t) ngx_pagesize);

From 2c8dd1c33e19842fcf1e87b53cb86aeeea094dda Mon Sep 17 00:00:00 2001
From: Maxim Dounin <mdounin@mdounin.ru>
Date: Sat, 30 Oct 2021 02:39:19 +0300
Subject: [PATCH 25/27] Changed ngx_chain_update_chains() to test tag first
 (ticket #2248).

Without this change, aio used with HTTP/2 can result in connection hang,
as observed with "aio threads; aio_write on;" and proxying (ticket #2248).

The problem is that HTTP/2 updates buffers outside of the output filters
(notably, marks them as sent), and then posts a write event to call
output filters.  If a filter does not call the next one for some reason
(for example, because of an AIO operation in progress), this might
result in a state when the owner of a buffer already called
ngx_chain_update_chains() and can reuse the buffer, while the same buffer
is still sitting in the busy chain of some other filter.

In the particular case a buffer was sitting in output chain's ctx->busy,
and was reused by event pipe.  Output chain's ctx->busy was permanently
blocked by it, and this resulted in connection hang.

Fix is to change ngx_chain_update_chains() to skip buffers from other
modules unconditionally, without trying to wait for these buffers to
become empty.
---
 src/core/ngx_buf.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/core/ngx_buf.c b/src/core/ngx_buf.c
index c3783c446..811f24d96 100644
--- a/src/core/ngx_buf.c
+++ b/src/core/ngx_buf.c
@@ -203,16 +203,16 @@ ngx_chain_update_chains(ngx_pool_t *p, ngx_chain_t **free, ngx_chain_t **busy,
     while (*busy) {
         cl = *busy;
 
-        if (ngx_buf_size(cl->buf) != 0) {
-            break;
-        }
-
         if (cl->buf->tag != tag) {
             *busy = cl->next;
             ngx_free_chain(p, cl);
             continue;
         }
 
+        if (ngx_buf_size(cl->buf) != 0) {
+            break;
+        }
+
         cl->buf->pos = cl->buf->start;
         cl->buf->last = cl->buf->start;
 

From e0086aaeea3810b5485f53d2f38f74cc74effea7 Mon Sep 17 00:00:00 2001
From: Maxim Dounin <mdounin@mdounin.ru>
Date: Tue, 2 Nov 2021 17:49:22 +0300
Subject: [PATCH 26/27] nginx-1.21.4-RELEASE

---
 docs/xml/nginx/changes.xml | 131 +++++++++++++++++++++++++++++++++++++
 1 file changed, 131 insertions(+)

diff --git a/docs/xml/nginx/changes.xml b/docs/xml/nginx/changes.xml
index 65772d028..0808b5c6e 100644
--- a/docs/xml/nginx/changes.xml
+++ b/docs/xml/nginx/changes.xml
@@ -5,6 +5,137 @@
 <change_log title="nginx">
 
 
+<changes ver="1.21.4" date="2021-11-02">
+
+<change type="change">
+<para lang="ru">
+поддержка NPN вместо ALPN для установления HTTP/2-соединений
+упразднена.
+</para>
+<para lang="en">
+support for NPN instead of ALPN to establish HTTP/2 connections
+has been removed.
+</para>
+</change>
+
+<change type="change">
+<para lang="ru">
+теперь nginx закрывает SSL соединение, если клиент использует ALPN,
+но nginx не поддерживает ни один из присланных клиентом протоколов.
+</para>
+<para lang="en">
+now nginx rejects SSL connections if ALPN is used by the client,
+but no supported protocols can be negotiated.
+</para>
+</change>
+
+<change type="change">
+<para lang="ru">
+в директиве sendfile_max_chunk значение по умолчанию
+изменено на 2 мегабайта.
+</para>
+<para lang="en">
+the default value of the "sendfile_max_chunk" directive
+was changed to 2 megabytes.
+</para>
+</change>
+
+<change type="feature">
+<para lang="ru">
+директива proxy_half_close в модуле stream.
+</para>
+<para lang="en">
+the "proxy_half_close" directive in the stream module.
+</para>
+</change>
+
+<change type="feature">
+<para lang="ru">
+директива ssl_alpn в модуле stream.
+</para>
+<para lang="en">
+the "ssl_alpn" directive in the stream module.
+</para>
+</change>
+
+<change type="feature">
+<para lang="ru">
+переменная $ssl_alpn_protocol.
+</para>
+<para lang="en">
+the $ssl_alpn_protocol variable.
+</para>
+</change>
+
+<change type="feature">
+<para lang="ru">
+поддержка SSL_sendfile() при использовании OpenSSL 3.0.
+</para>
+<para lang="en">
+support for SSL_sendfile() when using OpenSSL 3.0.
+</para>
+</change>
+
+<change type="feature">
+<para lang="ru">
+директива mp4_start_key_frame в модуле ngx_http_mp4_module.<br/>
+Спасибо Tracey Jaquith.
+</para>
+<para lang="en">
+the "mp4_start_key_frame" directive in the ngx_http_mp4_module.<br/>
+Thanks to Tracey Jaquith.
+</para>
+</change>
+
+<change type="bugfix">
+<para lang="ru">
+в переменной $content_length при использовании chunked transfer encoding.
+</para>
+<para lang="en">
+in the $content_length variable when using chunked transfer encoding.
+</para>
+</change>
+
+<change type="bugfix">
+<para lang="ru">
+при получении ответа некорректной длины от проксируемого бэкенда
+nginx мог тем не менее закэшировать соединение.<br/>
+Спасибо Awdhesh Mathpal.
+</para>
+<para lang="en">
+after receiving a response with incorrect length from a proxied backend
+nginx might nevertheless cache the connection.<br/>
+Thanks to Awdhesh Mathpal.
+</para>
+</change>
+
+<change type="bugfix">
+<para lang="ru">
+некорректные заголовки от бэкендов
+логгировались на уровне info вместо error;
+ошибка появилась в 1.21.1.
+</para>
+<para lang="en">
+invalid headers from backends
+were logged at the "info" level instead of "error";
+the bug had appeared in 1.21.1.
+</para>
+</change>
+
+<change type="bugfix">
+<para lang="ru">
+при использовании HTTP/2 и директивы aio_write
+запросы могли зависать.
+</para>
+<para lang="en">
+requests might hang
+when using HTTP/2 and the "aio_write" directive.
+</para>
+</change>
+
+</changes>
+
+
 <changes ver="1.21.3" date="2021-09-07">
 
 <change type="change">

From 3334585539168947650a37d74dd32973ab451d70 Mon Sep 17 00:00:00 2001
From: Maxim Dounin <mdounin@mdounin.ru>
Date: Tue, 2 Nov 2021 17:49:22 +0300
Subject: [PATCH 27/27] release-1.21.4 tag

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index 34b2f18af..ef68adcf4 100644
--- a/.hgtags
+++ b/.hgtags
@@ -464,3 +464,4 @@ df34dcc9ac072ffd0945e5a1f3eb7987e8275375 release-1.21.0
 a68ac0677f8553b1f84d357bc9da114731ab5f47 release-1.21.1
 bfbc52374adcbf2f9060afd62de940f6fab3bba5 release-1.21.2
 2217a9c1d0b86026f22700b3c089545db1964f55 release-1.21.3
+39be8a682c58308d9399cddd57e37f9fdb7bdf3e release-1.21.4