QUIC: connection shutdown.

The function ngx_quic_shutdown_connection() waits until all non-cancelable
streams are closed, and then closes the connection.  In HTTP/3 cancelable
streams are all unidirectional streams except push streams.

The function is called from HTTP/3 when client reaches keepalive_requests.

src/event/quic/ngx_event_quic.c

@@ -174,9 +174,13 @@ typedef struct {
     ngx_uint_t                        error_ftype;
     const char                       *error_reason;
 
+    ngx_uint_t                        shutdown_code;
+    const char                       *shutdown_reason;
+
     unsigned                          error_app:1;
     unsigned                          send_timer_set:1;
     unsigned                          closing:1;
+    unsigned                          shutdown:1;
     unsigned                          draining:1;
     unsigned                          key_phase:1;
     unsigned                          validated:1;
@@ -384,6 +388,7 @@ static ngx_chain_t *ngx_quic_stream_send_chain(ngx_connection_t *c,
     ngx_chain_t *in, off_t limit);
 static size_t ngx_quic_max_stream_flow(ngx_connection_t *c);
 static void ngx_quic_stream_cleanup_handler(void *data);
+static void ngx_quic_shutdown_quic(ngx_connection_t *c);
 static ngx_quic_frame_t *ngx_quic_alloc_frame(ngx_connection_t *c);
 static void ngx_quic_free_frame(ngx_connection_t *c, ngx_quic_frame_t *frame);
 
@@ -684,6 +689,7 @@ ngx_quic_connstate_dbg(ngx_connection_t *c)
             }
         }
 
+        p = ngx_slprintf(p, last, "%s", qc->shutdown ? " shutdown" : "");
         p = ngx_slprintf(p, last, "%s", qc->closing ? " closing" : "");
         p = ngx_slprintf(p, last, "%s", qc->draining ? " draining" : "");
         p = ngx_slprintf(p, last, "%s", qc->key_phase ? " kp" : "");
@@ -2138,6 +2144,21 @@ ngx_quic_finalize_connection(ngx_connection_t *c, ngx_uint_t err,
 }
 
 
+void
+ngx_quic_shutdown_connection(ngx_connection_t *c, ngx_uint_t err,
+    const char *reason)
+{
+    ngx_quic_connection_t  *qc;
+
+    qc = ngx_quic_get_connection(c);
+    qc->shutdown = 1;
+    qc->shutdown_code = err;
+    qc->shutdown_reason = reason;
+
+    ngx_quic_shutdown_quic(c);
+}
+
+
 static void
 ngx_quic_close_timer_handler(ngx_event_t *ev)
 {
@@ -5945,6 +5966,10 @@ ngx_quic_create_client_stream(ngx_connection_t *c, uint64_t id)
 
     qc = ngx_quic_get_connection(c);
 
+    if (qc->shutdown) {
+        return NGX_QUIC_STREAM_GONE;
+    }
+
     if (id & NGX_QUIC_STREAM_UNIDIRECTIONAL) {
 
         if (id & NGX_QUIC_STREAM_SERVER_INITIATED) {
@@ -6016,6 +6041,10 @@ ngx_quic_create_client_stream(ngx_connection_t *c, uint64_t id)
         }
 
         sn->c->listening->handler(sn->c);
+
+        if (qc->shutdown) {
+            return NGX_QUIC_STREAM_GONE;
+        }
     }
 
     return ngx_quic_create_stream(c, id, n);
@@ -6410,7 +6439,7 @@ ngx_quic_stream_cleanup_handler(void *data)
         if (!c->read->pending_eof && !c->read->error) {
             frame = ngx_quic_alloc_frame(pc);
             if (frame == NULL) {
-                return;
+                goto done;
             }
 
             frame->level = ssl_encryption_application;
@@ -6425,7 +6454,7 @@ ngx_quic_stream_cleanup_handler(void *data)
     if ((qs->id & NGX_QUIC_STREAM_SERVER_INITIATED) == 0) {
         frame = ngx_quic_alloc_frame(pc);
         if (frame == NULL) {
-            return;
+            goto done;
         }
 
         frame->level = ssl_encryption_application;
@@ -6444,12 +6473,12 @@ ngx_quic_stream_cleanup_handler(void *data)
 
         if (qs->id & NGX_QUIC_STREAM_UNIDIRECTIONAL) {
             /* do not send fin for client unidirectional streams */
-            return;
+            goto done;
         }
     }
 
     if (c->write->error) {
-        goto error;
+        goto done;
     }
 
     ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
@@ -6457,7 +6486,7 @@ ngx_quic_stream_cleanup_handler(void *data)
 
     frame = ngx_quic_alloc_frame(pc);
     if (frame == NULL) {
-        return;
+        goto done;
     }
 
     frame->level = ssl_encryption_application;
@@ -6473,9 +6502,46 @@ ngx_quic_stream_cleanup_handler(void *data)
 
     ngx_quic_queue_frame(qc, frame);
 
-error:
+done:
 
     (void) ngx_quic_output(pc);
+
+    if (qc->shutdown) {
+        ngx_quic_shutdown_quic(pc);
+    }
+}
+
+
+static void
+ngx_quic_shutdown_quic(ngx_connection_t *c)
+{
+    ngx_rbtree_t           *tree;
+    ngx_rbtree_node_t      *node;
+    ngx_quic_stream_t      *qs;
+    ngx_quic_connection_t  *qc;
+
+    qc = ngx_quic_get_connection(c);
+
+    if (qc->closing) {
+        return;
+    }
+
+    tree = &qc->streams.tree;
+
+    if (tree->root != tree->sentinel) {
+        for (node = ngx_rbtree_min(tree->root, tree->sentinel);
+             node;
+             node = ngx_rbtree_next(tree, node))
+        {
+            qs = (ngx_quic_stream_t *) node;
+
+            if (!qs->cancelable) {
+                return;
+            }
+        }
+    }
+
+    ngx_quic_finalize_connection(c, qc->shutdown_code, qc->shutdown_reason);
 }
 
 

src/event/quic/ngx_event_quic.h

@@ -120,6 +120,7 @@ struct ngx_quic_stream_s {
     uint64_t                   send_max_data;
     ngx_buf_t                 *b;
     ngx_quic_frames_stream_t   fs;
+    ngx_uint_t                 cancelable;  /* unsigned  cancelable:1; */
 };
 
 
@@ -130,6 +131,8 @@ void ngx_quic_run(ngx_connection_t *c, ngx_quic_conf_t *conf);
 ngx_connection_t *ngx_quic_open_stream(ngx_connection_t *c, ngx_uint_t bidi);
 void ngx_quic_finalize_connection(ngx_connection_t *c, ngx_uint_t err,
     const char *reason);
+void ngx_quic_shutdown_connection(ngx_connection_t *c, ngx_uint_t err,
+    const char *reason);
 ngx_int_t ngx_quic_reset_stream(ngx_connection_t *c, ngx_uint_t err);
 uint32_t ngx_quic_version(ngx_connection_t *c);
 ngx_int_t ngx_quic_get_packet_dcid(ngx_log_t *log, u_char *data, size_t len,

src/http/v3/ngx_http_v3.h

@@ -82,6 +82,9 @@
 #define ngx_http_v3_finalize_connection(c, code, reason)                      \
     ngx_quic_finalize_connection(c->quic->parent, code, reason)
 
+#define ngx_http_v3_shutdown_connection(c, code, reason)                      \
+    ngx_quic_shutdown_connection(c->quic->parent, code, reason)
+
 
 typedef struct {
     ngx_quic_tp_t                 quic;

src/http/v3/ngx_http_v3_request.c

@@ -93,6 +93,9 @@ ngx_http_v3_init(ngx_connection_t *c)
             ngx_http_close_connection(c);
             return;
         }
+
+        ngx_http_v3_shutdown_connection(c, NGX_HTTP_V3_ERR_NO_ERROR,
+                                        "reached maximum number of requests");
     }
 
     cscf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_core_module);

src/http/v3/ngx_http_v3_streams.c

@@ -80,6 +80,8 @@ ngx_http_v3_init_uni_stream(ngx_connection_t *c)
 
     ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http3 init uni stream");
 
+    c->quic->cancelable = 1;
+
     us = ngx_pcalloc(c->pool, sizeof(ngx_http_v3_uni_stream_t));
     if (us == NULL) {
         ngx_http_close_connection(c);
@@ -436,6 +438,8 @@ ngx_http_v3_get_uni_stream(ngx_connection_t *c, ngx_uint_t type)
         return NULL;
     }
 
+    sc->quic->cancelable = 1;
+
     ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0,
                    "http3 create uni stream, type:%ui", type);