From 9533df5b728833dd516f44d18953a3731c29e787 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Mon, 15 Mar 2021 16:39:33 +0300 Subject: [PATCH] QUIC: connection shutdown. The function ngx_quic_shutdown_connection() waits until all non-cancelable streams are closed, and then closes the connection. In HTTP/3 cancelable streams are all unidirectional streams except push streams. The function is called from HTTP/3 when client reaches keepalive_requests. --- src/event/quic/ngx_event_quic.c | 78 ++++++++++++++++++++++++++++--- src/event/quic/ngx_event_quic.h | 3 ++ src/http/v3/ngx_http_v3.h | 3 ++ src/http/v3/ngx_http_v3_request.c | 3 ++ src/http/v3/ngx_http_v3_streams.c | 4 ++ 5 files changed, 85 insertions(+), 6 deletions(-) diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c index cb6aff3fb..f38b15910 100644 --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -174,9 +174,13 @@ typedef struct { ngx_uint_t error_ftype; const char *error_reason; + ngx_uint_t shutdown_code; + const char *shutdown_reason; + unsigned error_app:1; unsigned send_timer_set:1; unsigned closing:1; + unsigned shutdown:1; unsigned draining:1; unsigned key_phase:1; unsigned validated:1; @@ -384,6 +388,7 @@ static ngx_chain_t *ngx_quic_stream_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit); static size_t ngx_quic_max_stream_flow(ngx_connection_t *c); static void ngx_quic_stream_cleanup_handler(void *data); +static void ngx_quic_shutdown_quic(ngx_connection_t *c); static ngx_quic_frame_t *ngx_quic_alloc_frame(ngx_connection_t *c); static void ngx_quic_free_frame(ngx_connection_t *c, ngx_quic_frame_t *frame); @@ -684,6 +689,7 @@ ngx_quic_connstate_dbg(ngx_connection_t *c) } } + p = ngx_slprintf(p, last, "%s", qc->shutdown ? " shutdown" : ""); p = ngx_slprintf(p, last, "%s", qc->closing ? " closing" : ""); p = ngx_slprintf(p, last, "%s", qc->draining ? " draining" : ""); p = ngx_slprintf(p, last, "%s", qc->key_phase ? " kp" : ""); @@ -2138,6 +2144,21 @@ ngx_quic_finalize_connection(ngx_connection_t *c, ngx_uint_t err, } +void +ngx_quic_shutdown_connection(ngx_connection_t *c, ngx_uint_t err, + const char *reason) +{ + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + qc->shutdown = 1; + qc->shutdown_code = err; + qc->shutdown_reason = reason; + + ngx_quic_shutdown_quic(c); +} + + static void ngx_quic_close_timer_handler(ngx_event_t *ev) { @@ -5945,6 +5966,10 @@ ngx_quic_create_client_stream(ngx_connection_t *c, uint64_t id) qc = ngx_quic_get_connection(c); + if (qc->shutdown) { + return NGX_QUIC_STREAM_GONE; + } + if (id & NGX_QUIC_STREAM_UNIDIRECTIONAL) { if (id & NGX_QUIC_STREAM_SERVER_INITIATED) { @@ -6016,6 +6041,10 @@ ngx_quic_create_client_stream(ngx_connection_t *c, uint64_t id) } sn->c->listening->handler(sn->c); + + if (qc->shutdown) { + return NGX_QUIC_STREAM_GONE; + } } return ngx_quic_create_stream(c, id, n); @@ -6410,7 +6439,7 @@ ngx_quic_stream_cleanup_handler(void *data) if (!c->read->pending_eof && !c->read->error) { frame = ngx_quic_alloc_frame(pc); if (frame == NULL) { - return; + goto done; } frame->level = ssl_encryption_application; @@ -6425,7 +6454,7 @@ ngx_quic_stream_cleanup_handler(void *data) if ((qs->id & NGX_QUIC_STREAM_SERVER_INITIATED) == 0) { frame = ngx_quic_alloc_frame(pc); if (frame == NULL) { - return; + goto done; } frame->level = ssl_encryption_application; @@ -6444,12 +6473,12 @@ ngx_quic_stream_cleanup_handler(void *data) if (qs->id & NGX_QUIC_STREAM_UNIDIRECTIONAL) { /* do not send fin for client unidirectional streams */ - return; + goto done; } } if (c->write->error) { - goto error; + goto done; } ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, @@ -6457,7 +6486,7 @@ ngx_quic_stream_cleanup_handler(void *data) frame = ngx_quic_alloc_frame(pc); if (frame == NULL) { - return; + goto done; } frame->level = ssl_encryption_application; @@ -6473,9 +6502,46 @@ ngx_quic_stream_cleanup_handler(void *data) ngx_quic_queue_frame(qc, frame); -error: +done: (void) ngx_quic_output(pc); + + if (qc->shutdown) { + ngx_quic_shutdown_quic(pc); + } +} + + +static void +ngx_quic_shutdown_quic(ngx_connection_t *c) +{ + ngx_rbtree_t *tree; + ngx_rbtree_node_t *node; + ngx_quic_stream_t *qs; + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + + if (qc->closing) { + return; + } + + tree = &qc->streams.tree; + + if (tree->root != tree->sentinel) { + for (node = ngx_rbtree_min(tree->root, tree->sentinel); + node; + node = ngx_rbtree_next(tree, node)) + { + qs = (ngx_quic_stream_t *) node; + + if (!qs->cancelable) { + return; + } + } + } + + ngx_quic_finalize_connection(c, qc->shutdown_code, qc->shutdown_reason); } diff --git a/src/event/quic/ngx_event_quic.h b/src/event/quic/ngx_event_quic.h index be0eec699..0a38d911c 100644 --- a/src/event/quic/ngx_event_quic.h +++ b/src/event/quic/ngx_event_quic.h @@ -120,6 +120,7 @@ struct ngx_quic_stream_s { uint64_t send_max_data; ngx_buf_t *b; ngx_quic_frames_stream_t fs; + ngx_uint_t cancelable; /* unsigned cancelable:1; */ }; @@ -130,6 +131,8 @@ void ngx_quic_run(ngx_connection_t *c, ngx_quic_conf_t *conf); ngx_connection_t *ngx_quic_open_stream(ngx_connection_t *c, ngx_uint_t bidi); void ngx_quic_finalize_connection(ngx_connection_t *c, ngx_uint_t err, const char *reason); +void ngx_quic_shutdown_connection(ngx_connection_t *c, ngx_uint_t err, + const char *reason); ngx_int_t ngx_quic_reset_stream(ngx_connection_t *c, ngx_uint_t err); uint32_t ngx_quic_version(ngx_connection_t *c); ngx_int_t ngx_quic_get_packet_dcid(ngx_log_t *log, u_char *data, size_t len, diff --git a/src/http/v3/ngx_http_v3.h b/src/http/v3/ngx_http_v3.h index a8a5c5cd4..be0ed127d 100644 --- a/src/http/v3/ngx_http_v3.h +++ b/src/http/v3/ngx_http_v3.h @@ -82,6 +82,9 @@ #define ngx_http_v3_finalize_connection(c, code, reason) \ ngx_quic_finalize_connection(c->quic->parent, code, reason) +#define ngx_http_v3_shutdown_connection(c, code, reason) \ + ngx_quic_shutdown_connection(c->quic->parent, code, reason) + typedef struct { ngx_quic_tp_t quic; diff --git a/src/http/v3/ngx_http_v3_request.c b/src/http/v3/ngx_http_v3_request.c index 0c055ba0e..d4a5faccf 100644 --- a/src/http/v3/ngx_http_v3_request.c +++ b/src/http/v3/ngx_http_v3_request.c @@ -93,6 +93,9 @@ ngx_http_v3_init(ngx_connection_t *c) ngx_http_close_connection(c); return; } + + ngx_http_v3_shutdown_connection(c, NGX_HTTP_V3_ERR_NO_ERROR, + "reached maximum number of requests"); } cscf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_core_module); diff --git a/src/http/v3/ngx_http_v3_streams.c b/src/http/v3/ngx_http_v3_streams.c index e09556c93..eac49a659 100644 --- a/src/http/v3/ngx_http_v3_streams.c +++ b/src/http/v3/ngx_http_v3_streams.c @@ -80,6 +80,8 @@ ngx_http_v3_init_uni_stream(ngx_connection_t *c) ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http3 init uni stream"); + c->quic->cancelable = 1; + us = ngx_pcalloc(c->pool, sizeof(ngx_http_v3_uni_stream_t)); if (us == NULL) { ngx_http_close_connection(c); @@ -436,6 +438,8 @@ ngx_http_v3_get_uni_stream(ngx_connection_t *c, ngx_uint_t type) return NULL; } + sc->quic->cancelable = 1; + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, "http3 create uni stream, type:%ui", type);