mirror/nginx
1
0
Fork 0
mirror of https://github.com/nginx/nginx.git synced 2024-12-04 13:59:00 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Buffers reuse in chunked filter.

Browse Source 
There were 2 buffers allocated on each buffer chain sent through chunked
filter (one buffer for chunk size, another one for trailing CRLF, about
120 bytes in total on 32-bit platforms).  This resulted in large memory
consumption with long-lived requests sending many buffer chains.  Usual
example of problematic scenario is streaming though proxy with
proxy_buffering set to off.

Introduced buffers reuse reduces memory consumption in the above problematic
scenario.

See here for initial report:
http://mailman.nginx.org/pipermail/nginx/2010-April/019814.html
This commit is contained in:
Maxim Dounin 2011-09-15 15:25:42 +00:00
parent 57b4fdb6c8
commit a890b313f3
1 changed files with 68 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
src/http/modules/ngx_http_chunked_filter_module.c
View File

@ -9,6 +9,12 @@
#include <ngx_http.h>
typedef struct {
ngx_chain_t *free;
ngx_chain_t *busy;
} ngx_http_chunked_filter_ctx_t;
static ngx_int_t ngx_http_chunked_filter_init(ngx_conf_t *cf);
@ -50,7 +56,8 @@ static ngx_http_output_body_filter_pt ngx_http_next_body_filter;
static ngx_int_t
ngx_http_chunked_header_filter(ngx_http_request_t *r)
{
ngx_http_core_loc_conf_t *clcf;
ngx_http_core_loc_conf_t *clcf;
ngx_http_chunked_filter_ctx_t *ctx;
if (r->headers_out.status == NGX_HTTP_NOT_MODIFIED
|| r->headers_out.status == NGX_HTTP_NO_CONTENT
@ -70,6 +77,14 @@ ngx_http_chunked_header_filter(ngx_http_request_t *r)
if (clcf->chunked_transfer_encoding) {
r->chunked = 1;
ctx = ngx_pcalloc(r->pool,
sizeof(ngx_http_chunked_filter_ctx_t));
if (ctx == NULL) {
return NGX_ERROR;
}
ngx_http_set_ctx(r, ctx, ngx_http_chunked_filter_module);
} else {
r->keepalive = 0;
}
@ -83,17 +98,21 @@ ngx_http_chunked_header_filter(ngx_http_request_t *r)
static ngx_int_t
ngx_http_chunked_body_filter(ngx_http_request_t *r, ngx_chain_t *in)
{
u_char *chunk;
off_t size;
ngx_buf_t *b;
ngx_chain_t out, tail, *cl, *tl, **ll;
u_char *chunk;
off_t size;
ngx_int_t rc;
ngx_buf_t *b;
ngx_chain_t *out, *cl, *tl, **ll;
ngx_http_chunked_filter_ctx_t *ctx;
if (in == NULL || !r->chunked || r->header_only) {
return ngx_http_next_body_filter(r, in);
}
out.buf = NULL;
ll = &out.next;
ctx = ngx_http_get_module_ctx(r, ngx_http_chunked_filter_module);
out = NULL;
ll = &out;
size = 0;
cl = in;
@ -127,31 +146,46 @@ ngx_http_chunked_body_filter(ngx_http_request_t *r, ngx_chain_t *in)
}
if (size) {
b = ngx_calloc_buf(r->pool);
if (b == NULL) {
tl = ngx_chain_get_free_buf(r->pool, &ctx->free);
if (tl == NULL) {
return NGX_ERROR;
}
/* the "0000000000000000" is 64-bit hexadimal string */
b = tl->buf;
chunk = b->start;
chunk = ngx_palloc(r->pool, sizeof("0000000000000000" CRLF) - 1);
if (chunk == NULL) {
return NGX_ERROR;
/* the "0000000000000000" is 64-bit hexadecimal string */
chunk = ngx_palloc(r->pool, sizeof("0000000000000000" CRLF) - 1);
if (chunk == NULL) {
return NGX_ERROR;
}
b->start = chunk;
b->end = chunk + sizeof("0000000000000000" CRLF) - 1;
}
b->tag = (ngx_buf_tag_t) &ngx_http_chunked_filter_module;
b->memory = 0;
b->temporary = 1;
b->pos = chunk;
b->last = ngx_sprintf(chunk, "%xO" CRLF, size);
out.buf = b;
tl->next = out;
out = tl;
}
if (cl->buf->last_buf) {
b = ngx_calloc_buf(r->pool);
if (b == NULL) {
tl = ngx_chain_get_free_buf(r->pool, &ctx->free);
if (tl == NULL) {
return NGX_ERROR;
}
b = tl->buf;
b->tag = (ngx_buf_tag_t) &ngx_http_chunked_filter_module;
b->temporary = 0;
b->memory = 1;
b->last_buf = 1;
b->pos = (u_char *) CRLF "0" CRLF CRLF;
@ -159,35 +193,38 @@ ngx_http_chunked_body_filter(ngx_http_request_t *r, ngx_chain_t *in)
cl->buf->last_buf = 0;
*ll = tl;
if (size == 0) {
b->pos += 2;
out.buf = b;
out.next = NULL;
return ngx_http_next_body_filter(r, &out);
}
} else {
if (size == 0) {
*ll = NULL;
return ngx_http_next_body_filter(r, out.next);
}
b = ngx_calloc_buf(r->pool);
if (b == NULL) {
} else if (size > 0) {
tl = ngx_chain_get_free_buf(r->pool, &ctx->free);
if (tl == NULL) {
return NGX_ERROR;
}
b = tl->buf;
b->tag = (ngx_buf_tag_t) &ngx_http_chunked_filter_module;
b->temporary = 0;
b->memory = 1;
b->pos = (u_char *) CRLF;
b->last = b->pos + 2;
*ll = tl;
} else {
*ll = NULL;
}
tail.buf = b;
tail.next = NULL;
*ll = &tail;
rc = ngx_http_next_body_filter(r, out);
return ngx_http_next_body_filter(r, &out);
ngx_chain_update_chains(&ctx->free, &ctx->busy, &out,
(ngx_buf_tag_t) &ngx_http_chunked_filter_module);
return rc;
}