mirror/nginx
1
0
Fork 0
mirror of https://github.com/nginx/nginx.git synced 2024-11-23 20:19:02 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Upstream: re-resolvable servers.

Browse Source 
Specifying the upstream server by a hostname together with the
"resolve" parameter will make the hostname to be periodically
resolved, and upstream servers added/removed as necessary.

This requires a "resolver" at the "http" configuration block.

The "resolver_timeout" parameter also affects when the failed
DNS requests will be attempted again.  Responses with NXDOMAIN
will be attempted again in 10 seconds.

Upstream has a configuration generation number that is incremented each
time servers are added/removed to the primary/backup list.  This number
is remembered by the peer.init method, and if peer.get detects a change
in configuration, it returns NGX_BUSY.

Each server has a reference counter.  It is incremented by peer.get and
decremented by peer.free.  When a server is removed, it is removed from
the list of servers and is marked as "zombie".  The memory allocated by
a zombie peer is freed only when its reference count becomes zero.

Co-authored-by: Roman Arutyunyan <arut@nginx.com>
Co-authored-by: Sergey Kandaurov <pluknet@nginx.com>
Co-authored-by: Vladimir Homutov <vl@nginx.com>
This commit is contained in:
Ruslan Ermilov 2014-02-15 15:12:34 +04:00 committed by Aleksei Bavshin
parent ebd18ec181
commit db6870e06d
18 changed files with 1704 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
src/http/modules/ngx_http_upstream_hash_module.c
View File

@ -24,6 +24,9 @@ typedef struct {
typedef struct {
ngx_http_complex_value_t key;
#if (NGX_HTTP_UPSTREAM_ZONE)
ngx_uint_t config;
#endif
ngx_http_upstream_chash_points_t *points;
} ngx_http_upstream_hash_srv_conf_t;
@ -49,6 +52,8 @@ static ngx_int_t ngx_http_upstream_get_hash_peer(ngx_peer_connection_t *pc,
static ngx_int_t ngx_http_upstream_init_chash(ngx_conf_t *cf,
ngx_http_upstream_srv_conf_t *us);
static ngx_int_t ngx_http_upstream_update_chash(ngx_pool_t *pool,
ngx_http_upstream_srv_conf_t *us);
static int ngx_libc_cdecl
ngx_http_upstream_chash_cmp_points(const void *one, const void *two);
static ngx_uint_t ngx_http_upstream_find_chash_point(
@ -178,11 +183,18 @@ ngx_http_upstream_get_hash_peer(ngx_peer_connection_t *pc, void *data)
ngx_http_upstream_rr_peers_rlock(hp->rrp.peers);
if (hp->tries > 20 || hp->rrp.peers->single || hp->key.len == 0) {
if (hp->tries > 20 || hp->rrp.peers->number < 2 || hp->key.len == 0) {
ngx_http_upstream_rr_peers_unlock(hp->rrp.peers);
return hp->get_rr_peer(pc, &hp->rrp);
}
#if (NGX_HTTP_UPSTREAM_ZONE)
if (hp->rrp.peers->config && hp->rrp.config != *hp->rrp.peers->config) {
ngx_http_upstream_rr_peers_unlock(hp->rrp.peers);
return hp->get_rr_peer(pc, &hp->rrp);
}
#endif
now = ngx_time();
pc->cached = 0;
@ -262,6 +274,7 @@ ngx_http_upstream_get_hash_peer(ngx_peer_connection_t *pc, void *data)
}
hp->rrp.current = peer;
ngx_http_upstream_rr_peer_ref(hp->rrp.peers, peer);
pc->sockaddr = peer->sockaddr;
pc->socklen = peer->socklen;
@ -284,6 +297,26 @@ ngx_http_upstream_get_hash_peer(ngx_peer_connection_t *pc, void *data)
static ngx_int_t
ngx_http_upstream_init_chash(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us)
{
if (ngx_http_upstream_init_round_robin(cf, us) != NGX_OK) {
return NGX_ERROR;
}
us->peer.init = ngx_http_upstream_init_chash_peer;
#if (NGX_HTTP_UPSTREAM_ZONE)
if (us->shm_zone) {
return NGX_OK;
}
#endif
return ngx_http_upstream_update_chash(cf->pool, us);
}
static ngx_int_t
ngx_http_upstream_update_chash(ngx_pool_t *pool,
ngx_http_upstream_srv_conf_t *us)
{
u_char *host, *port, c;
size_t host_len, port_len, size;
@ -299,25 +332,32 @@ ngx_http_upstream_init_chash(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us)
u_char byte[4];
} prev_hash;
if (ngx_http_upstream_init_round_robin(cf, us) != NGX_OK) {
return NGX_ERROR;
}
hcf = ngx_http_conf_upstream_srv_conf(us, ngx_http_upstream_hash_module);
us->peer.init = ngx_http_upstream_init_chash_peer;
if (hcf->points) {
ngx_free(hcf->points);
hcf->points = NULL;
}
peers = us->peer.data;
npoints = peers->total_weight * 160;
size = sizeof(ngx_http_upstream_chash_points_t)
+ sizeof(ngx_http_upstream_chash_point_t) * (npoints - 1);
- sizeof(ngx_http_upstream_chash_point_t)
+ sizeof(ngx_http_upstream_chash_point_t) * npoints;
points = ngx_palloc(cf->pool, size);
points = pool ? ngx_palloc(pool, size) : ngx_alloc(size, ngx_cycle->log);
if (points == NULL) {
return NGX_ERROR;
}
points->number = 0;
if (npoints == 0) {
hcf->points = points;
return NGX_OK;
}
for (peer = peers->peer; peer; peer = peer->next) {
server = &peer->server;
@ -401,7 +441,6 @@ ngx_http_upstream_init_chash(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us)
points->number = i + 1;
hcf = ngx_http_conf_upstream_srv_conf(us, ngx_http_upstream_hash_module);
hcf->points = points;
return NGX_OK;
@ -481,7 +520,22 @@ ngx_http_upstream_init_chash_peer(ngx_http_request_t *r,
ngx_http_upstream_rr_peers_rlock(hp->rrp.peers);
#if (NGX_HTTP_UPSTREAM_ZONE)
if (hp->rrp.peers->config
&& (hcf->points == NULL || hcf->config != *hp->rrp.peers->config))
{
if (ngx_http_upstream_update_chash(NULL, us) != NGX_OK) {
ngx_http_upstream_rr_peers_unlock(hp->rrp.peers);
return NGX_ERROR;
}
hcf->config = *hp->rrp.peers->config;
}
#endif
if (hcf->points->number) {
hp->hash = ngx_http_upstream_find_chash_point(hcf->points, hash);
}
ngx_http_upstream_rr_peers_unlock(hp->rrp.peers);
@ -517,6 +571,20 @@ ngx_http_upstream_get_chash_peer(ngx_peer_connection_t *pc, void *data)
pc->cached = 0;
pc->connection = NULL;
if (hp->rrp.peers->number == 0) {
pc->name = hp->rrp.peers->name;
ngx_http_upstream_rr_peers_unlock(hp->rrp.peers);
return NGX_BUSY;
}
#if (NGX_HTTP_UPSTREAM_ZONE)
if (hp->rrp.peers->config && hp->rrp.config != *hp->rrp.peers->config) {
pc->name = hp->rrp.peers->name;
ngx_http_upstream_rr_peers_unlock(hp->rrp.peers);
return NGX_BUSY;
}
#endif
now = ngx_time();
hcf = hp->conf;
@ -597,6 +665,7 @@ ngx_http_upstream_get_chash_peer(ngx_peer_connection_t *pc, void *data)
found:
hp->rrp.current = best;
ngx_http_upstream_rr_peer_ref(hp->rrp.peers, best);
pc->sockaddr = best->sockaddr;
pc->socklen = best->socklen;
@ -664,6 +733,7 @@ ngx_http_upstream_hash(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
}
uscf->flags = NGX_HTTP_UPSTREAM_CREATE
|NGX_HTTP_UPSTREAM_MODIFY
|NGX_HTTP_UPSTREAM_WEIGHT
|NGX_HTTP_UPSTREAM_MAX_CONNS
|NGX_HTTP_UPSTREAM_MAX_FAILS

12
src/http/modules/ngx_http_upstream_ip_hash_module.c
View File

@ -163,11 +163,19 @@ ngx_http_upstream_get_ip_hash_peer(ngx_peer_connection_t *pc, void *data)
ngx_http_upstream_rr_peers_rlock(iphp->rrp.peers);
if (iphp->tries > 20 || iphp->rrp.peers->single) {
if (iphp->tries > 20 || iphp->rrp.peers->number < 2) {
ngx_http_upstream_rr_peers_unlock(iphp->rrp.peers);
return iphp->get_rr_peer(pc, &iphp->rrp);
}
#if (NGX_HTTP_UPSTREAM_ZONE)
if (iphp->rrp.peers->config && iphp->rrp.config != *iphp->rrp.peers->config)
{
ngx_http_upstream_rr_peers_unlock(iphp->rrp.peers);
return iphp->get_rr_peer(pc, &iphp->rrp);
}
#endif
now = ngx_time();
pc->cached = 0;
@ -232,6 +240,7 @@ ngx_http_upstream_get_ip_hash_peer(ngx_peer_connection_t *pc, void *data)
}
iphp->rrp.current = peer;
ngx_http_upstream_rr_peer_ref(iphp->rrp.peers, peer);
pc->sockaddr = peer->sockaddr;
pc->socklen = peer->socklen;
@ -268,6 +277,7 @@ ngx_http_upstream_ip_hash(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
uscf->peer.init_upstream = ngx_http_upstream_init_ip_hash;
uscf->flags = NGX_HTTP_UPSTREAM_CREATE
|NGX_HTTP_UPSTREAM_MODIFY
|NGX_HTTP_UPSTREAM_WEIGHT
|NGX_HTTP_UPSTREAM_MAX_CONNS
|NGX_HTTP_UPSTREAM_MAX_FAILS

12
src/http/modules/ngx_http_upstream_least_conn_module.c
View File

@ -124,6 +124,12 @@ ngx_http_upstream_get_least_conn_peer(ngx_peer_connection_t *pc, void *data)
ngx_http_upstream_rr_peers_wlock(peers);
#if (NGX_HTTP_UPSTREAM_ZONE)
if (peers->config && rrp->config != *peers->config) {
goto busy;
}
#endif
best = NULL;
total = 0;
@ -244,6 +250,7 @@ ngx_http_upstream_get_least_conn_peer(ngx_peer_connection_t *pc, void *data)
best->conns++;
rrp->current = best;
ngx_http_upstream_rr_peer_ref(peers, best);
n = p / (8 * sizeof(uintptr_t));
m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t));
@ -280,6 +287,10 @@ failed:
ngx_http_upstream_rr_peers_wlock(peers);
}
#if (NGX_HTTP_UPSTREAM_ZONE)
busy:
#endif
ngx_http_upstream_rr_peers_unlock(peers);
pc->name = peers->name;
@ -303,6 +314,7 @@ ngx_http_upstream_least_conn(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
uscf->peer.init_upstream = ngx_http_upstream_init_least_conn;
uscf->flags = NGX_HTTP_UPSTREAM_CREATE
|NGX_HTTP_UPSTREAM_MODIFY
|NGX_HTTP_UPSTREAM_WEIGHT
|NGX_HTTP_UPSTREAM_MAX_CONNS
|NGX_HTTP_UPSTREAM_MAX_FAILS

35
src/http/modules/ngx_http_upstream_random_module.c
View File

@ -17,6 +17,9 @@ typedef struct {
typedef struct {
ngx_uint_t two;
#if (NGX_HTTP_UPSTREAM_ZONE)
ngx_uint_t config;
#endif
ngx_http_upstream_random_range_t *ranges;
} ngx_http_upstream_random_srv_conf_t;
@ -127,6 +130,11 @@ ngx_http_upstream_update_random(ngx_pool_t *pool,
rcf = ngx_http_conf_upstream_srv_conf(us, ngx_http_upstream_random_module);
if (rcf->ranges) {
ngx_free(rcf->ranges);
rcf->ranges = NULL;
}
peers = us->peer.data;
size = peers->number * sizeof(ngx_http_upstream_random_range_t);
@ -186,11 +194,15 @@ ngx_http_upstream_init_random_peer(ngx_http_request_t *r,
ngx_http_upstream_rr_peers_rlock(rp->rrp.peers);
#if (NGX_HTTP_UPSTREAM_ZONE)
if (rp->rrp.peers->shpool && rcf->ranges == NULL) {
if (rp->rrp.peers->config
&& (rcf->ranges == NULL || rcf->config != *rp->rrp.peers->config))
{
if (ngx_http_upstream_update_random(NULL, us) != NGX_OK) {
ngx_http_upstream_rr_peers_unlock(rp->rrp.peers);
return NGX_ERROR;
}
rcf->config = *rp->rrp.peers->config;
}
#endif
@ -220,11 +232,18 @@ ngx_http_upstream_get_random_peer(ngx_peer_connection_t *pc, void *data)
ngx_http_upstream_rr_peers_rlock(peers);
if (rp->tries > 20 || peers->single) {
if (rp->tries > 20 || peers->number < 2) {
ngx_http_upstream_rr_peers_unlock(peers);
return ngx_http_upstream_get_round_robin_peer(pc, rrp);
}
#if (NGX_HTTP_UPSTREAM_ZONE)
if (peers->config && rrp->config != *peers->config) {
ngx_http_upstream_rr_peers_unlock(peers);
return ngx_http_upstream_get_round_robin_peer(pc, rrp);
}
#endif
pc->cached = 0;
pc->connection = NULL;
@ -274,6 +293,7 @@ ngx_http_upstream_get_random_peer(ngx_peer_connection_t *pc, void *data)
}
rrp->current = peer;
ngx_http_upstream_rr_peer_ref(peers, peer);
if (now - peer->checked > peer->fail_timeout) {
peer->checked = now;
@ -314,11 +334,18 @@ ngx_http_upstream_get_random2_peer(ngx_peer_connection_t *pc, void *data)
ngx_http_upstream_rr_peers_wlock(peers);
if (rp->tries > 20 || peers->single) {
if (rp->tries > 20 || peers->number < 2) {
ngx_http_upstream_rr_peers_unlock(peers);
return ngx_http_upstream_get_round_robin_peer(pc, rrp);
}
#if (NGX_HTTP_UPSTREAM_ZONE)
if (peers->config && rrp->config != *peers->config) {
ngx_http_upstream_rr_peers_unlock(peers);
return ngx_http_upstream_get_round_robin_peer(pc, rrp);
}
#endif
pc->cached = 0;
pc->connection = NULL;
@ -384,6 +411,7 @@ ngx_http_upstream_get_random2_peer(ngx_peer_connection_t *pc, void *data)
}
rrp->current = peer;
ngx_http_upstream_rr_peer_ref(peers, peer);
if (now - peer->checked > peer->fail_timeout) {
peer->checked = now;
@ -467,6 +495,7 @@ ngx_http_upstream_random(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
uscf->peer.init_upstream = ngx_http_upstream_init_random;
uscf->flags = NGX_HTTP_UPSTREAM_CREATE
|NGX_HTTP_UPSTREAM_MODIFY
|NGX_HTTP_UPSTREAM_WEIGHT
|NGX_HTTP_UPSTREAM_MAX_CONNS
|NGX_HTTP_UPSTREAM_MAX_FAILS

362
src/http/modules/ngx_http_upstream_zone_module.c
View File

@ -18,6 +18,13 @@ static ngx_http_upstream_rr_peers_t *ngx_http_upstream_zone_copy_peers(
ngx_slab_pool_t *shpool, ngx_http_upstream_srv_conf_t *uscf);
static ngx_http_upstream_rr_peer_t *ngx_http_upstream_zone_copy_peer(
ngx_http_upstream_rr_peers_t *peers, ngx_http_upstream_rr_peer_t *src);
static void ngx_http_upstream_zone_set_single(
ngx_http_upstream_srv_conf_t *uscf);
static void ngx_http_upstream_zone_remove_peer_locked(
ngx_http_upstream_rr_peers_t *peers, ngx_http_upstream_rr_peer_t *peer);
static ngx_int_t ngx_http_upstream_zone_init_worker(ngx_cycle_t *cycle);
static void ngx_http_upstream_zone_resolve_timer(ngx_event_t *event);
static void ngx_http_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx);
static ngx_command_t ngx_http_upstream_zone_commands[] = {
@ -55,7 +62,7 @@ ngx_module_t ngx_http_upstream_zone_module = {
NGX_HTTP_MODULE, /* module type */
NULL, /* init master */
NULL, /* init module */
NULL, /* init process */
ngx_http_upstream_zone_init_worker, /* init process */
NULL, /* init thread */
NULL, /* exit thread */
NULL, /* exit process */
@ -188,9 +195,15 @@ ngx_http_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,
ngx_http_upstream_srv_conf_t *uscf)
{
ngx_str_t *name;
ngx_uint_t *config;
ngx_http_upstream_rr_peer_t *peer, **peerp;
ngx_http_upstream_rr_peers_t *peers, *backup;
config = ngx_slab_calloc(shpool, sizeof(ngx_uint_t));
if (config == NULL) {
return NULL;
}
peers = ngx_slab_alloc(shpool, sizeof(ngx_http_upstream_rr_peers_t));
if (peers == NULL) {
return NULL;
@ -214,6 +227,7 @@ ngx_http_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,
peers->name = name;
peers->shpool = shpool;
peers->config = config;
for (peerp = &peers->peer; *peerp; peerp = &peer->next) {
/* pool is unlocked */
@ -223,6 +237,17 @@ ngx_http_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,
}
*peerp = peer;
(*peers->config)++;
}
for (peerp = &peers->resolve; *peerp; peerp = &peer->next) {
peer = ngx_http_upstream_zone_copy_peer(peers, *peerp);
if (peer == NULL) {
return NULL;
}
*peerp = peer;
(*peers->config)++;
}
if (peers->next == NULL) {
@ -239,6 +264,7 @@ ngx_http_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,
backup->name = name;
backup->shpool = shpool;
backup->config = config;
for (peerp = &backup->peer; *peerp; peerp = &peer->next) {
/* pool is unlocked */
@ -248,6 +274,17 @@ ngx_http_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,
}
*peerp = peer;
(*backup->config)++;
}
for (peerp = &backup->resolve; *peerp; peerp = &peer->next) {
peer = ngx_http_upstream_zone_copy_peer(backup, *peerp);
if (peer == NULL) {
return NULL;
}
*peerp = peer;
(*backup->config)++;
}
peers->next = backup;
@ -279,6 +316,7 @@ ngx_http_upstream_zone_copy_peer(ngx_http_upstream_rr_peers_t *peers,
dst->sockaddr = NULL;
dst->name.data = NULL;
dst->server.data = NULL;
dst->host = NULL;
}
dst->sockaddr = ngx_slab_calloc_locked(pool, sizeof(ngx_sockaddr_t));
@ -301,12 +339,37 @@ ngx_http_upstream_zone_copy_peer(ngx_http_upstream_rr_peers_t *peers,
}
ngx_memcpy(dst->server.data, src->server.data, src->server.len);
if (src->host) {
dst->host = ngx_slab_calloc_locked(pool,
sizeof(ngx_http_upstream_host_t));
if (dst->host == NULL) {
goto failed;
}
dst->host->name.data = ngx_slab_alloc_locked(pool,
src->host->name.len);
if (dst->host->name.data == NULL) {
goto failed;
}
dst->host->peers = peers;
dst->host->peer = dst;
dst->host->name.len = src->host->name.len;
ngx_memcpy(dst->host->name.data, src->host->name.data,
src->host->name.len);
}
}
return dst;
failed:
if (dst->host) {
ngx_slab_free_locked(pool, dst->host);
}
if (dst->server.data) {
ngx_slab_free_locked(pool, dst->server.data);
}
@ -323,3 +386,300 @@ failed:
return NULL;
}
static void
ngx_http_upstream_zone_set_single(ngx_http_upstream_srv_conf_t *uscf)
{
ngx_http_upstream_rr_peers_t *peers;
peers = uscf->peer.data;
if (peers->number == 1
&& (peers->next == NULL || peers->next->number == 0))
{
peers->single = 1;
} else {
peers->single = 0;
}
}
static void
ngx_http_upstream_zone_remove_peer_locked(ngx_http_upstream_rr_peers_t *peers,
ngx_http_upstream_rr_peer_t *peer)
{
peers->total_weight -= peer->weight;
peers->number--;
peers->tries -= (peer->down == 0);
(*peers->config)++;
peers->weighted = (peers->total_weight != peers->number);
ngx_http_upstream_rr_peer_free(peers, peer);
}
static ngx_int_t
ngx_http_upstream_zone_init_worker(ngx_cycle_t *cycle)
{
ngx_uint_t i;
ngx_event_t *event;
ngx_http_upstream_rr_peer_t *peer;
ngx_http_upstream_rr_peers_t *peers;
ngx_http_upstream_srv_conf_t *uscf, **uscfp;
ngx_http_upstream_main_conf_t *umcf;
if (ngx_process != NGX_PROCESS_WORKER
&& ngx_process != NGX_PROCESS_SINGLE)
{
return NGX_OK;
}
umcf = ngx_http_cycle_get_module_main_conf(cycle, ngx_http_upstream_module);
if (umcf == NULL) {
return NGX_OK;
}
uscfp = umcf->upstreams.elts;
for (i = 0; i < umcf->upstreams.nelts; i++) {
uscf = uscfp[i];
if (uscf->shm_zone == NULL) {
continue;
}
peers = uscf->peer.data;
do {
ngx_http_upstream_rr_peers_wlock(peers);
for (peer = peers->resolve; peer; peer = peer->next) {
if (peer->host->worker != ngx_worker) {
continue;
}
event = &peer->host->event;
ngx_memzero(event, sizeof(ngx_event_t));
event->data = uscf;
event->handler = ngx_http_upstream_zone_resolve_timer;
event->log = cycle->log;
event->cancelable = 1;
ngx_add_timer(event, 1);
}
ngx_http_upstream_rr_peers_unlock(peers);
peers = peers->next;
} while (peers);
}
return NGX_OK;
}
static void
ngx_http_upstream_zone_resolve_timer(ngx_event_t *event)
{
ngx_resolver_ctx_t *ctx;
ngx_http_upstream_host_t *host;
ngx_http_upstream_srv_conf_t *uscf;
host = (ngx_http_upstream_host_t *) event;
uscf = event->data;
ctx = ngx_resolve_start(uscf->resolver, NULL);
if (ctx == NULL) {
goto retry;
}
if (ctx == NGX_NO_RESOLVER) {
ngx_log_error(NGX_LOG_ERR, event->log, 0,
"no resolver defined to resolve %V", &host->name);
return;
}
ctx->name = host->name;
ctx->handler = ngx_http_upstream_zone_resolve_handler;
ctx->data = host;
ctx->timeout = uscf->resolver_timeout;
ctx->cancelable = 1;
if (ngx_resolve_name(ctx) == NGX_OK) {
return;
}
retry:
ngx_add_timer(event, ngx_max(uscf->resolver_timeout, 1000));
}
static void
ngx_http_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx)
{
time_t now;
in_port_t port;
ngx_msec_t timer;
ngx_uint_t i, j;
ngx_event_t *event;
ngx_resolver_addr_t *addr;
ngx_http_upstream_host_t *host;
ngx_http_upstream_rr_peer_t *peer, *template, **peerp;
ngx_http_upstream_rr_peers_t *peers;
ngx_http_upstream_srv_conf_t *uscf;
host = ctx->data;
event = &host->event;
uscf = event->data;
peers = host->peers;
template = host->peer;
ngx_http_upstream_rr_peers_wlock(peers);
now = ngx_time();
if (ctx->state) {
ngx_log_error(NGX_LOG_ERR, event->log, 0,
"%V could not be resolved (%i: %s)",
&ctx->name, ctx->state,
ngx_resolver_strerror(ctx->state));
if (ctx->state != NGX_RESOLVE_NXDOMAIN) {
ngx_http_upstream_rr_peers_unlock(peers);
ngx_resolve_name_done(ctx);
ngx_add_timer(event, ngx_max(uscf->resolver_timeout, 1000));
return;
}
/* NGX_RESOLVE_NXDOMAIN */
ctx->naddrs = 0;
}
#if (NGX_DEBUG)
{
u_char text[NGX_SOCKADDR_STRLEN];
size_t len;
for (i = 0; i < ctx->naddrs; i++) {
len = ngx_sock_ntop(ctx->addrs[i].sockaddr, ctx->addrs[i].socklen,
text, NGX_SOCKADDR_STRLEN, 0);
ngx_log_debug3(NGX_LOG_DEBUG_HTTP, event->log, 0,
"name %V was resolved to %*s", &host->name, len, text);
}
}
#endif
for (peerp = &peers->peer; *peerp; /* void */ ) {
peer = *peerp;
if (peer->host != host) {
goto next;
}
for (j = 0; j < ctx->naddrs; j++) {
addr = &ctx->addrs[j];
if (addr->name.len == 0
&& ngx_cmp_sockaddr(peer->sockaddr, peer->socklen,
addr->sockaddr, addr->socklen, 0)
== NGX_OK)
{
addr->name.len = 1;
goto next;
}
}
*peerp = peer->next;
ngx_http_upstream_zone_remove_peer_locked(peers, peer);
ngx_http_upstream_zone_set_single(uscf);
continue;
next:
peerp = &peer->next;
}
for (i = 0; i < ctx->naddrs; i++) {
addr = &ctx->addrs[i];
if (addr->name.len == 1) {
addr->name.len = 0;
continue;
}
ngx_shmtx_lock(&peers->shpool->mutex);
peer = ngx_http_upstream_zone_copy_peer(peers, NULL);
ngx_shmtx_unlock(&peers->shpool->mutex);
if (peer == NULL) {
ngx_log_error(NGX_LOG_ERR, event->log, 0,
"cannot add new server to upstream \"%V\", "
"memory exhausted", peers->name);
break;
}
ngx_memcpy(peer->sockaddr, addr->sockaddr, addr->socklen);
port = ((struct sockaddr_in *) template->sockaddr)->sin_port;
switch (peer->sockaddr->sa_family) {
#if (NGX_HAVE_INET6)
case AF_INET6:
((struct sockaddr_in6 *) peer->sockaddr)->sin6_port = port;
break;
#endif
default: /* AF_INET */
((struct sockaddr_in *) peer->sockaddr)->sin_port = port;
}
peer->socklen = addr->socklen;
peer->name.len = ngx_sock_ntop(peer->sockaddr, peer->socklen,
peer->name.data, NGX_SOCKADDR_STRLEN, 1);
peer->host = template->host;
peer->server = template->server;
peer->weight = template->weight;
peer->effective_weight = peer->weight;
peer->max_conns = template->max_conns;
peer->max_fails = template->max_fails;
peer->fail_timeout = template->fail_timeout;
peer->down = template->down;
*peerp = peer;
peerp = &peer->next;
peers->number++;
peers->tries += (peer->down == 0);
peers->total_weight += peer->weight;
peers->weighted = (peers->total_weight != peers->number);
(*peers->config)++;
ngx_http_upstream_zone_set_single(uscf);
}
ngx_http_upstream_rr_peers_unlock(peers);
timer = (ngx_msec_t) 1000 * (ctx->valid > now ? ctx->valid - now + 1 : 1);
ngx_resolve_name_done(ctx);
ngx_add_timer(event, timer);
}

78
src/http/ngx_http_upstream.c
View File

@ -1565,6 +1565,26 @@ ngx_http_upstream_connect(ngx_http_request_t *r, ngx_http_upstream_t *u)
u->state->peer = u->peer.name;
#if (NGX_HTTP_UPSTREAM_ZONE)
if (u->upstream && u->upstream->shm_zone
&& (u->upstream->flags & NGX_HTTP_UPSTREAM_MODIFY))
{
u->state->peer = ngx_palloc(r->pool,
sizeof(ngx_str_t) + u->peer.name->len);
if (u->state->peer == NULL) {
ngx_http_upstream_finalize_request(r, u,
NGX_HTTP_INTERNAL_SERVER_ERROR);
return;
}
u->state->peer->len = u->peer.name->len;
u->state->peer->data = (u_char *) (u->state->peer + 1);
ngx_memcpy(u->state->peer->data, u->peer.name->data, u->peer.name->len);
u->peer.name = u->state->peer;
}
#endif
if (rc == NGX_BUSY) {
ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "no live upstreams");
ngx_http_upstream_next(r, u, NGX_HTTP_UPSTREAM_FT_NOLIVE);
@ -6066,6 +6086,7 @@ ngx_http_upstream(ngx_conf_t *cf, ngx_command_t *cmd, void *dummy)
u.no_port = 1;
uscf = ngx_http_upstream_add(cf, &u, NGX_HTTP_UPSTREAM_CREATE
|NGX_HTTP_UPSTREAM_MODIFY
|NGX_HTTP_UPSTREAM_WEIGHT
|NGX_HTTP_UPSTREAM_MAX_CONNS
|NGX_HTTP_UPSTREAM_MAX_FAILS
@ -6171,6 +6192,9 @@ ngx_http_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
ngx_url_t u;
ngx_int_t weight, max_conns, max_fails;
ngx_uint_t i;
#if (NGX_HTTP_UPSTREAM_ZONE)
ngx_uint_t resolve;
#endif
ngx_http_upstream_server_t *us;
us = ngx_array_push(uscf->servers);
@ -6186,6 +6210,9 @@ ngx_http_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
max_conns = 0;
max_fails = 1;
fail_timeout = 10;
#if (NGX_HTTP_UPSTREAM_ZONE)
resolve = 0;
#endif
for (i = 2; i < cf->args->nelts; i++) {
@ -6274,6 +6301,13 @@ ngx_http_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
continue;
}
#if (NGX_HTTP_UPSTREAM_ZONE)
if (ngx_strcmp(value[i].data, "resolve") == 0) {
resolve = 1;
continue;
}
#endif
goto invalid;
}
@ -6282,6 +6316,13 @@ ngx_http_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
u.url = value[1];
u.default_port = 80;
#if (NGX_HTTP_UPSTREAM_ZONE)
if (resolve) {
/* resolve at run time */
u.no_resolve = 1;
}
#endif
if (ngx_parse_url(cf->pool, &u) != NGX_OK) {
if (u.err) {
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
@ -6292,8 +6333,45 @@ ngx_http_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
}
us->name = u.url;
#if (NGX_HTTP_UPSTREAM_ZONE)
if (resolve && u.naddrs == 0) {
ngx_addr_t *addr;
/* save port */
addr = ngx_pcalloc(cf->pool, sizeof(ngx_addr_t));
if (addr == NULL) {
return NGX_CONF_ERROR;
}
addr->sockaddr = ngx_palloc(cf->pool, u.socklen);
if (addr->sockaddr == NULL) {
return NGX_CONF_ERROR;
}
ngx_memcpy(addr->sockaddr, &u.sockaddr, u.socklen);
addr->socklen = u.socklen;
us->addrs = addr;
us->naddrs = 1;
us->host = u.host;
} else {
us->addrs = u.addrs;
us->naddrs = u.naddrs;
}
#else
us->addrs = u.addrs;
us->naddrs = u.naddrs;
#endif
us->weight = weight;
us->max_conns = max_conns;
us->max_fails = max_fails;

9
src/http/ngx_http_upstream.h
View File

@ -104,7 +104,11 @@ typedef struct {
unsigned backup:1;
NGX_COMPAT_BEGIN(6)
#if (NGX_HTTP_UPSTREAM_ZONE)
ngx_str_t host;
#endif
NGX_COMPAT_BEGIN(4)
NGX_COMPAT_END
} ngx_http_upstream_server_t;
@ -115,6 +119,7 @@ typedef struct {
#define NGX_HTTP_UPSTREAM_FAIL_TIMEOUT 0x0008
#define NGX_HTTP_UPSTREAM_DOWN 0x0010
#define NGX_HTTP_UPSTREAM_BACKUP 0x0020
#define NGX_HTTP_UPSTREAM_MODIFY 0x0040
#define NGX_HTTP_UPSTREAM_MAX_CONNS 0x0100
@ -133,6 +138,8 @@ struct ngx_http_upstream_srv_conf_s {
#if (NGX_HTTP_UPSTREAM_ZONE)
ngx_shm_zone_t *shm_zone;
ngx_resolver_t *resolver;
ngx_msec_t resolver_timeout;
#endif
};

197
src/http/ngx_http_upstream_round_robin.c
View File

@ -32,10 +32,15 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
ngx_http_upstream_srv_conf_t *us)
{
ngx_url_t u;
ngx_uint_t i, j, n, w, t;
ngx_uint_t i, j, n, r, w, t;
ngx_http_upstream_server_t *server;
ngx_http_upstream_rr_peer_t *peer, **peerp;
ngx_http_upstream_rr_peers_t *peers, *backup;
#if (NGX_HTTP_UPSTREAM_ZONE)
ngx_uint_t resolve;
ngx_http_core_loc_conf_t *clcf;
ngx_http_upstream_rr_peer_t **rpeerp;
#endif
us->peer.init = ngx_http_upstream_init_round_robin_peer;
@ -43,14 +48,33 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
server = us->servers->elts;
n = 0;
r = 0;
w = 0;
t = 0;
#if (NGX_HTTP_UPSTREAM_ZONE)
resolve = 0;
#endif
for (i = 0; i < us->servers->nelts; i++) {
#if (NGX_HTTP_UPSTREAM_ZONE)
if (server[i].host.len) {
resolve = 1;
}
#endif
if (server[i].backup) {
continue;
}
#if (NGX_HTTP_UPSTREAM_ZONE)
if (server[i].host.len) {
r++;
continue;
}
#endif
n += server[i].naddrs;
w += server[i].naddrs * server[i].weight;
@ -59,7 +83,53 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
}
}
if (n == 0) {
#if (NGX_HTTP_UPSTREAM_ZONE)
if (us->shm_zone) {
if (resolve && !(us->flags & NGX_HTTP_UPSTREAM_MODIFY)) {
ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
"load balancing method does not support"
" resolving names at run time in"
" upstream \"%V\" in %s:%ui",
&us->host, us->file_name, us->line);
return NGX_ERROR;
}
clcf = ngx_http_conf_get_module_loc_conf(cf, ngx_http_core_module);
us->resolver = clcf->resolver;
us->resolver_timeout = clcf->resolver_timeout;
/*
* Without "resolver_timeout" in http{}, the value is unset.
* Even if we set it in ngx_http_core_merge_loc_conf(), it's
* still dependent on the module order and unreliable.
*/
ngx_conf_init_msec_value(us->resolver_timeout, 30000);
if (resolve
&& (us->resolver == NULL
|| us->resolver->connections.nelts == 0))
{
ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
"no resolver defined to resolve names"
" at run time in upstream \"%V\" in %s:%ui",
&us->host, us->file_name, us->line);
return NGX_ERROR;
}
} else if (resolve) {
ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
"resolving names at run time requires"
" upstream \"%V\" in %s:%ui"
" to be in shared memory",
&us->host, us->file_name, us->line);
return NGX_ERROR;
}
#endif
if (n + r == 0) {
ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
"no servers in upstream \"%V\" in %s:%ui",
&us->host, us->file_name, us->line);
@ -71,7 +141,8 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
return NGX_ERROR;
}
peer = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peer_t) * n);
peer = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peer_t)
* (n + r));
if (peer == NULL) {
return NGX_ERROR;
}
@ -86,11 +157,46 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
n = 0;
peerp = &peers->peer;
#if (NGX_HTTP_UPSTREAM_ZONE)
rpeerp = &peers->resolve;
#endif
for (i = 0; i < us->servers->nelts; i++) {
if (server[i].backup) {
continue;
}
#if (NGX_HTTP_UPSTREAM_ZONE)
if (server[i].host.len) {
peer[n].host = ngx_pcalloc(cf->pool,
sizeof(ngx_http_upstream_host_t));
if (peer[n].host == NULL) {
return NGX_ERROR;
}
peer[n].host->name = server[i].host;
peer[n].sockaddr = server[i].addrs[0].sockaddr;
peer[n].socklen = server[i].addrs[0].socklen;
peer[n].name = server[i].addrs[0].name;
peer[n].weight = server[i].weight;
peer[n].effective_weight = server[i].weight;
peer[n].current_weight = 0;
peer[n].max_conns = server[i].max_conns;
peer[n].max_fails = server[i].max_fails;
peer[n].fail_timeout = server[i].fail_timeout;
peer[n].down = server[i].down;
peer[n].server = server[i].name;
*rpeerp = &peer[n];
rpeerp = &peer[n].next;
n++;
continue;
}
#endif
for (j = 0; j < server[i].naddrs; j++) {
peer[n].sockaddr = server[i].addrs[j].sockaddr;
peer[n].socklen = server[i].addrs[j].socklen;
@ -115,6 +221,7 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
/* backup servers */
n = 0;
r = 0;
w = 0;
t = 0;
@ -123,6 +230,13 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
continue;
}
#if (NGX_HTTP_UPSTREAM_ZONE)
if (server[i].host.len) {
r++;
continue;
}
#endif
n += server[i].naddrs;
w += server[i].naddrs * server[i].weight;
@ -131,7 +245,7 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
}
}
if (n == 0) {
if (n + r == 0) {
return NGX_OK;
}
@ -140,12 +254,16 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
return NGX_ERROR;
}
peer = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peer_t) * n);
peer = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peer_t)
* (n + r));
if (peer == NULL) {
return NGX_ERROR;
}
if (n > 0) {
peers->single = 0;
}
backup->single = 0;
backup->number = n;
backup->weighted = (w != n);
@ -156,11 +274,46 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
n = 0;
peerp = &backup->peer;
#if (NGX_HTTP_UPSTREAM_ZONE)
rpeerp = &backup->resolve;
#endif
for (i = 0; i < us->servers->nelts; i++) {
if (!server[i].backup) {
continue;
}
#if (NGX_HTTP_UPSTREAM_ZONE)
if (server[i].host.len) {
peer[n].host = ngx_pcalloc(cf->pool,
sizeof(ngx_http_upstream_host_t));
if (peer[n].host == NULL) {
return NGX_ERROR;
}
peer[n].host->name = server[i].host;
peer[n].sockaddr = server[i].addrs[0].sockaddr;
peer[n].socklen = server[i].addrs[0].socklen;
peer[n].name = server[i].addrs[0].name;
peer[n].weight = server[i].weight;
peer[n].effective_weight = server[i].weight;
peer[n].current_weight = 0;
peer[n].max_conns = server[i].max_conns;
peer[n].max_fails = server[i].max_fails;
peer[n].fail_timeout = server[i].fail_timeout;
peer[n].down = server[i].down;
peer[n].server = server[i].name;
*rpeerp = &peer[n];
rpeerp = &peer[n].next;
n++;
continue;
}
#endif
for (j = 0; j < server[i].naddrs; j++) {
peer[n].sockaddr = server[i].addrs[j].sockaddr;
peer[n].socklen = server[i].addrs[j].socklen;
@ -273,7 +426,12 @@ ngx_http_upstream_init_round_robin_peer(ngx_http_request_t *r,
rrp->peers = us->peer.data;
rrp->current = NULL;
rrp->config = 0;
ngx_http_upstream_rr_peers_rlock(rrp->peers);
#if (NGX_HTTP_UPSTREAM_ZONE)
rrp->config = rrp->peers->config ? *rrp->peers->config : 0;
#endif
n = rrp->peers->number;
@ -281,6 +439,10 @@ ngx_http_upstream_init_round_robin_peer(ngx_http_request_t *r,
n = rrp->peers->next->number;
}
r->upstream->peer.tries = ngx_http_upstream_tries(rrp->peers);
ngx_http_upstream_rr_peers_unlock(rrp->peers);
if (n <= 8 * sizeof(uintptr_t)) {
rrp->tried = &rrp->data;
rrp->data = 0;
@ -296,7 +458,6 @@ ngx_http_upstream_init_round_robin_peer(ngx_http_request_t *r,
r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
r->upstream->peer.tries = ngx_http_upstream_tries(rrp->peers);
#if (NGX_HTTP_SSL)
r->upstream->peer.set_session =
ngx_http_upstream_set_round_robin_peer_session;
@ -446,6 +607,12 @@ ngx_http_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data)
peers = rrp->peers;
ngx_http_upstream_rr_peers_wlock(peers);
#if (NGX_HTTP_UPSTREAM_ZONE)
if (peers->config && rrp->config != *peers->config) {
goto busy;
}
#endif
if (peers->single) {
peer = peers->peer;
@ -458,6 +625,7 @@ ngx_http_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data)
}
rrp->current = peer;
ngx_http_upstream_rr_peer_ref(peers, peer);
} else {
@ -510,6 +678,10 @@ failed:
ngx_http_upstream_rr_peers_wlock(peers);
}
#if (NGX_HTTP_UPSTREAM_ZONE)
busy:
#endif
ngx_http_upstream_rr_peers_unlock(peers);
pc->name = peers->name;
@ -580,6 +752,7 @@ ngx_http_upstream_get_peer(ngx_http_upstream_rr_peer_data_t *rrp)
}
rrp->current = best;
ngx_http_upstream_rr_peer_ref(rrp->peers, best);
n = p / (8 * sizeof(uintptr_t));
m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t));
@ -617,9 +790,16 @@ ngx_http_upstream_free_round_robin_peer(ngx_peer_connection_t *pc, void *data,
if (rrp->peers->single) {
if (peer->fails) {
peer->fails = 0;
}
peer->conns--;
if (ngx_http_upstream_rr_peer_unref(rrp->peers, peer) == NGX_OK) {
ngx_http_upstream_rr_peer_unlock(rrp->peers, peer);
}
ngx_http_upstream_rr_peers_unlock(rrp->peers);
pc->tries = 0;
@ -661,7 +841,10 @@ ngx_http_upstream_free_round_robin_peer(ngx_peer_connection_t *pc, void *data,
peer->conns--;
if (ngx_http_upstream_rr_peer_unref(rrp->peers, peer) == NGX_OK) {
ngx_http_upstream_rr_peer_unlock(rrp->peers, peer);
}
ngx_http_upstream_rr_peers_unlock(rrp->peers);
if (pc->tries) {

86
src/http/ngx_http_upstream_round_robin.h
View File

@ -14,8 +14,23 @@
#include <ngx_http.h>
typedef struct ngx_http_upstream_rr_peers_s ngx_http_upstream_rr_peers_t;
typedef struct ngx_http_upstream_rr_peer_s ngx_http_upstream_rr_peer_t;
#if (NGX_HTTP_UPSTREAM_ZONE)
typedef struct {
ngx_event_t event; /* must be first */
ngx_uint_t worker;
ngx_str_t name;
ngx_http_upstream_rr_peers_t *peers;
ngx_http_upstream_rr_peer_t *peer;
} ngx_http_upstream_host_t;
#endif
struct ngx_http_upstream_rr_peer_s {
struct sockaddr *sockaddr;
socklen_t socklen;
@ -46,24 +61,28 @@ struct ngx_http_upstream_rr_peer_s {
#endif
#if (NGX_HTTP_UPSTREAM_ZONE)
unsigned zombie:1;
ngx_atomic_t lock;
ngx_uint_t refs;
ngx_http_upstream_host_t *host;
#endif
ngx_http_upstream_rr_peer_t *next;
NGX_COMPAT_BEGIN(32)
NGX_COMPAT_BEGIN(15)
NGX_COMPAT_END
};
typedef struct ngx_http_upstream_rr_peers_s ngx_http_upstream_rr_peers_t;
struct ngx_http_upstream_rr_peers_s {
ngx_uint_t number;
#if (NGX_HTTP_UPSTREAM_ZONE)
ngx_slab_pool_t *shpool;
ngx_atomic_t rwlock;
ngx_uint_t *config;
ngx_http_upstream_rr_peer_t *resolve;
ngx_http_upstream_rr_peers_t *zone_next;
#endif
@ -114,6 +133,65 @@ struct ngx_http_upstream_rr_peers_s {
ngx_rwlock_unlock(&peer->lock); \
}
#define ngx_http_upstream_rr_peer_ref(peers, peer) \
(peer)->refs++;
static ngx_inline void
ngx_http_upstream_rr_peer_free_locked(ngx_http_upstream_rr_peers_t *peers,
ngx_http_upstream_rr_peer_t *peer)
{
if (peer->refs) {
peer->zombie = 1;
return;
}
ngx_slab_free_locked(peers->shpool, peer->sockaddr);
ngx_slab_free_locked(peers->shpool, peer->name.data);
if (peer->server.data && (peer->host == NULL || peer->host->peer == peer)) {
ngx_slab_free_locked(peers->shpool, peer->server.data);
}
#if (NGX_HTTP_SSL)
if (peer->ssl_session) {
ngx_slab_free_locked(peers->shpool, peer->ssl_session);
}
#endif
ngx_slab_free_locked(peers->shpool, peer);
}
static ngx_inline void
ngx_http_upstream_rr_peer_free(ngx_http_upstream_rr_peers_t *peers,
ngx_http_upstream_rr_peer_t *peer)
{
ngx_shmtx_lock(&peers->shpool->mutex);
ngx_http_upstream_rr_peer_free_locked(peers, peer);
ngx_shmtx_unlock(&peers->shpool->mutex);
}
static ngx_inline ngx_int_t
ngx_http_upstream_rr_peer_unref(ngx_http_upstream_rr_peers_t *peers,
ngx_http_upstream_rr_peer_t *peer)
{
peer->refs--;
if (peers->shpool == NULL) {
return NGX_OK;
}
if (peer->refs == 0 && peer->zombie) {
ngx_http_upstream_rr_peer_free(peers, peer);
return NGX_DONE;
}
return NGX_OK;
}
#else
#define ngx_http_upstream_rr_peers_rlock(peers)
@ -121,6 +199,8 @@ struct ngx_http_upstream_rr_peers_s {
#define ngx_http_upstream_rr_peers_unlock(peers)
#define ngx_http_upstream_rr_peer_lock(peers, peer)
#define ngx_http_upstream_rr_peer_unlock(peers, peer)
#define ngx_http_upstream_rr_peer_ref(peers, peer)
#define ngx_http_upstream_rr_peer_unref(peers, peer) NGX_OK
#endif

19
src/stream/ngx_stream_proxy_module.c
View File

@ -742,6 +742,25 @@ ngx_stream_proxy_connect(ngx_stream_session_t *s)
u->state->peer = u->peer.name;
#if (NGX_STREAM_UPSTREAM_ZONE)
if (u->upstream && u->upstream->shm_zone
&& (u->upstream->flags & NGX_STREAM_UPSTREAM_MODIFY))
{
u->state->peer = ngx_palloc(s->connection->pool,
sizeof(ngx_str_t) + u->peer.name->len);
if (u->state->peer == NULL) {
ngx_stream_proxy_finalize(s, NGX_STREAM_INTERNAL_SERVER_ERROR);
return;
}
u->state->peer->len = u->peer.name->len;
u->state->peer->data = (u_char *) (u->state->peer + 1);
ngx_memcpy(u->state->peer->data, u->peer.name->data, u->peer.name->len);
u->peer.name = u->state->peer;
}
#endif
if (rc == NGX_BUSY) {
ngx_log_error(NGX_LOG_ERR, c->log, 0, "no live upstreams");
ngx_stream_proxy_finalize(s, NGX_STREAM_BAD_GATEWAY);

58
src/stream/ngx_stream_upstream.c
View File

@ -319,6 +319,7 @@ ngx_stream_upstream(ngx_conf_t *cf, ngx_command_t *cmd, void *dummy)
u.no_port = 1;
uscf = ngx_stream_upstream_add(cf, &u, NGX_STREAM_UPSTREAM_CREATE
|NGX_STREAM_UPSTREAM_MODIFY
|NGX_STREAM_UPSTREAM_WEIGHT
|NGX_STREAM_UPSTREAM_MAX_CONNS
|NGX_STREAM_UPSTREAM_MAX_FAILS
@ -408,6 +409,9 @@ ngx_stream_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
ngx_url_t u;
ngx_int_t weight, max_conns, max_fails;
ngx_uint_t i;
#if (NGX_STREAM_UPSTREAM_ZONE)
ngx_uint_t resolve;
#endif
ngx_stream_upstream_server_t *us;
us = ngx_array_push(uscf->servers);
@ -423,6 +427,9 @@ ngx_stream_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
max_conns = 0;
max_fails = 1;
fail_timeout = 10;
#if (NGX_STREAM_UPSTREAM_ZONE)
resolve = 0;
#endif
for (i = 2; i < cf->args->nelts; i++) {
@ -511,6 +518,13 @@ ngx_stream_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
continue;
}
#if (NGX_STREAM_UPSTREAM_ZONE)
if (ngx_strcmp(value[i].data, "resolve") == 0) {
resolve = 1;
continue;
}
#endif
goto invalid;
}
@ -518,6 +532,13 @@ ngx_stream_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
u.url = value[1];
#if (NGX_STREAM_UPSTREAM_ZONE)
if (resolve) {
/* resolve at run time */
u.no_resolve = 1;
}
#endif
if (ngx_parse_url(cf->pool, &u) != NGX_OK) {
if (u.err) {
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
@ -534,8 +555,45 @@ ngx_stream_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
}
us->name = u.url;
#if (NGX_STREAM_UPSTREAM_ZONE)
if (resolve && u.naddrs == 0) {
ngx_addr_t *addr;
/* save port */
addr = ngx_pcalloc(cf->pool, sizeof(ngx_addr_t));
if (addr == NULL) {
return NGX_CONF_ERROR;
}
addr->sockaddr = ngx_palloc(cf->pool, u.socklen);
if (addr->sockaddr == NULL) {
return NGX_CONF_ERROR;
}
ngx_memcpy(addr->sockaddr, &u.sockaddr, u.socklen);
addr->socklen = u.socklen;
us->addrs = addr;
us->naddrs = 1;
us->host = u.host;
} else {
us->addrs = u.addrs;
us->naddrs = u.naddrs;
}
#else
us->addrs = u.addrs;
us->naddrs = u.naddrs;
#endif
us->weight = weight;
us->max_conns = max_conns;
us->max_fails = max_fails;

9
src/stream/ngx_stream_upstream.h
View File

@ -21,6 +21,7 @@
#define NGX_STREAM_UPSTREAM_FAIL_TIMEOUT 0x0008
#define NGX_STREAM_UPSTREAM_DOWN 0x0010
#define NGX_STREAM_UPSTREAM_BACKUP 0x0020
#define NGX_STREAM_UPSTREAM_MODIFY 0x0040
#define NGX_STREAM_UPSTREAM_MAX_CONNS 0x0100
@ -62,7 +63,11 @@ typedef struct {
unsigned backup:1;
NGX_COMPAT_BEGIN(4)
#if (NGX_STREAM_UPSTREAM_ZONE)
ngx_str_t host;
#endif
NGX_COMPAT_BEGIN(2)
NGX_COMPAT_END
} ngx_stream_upstream_server_t;
@ -83,6 +88,8 @@ struct ngx_stream_upstream_srv_conf_s {
#if (NGX_STREAM_UPSTREAM_ZONE)
ngx_shm_zone_t *shm_zone;
ngx_resolver_t *resolver;
ngx_msec_t resolver_timeout;
#endif
};

88
src/stream/ngx_stream_upstream_hash_module.c
View File

@ -23,6 +23,9 @@ typedef struct {
typedef struct {
#if (NGX_STREAM_UPSTREAM_ZONE)
ngx_uint_t config;
#endif
ngx_stream_complex_value_t key;
ngx_stream_upstream_chash_points_t *points;
} ngx_stream_upstream_hash_srv_conf_t;
@ -49,6 +52,8 @@ static ngx_int_t ngx_stream_upstream_get_hash_peer(ngx_peer_connection_t *pc,
static ngx_int_t ngx_stream_upstream_init_chash(ngx_conf_t *cf,
ngx_stream_upstream_srv_conf_t *us);
static ngx_int_t ngx_stream_upstream_update_chash(ngx_pool_t *pool,
ngx_stream_upstream_srv_conf_t *us);
static int ngx_libc_cdecl
ngx_stream_upstream_chash_cmp_points(const void *one, const void *two);
static ngx_uint_t ngx_stream_upstream_find_chash_point(
@ -178,11 +183,18 @@ ngx_stream_upstream_get_hash_peer(ngx_peer_connection_t *pc, void *data)
ngx_stream_upstream_rr_peers_rlock(hp->rrp.peers);
if (hp->tries > 20 || hp->rrp.peers->single || hp->key.len == 0) {
if (hp->tries > 20 || hp->rrp.peers->number < 2 || hp->key.len == 0) {
ngx_stream_upstream_rr_peers_unlock(hp->rrp.peers);
return hp->get_rr_peer(pc, &hp->rrp);
}
#if (NGX_STREAM_UPSTREAM_ZONE)
if (hp->rrp.peers->config && hp->rrp.config != *hp->rrp.peers->config) {
ngx_stream_upstream_rr_peers_unlock(hp->rrp.peers);
return hp->get_rr_peer(pc, &hp->rrp);
}
#endif
now = ngx_time();
pc->connection = NULL;
@ -261,6 +273,7 @@ ngx_stream_upstream_get_hash_peer(ngx_peer_connection_t *pc, void *data)
}
hp->rrp.current = peer;
ngx_stream_upstream_rr_peer_ref(hp->rrp.peers, peer);
pc->sockaddr = peer->sockaddr;
pc->socklen = peer->socklen;
@ -284,6 +297,26 @@ ngx_stream_upstream_get_hash_peer(ngx_peer_connection_t *pc, void *data)
static ngx_int_t
ngx_stream_upstream_init_chash(ngx_conf_t *cf,
ngx_stream_upstream_srv_conf_t *us)
{
if (ngx_stream_upstream_init_round_robin(cf, us) != NGX_OK) {
return NGX_ERROR;
}
us->peer.init = ngx_stream_upstream_init_chash_peer;
#if (NGX_STREAM_UPSTREAM_ZONE)
if (us->shm_zone) {
return NGX_OK;
}
#endif
return ngx_stream_upstream_update_chash(cf->pool, us);
}
static ngx_int_t
ngx_stream_upstream_update_chash(ngx_pool_t *pool,
ngx_stream_upstream_srv_conf_t *us)
{
u_char *host, *port, c;
size_t host_len, port_len, size;
@ -299,25 +332,33 @@ ngx_stream_upstream_init_chash(ngx_conf_t *cf,
u_char byte[4];
} prev_hash;
if (ngx_stream_upstream_init_round_robin(cf, us) != NGX_OK) {
return NGX_ERROR;
}
hcf = ngx_stream_conf_upstream_srv_conf(us,
ngx_stream_upstream_hash_module);
us->peer.init = ngx_stream_upstream_init_chash_peer;
if (hcf->points) {
ngx_free(hcf->points);
hcf->points = NULL;
}
peers = us->peer.data;
npoints = peers->total_weight * 160;
size = sizeof(ngx_stream_upstream_chash_points_t)
+ sizeof(ngx_stream_upstream_chash_point_t) * (npoints - 1);
- sizeof(ngx_stream_upstream_chash_point_t)
+ sizeof(ngx_stream_upstream_chash_point_t) * npoints;
points = ngx_palloc(cf->pool, size);
points = pool ? ngx_palloc(pool, size) : ngx_alloc(size, ngx_cycle->log);
if (points == NULL) {
return NGX_ERROR;
}
points->number = 0;
if (npoints == 0) {
hcf->points = points;
return NGX_OK;
}
for (peer = peers->peer; peer; peer = peer->next) {
server = &peer->server;
@ -401,8 +442,6 @@ ngx_stream_upstream_init_chash(ngx_conf_t *cf,
points->number = i + 1;
hcf = ngx_stream_conf_upstream_srv_conf(us,
ngx_stream_upstream_hash_module);
hcf->points = points;
return NGX_OK;
@ -483,7 +522,22 @@ ngx_stream_upstream_init_chash_peer(ngx_stream_session_t *s,
ngx_stream_upstream_rr_peers_rlock(hp->rrp.peers);
#if (NGX_STREAM_UPSTREAM_ZONE)
if (hp->rrp.peers->config
&& (hcf->points == NULL || hcf->config != *hp->rrp.peers->config))
{
if (ngx_stream_upstream_update_chash(NULL, us) != NGX_OK) {
ngx_stream_upstream_rr_peers_unlock(hp->rrp.peers);
return NGX_ERROR;
}
hcf->config = *hp->rrp.peers->config;
}
#endif
if (hcf->points->number) {
hp->hash = ngx_stream_upstream_find_chash_point(hcf->points, hash);
}
ngx_stream_upstream_rr_peers_unlock(hp->rrp.peers);
@ -518,6 +572,20 @@ ngx_stream_upstream_get_chash_peer(ngx_peer_connection_t *pc, void *data)
pc->connection = NULL;
if (hp->rrp.peers->number == 0) {
pc->name = hp->rrp.peers->name;
ngx_stream_upstream_rr_peers_unlock(hp->rrp.peers);
return NGX_BUSY;
}
#if (NGX_STREAM_UPSTREAM_ZONE)
if (hp->rrp.peers->config && hp->rrp.config != *hp->rrp.peers->config) {
pc->name = hp->rrp.peers->name;
ngx_stream_upstream_rr_peers_unlock(hp->rrp.peers);
return NGX_BUSY;
}
#endif
now = ngx_time();
hcf = hp->conf;
@ -596,6 +664,7 @@ ngx_stream_upstream_get_chash_peer(ngx_peer_connection_t *pc, void *data)
}
hp->rrp.current = best;
ngx_stream_upstream_rr_peer_ref(hp->rrp.peers, best);
pc->sockaddr = best->sockaddr;
pc->socklen = best->socklen;
@ -663,6 +732,7 @@ ngx_stream_upstream_hash(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
}
uscf->flags = NGX_STREAM_UPSTREAM_CREATE
|NGX_STREAM_UPSTREAM_MODIFY
|NGX_STREAM_UPSTREAM_WEIGHT
|NGX_STREAM_UPSTREAM_MAX_CONNS
|NGX_STREAM_UPSTREAM_MAX_FAILS

12
src/stream/ngx_stream_upstream_least_conn_module.c
View File

@ -120,6 +120,12 @@ ngx_stream_upstream_get_least_conn_peer(ngx_peer_connection_t *pc, void *data)
ngx_stream_upstream_rr_peers_wlock(peers);
#if (NGX_STREAM_UPSTREAM_ZONE)
if (peers->config && rrp->config != *peers->config) {
goto busy;
}
#endif
best = NULL;
total = 0;
@ -240,6 +246,7 @@ ngx_stream_upstream_get_least_conn_peer(ngx_peer_connection_t *pc, void *data)
best->conns++;
rrp->current = best;
ngx_stream_upstream_rr_peer_ref(peers, best);
n = p / (8 * sizeof(uintptr_t));
m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t));
@ -276,6 +283,10 @@ failed:
ngx_stream_upstream_rr_peers_wlock(peers);
}
#if (NGX_STREAM_UPSTREAM_ZONE)
busy:
#endif
ngx_stream_upstream_rr_peers_unlock(peers);
pc->name = peers->name;
@ -299,6 +310,7 @@ ngx_stream_upstream_least_conn(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
uscf->peer.init_upstream = ngx_stream_upstream_init_least_conn;
uscf->flags = NGX_STREAM_UPSTREAM_CREATE
|NGX_STREAM_UPSTREAM_MODIFY
|NGX_STREAM_UPSTREAM_WEIGHT
|NGX_STREAM_UPSTREAM_MAX_CONNS
|NGX_STREAM_UPSTREAM_MAX_FAILS

35
src/stream/ngx_stream_upstream_random_module.c
View File

@ -17,6 +17,9 @@ typedef struct {
typedef struct {
ngx_uint_t two;
#if (NGX_STREAM_UPSTREAM_ZONE)
ngx_uint_t config;
#endif
ngx_stream_upstream_random_range_t *ranges;
} ngx_stream_upstream_random_srv_conf_t;
@ -125,6 +128,11 @@ ngx_stream_upstream_update_random(ngx_pool_t *pool,
rcf = ngx_stream_conf_upstream_srv_conf(us,
ngx_stream_upstream_random_module);
if (rcf->ranges) {
ngx_free(rcf->ranges);
rcf->ranges = NULL;
}
peers = us->peer.data;
size = peers->number * sizeof(ngx_stream_upstream_random_range_t);
@ -186,11 +194,15 @@ ngx_stream_upstream_init_random_peer(ngx_stream_session_t *s,
ngx_stream_upstream_rr_peers_rlock(rp->rrp.peers);
#if (NGX_STREAM_UPSTREAM_ZONE)
if (rp->rrp.peers->shpool && rcf->ranges == NULL) {
if (rp->rrp.peers->config
&& (rcf->ranges == NULL || rcf->config != *rp->rrp.peers->config))
{
if (ngx_stream_upstream_update_random(NULL, us) != NGX_OK) {
ngx_stream_upstream_rr_peers_unlock(rp->rrp.peers);
return NGX_ERROR;
}
rcf->config = *rp->rrp.peers->config;
}
#endif
@ -220,11 +232,18 @@ ngx_stream_upstream_get_random_peer(ngx_peer_connection_t *pc, void *data)
ngx_stream_upstream_rr_peers_rlock(peers);
if (rp->tries > 20 || peers->single) {
if (rp->tries > 20 || peers->number < 2) {
ngx_stream_upstream_rr_peers_unlock(peers);
return ngx_stream_upstream_get_round_robin_peer(pc, rrp);
}
#if (NGX_STREAM_UPSTREAM_ZONE)
if (peers->config && rrp->config != *peers->config) {
ngx_stream_upstream_rr_peers_unlock(peers);
return ngx_stream_upstream_get_round_robin_peer(pc, rrp);
}
#endif
pc->cached = 0;
pc->connection = NULL;
@ -274,6 +293,7 @@ ngx_stream_upstream_get_random_peer(ngx_peer_connection_t *pc, void *data)
}
rrp->current = peer;
ngx_stream_upstream_rr_peer_ref(peers, peer);
if (now - peer->checked > peer->fail_timeout) {
peer->checked = now;
@ -314,11 +334,18 @@ ngx_stream_upstream_get_random2_peer(ngx_peer_connection_t *pc, void *data)
ngx_stream_upstream_rr_peers_wlock(peers);
if (rp->tries > 20 || peers->single) {
if (rp->tries > 20 || peers->number < 2) {
ngx_stream_upstream_rr_peers_unlock(peers);
return ngx_stream_upstream_get_round_robin_peer(pc, rrp);
}
#if (NGX_STREAM_UPSTREAM_ZONE)
if (peers->config && rrp->config != *peers->config) {
ngx_stream_upstream_rr_peers_unlock(peers);
return ngx_stream_upstream_get_round_robin_peer(pc, rrp);
}
#endif
pc->cached = 0;
pc->connection = NULL;
@ -384,6 +411,7 @@ ngx_stream_upstream_get_random2_peer(ngx_peer_connection_t *pc, void *data)
}
rrp->current = peer;
ngx_stream_upstream_rr_peer_ref(peers, peer);
if (now - peer->checked > peer->fail_timeout) {
peer->checked = now;
@ -467,6 +495,7 @@ ngx_stream_upstream_random(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
uscf->peer.init_upstream = ngx_stream_upstream_init_random;
uscf->flags = NGX_STREAM_UPSTREAM_CREATE
|NGX_STREAM_UPSTREAM_MODIFY
|NGX_STREAM_UPSTREAM_WEIGHT
|NGX_STREAM_UPSTREAM_MAX_CONNS
|NGX_STREAM_UPSTREAM_MAX_FAILS

198
src/stream/ngx_stream_upstream_round_robin.c
View File

@ -38,10 +38,15 @@ ngx_stream_upstream_init_round_robin(ngx_conf_t *cf,
ngx_stream_upstream_srv_conf_t *us)
{
ngx_url_t u;
ngx_uint_t i, j, n, w, t;
ngx_uint_t i, j, n, r, w, t;
ngx_stream_upstream_server_t *server;
ngx_stream_upstream_rr_peer_t *peer, **peerp;
ngx_stream_upstream_rr_peers_t *peers, *backup;
#if (NGX_STREAM_UPSTREAM_ZONE)
ngx_uint_t resolve;
ngx_stream_core_srv_conf_t *cscf;
ngx_stream_upstream_rr_peer_t **rpeerp;
#endif
us->peer.init = ngx_stream_upstream_init_round_robin_peer;
@ -49,14 +54,33 @@ ngx_stream_upstream_init_round_robin(ngx_conf_t *cf,
server = us->servers->elts;
n = 0;
r = 0;
w = 0;
t = 0;
#if (NGX_STREAM_UPSTREAM_ZONE)
resolve = 0;
#endif
for (i = 0; i < us->servers->nelts; i++) {
#if (NGX_STREAM_UPSTREAM_ZONE)
if (server[i].host.len) {
resolve = 1;
}
#endif
if (server[i].backup) {
continue;
}
#if (NGX_STREAM_UPSTREAM_ZONE)
if (server[i].host.len) {
r++;
continue;
}
#endif
n += server[i].naddrs;
w += server[i].naddrs * server[i].weight;
@ -65,7 +89,54 @@ ngx_stream_upstream_init_round_robin(ngx_conf_t *cf,
}
}
if (n == 0) {
#if (NGX_STREAM_UPSTREAM_ZONE)
if (us->shm_zone) {
if (resolve && !(us->flags & NGX_STREAM_UPSTREAM_MODIFY)) {
ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
"load balancing method does not support"
" resolving names at run time in"
" upstream \"%V\" in %s:%ui",
&us->host, us->file_name, us->line);
return NGX_ERROR;
}
cscf = ngx_stream_conf_get_module_srv_conf(cf,
ngx_stream_core_module);
us->resolver = cscf->resolver;
us->resolver_timeout = cscf->resolver_timeout;
/*
* Without "resolver_timeout" in stream{}, the value is unset.
* Even if we set it in ngx_stream_core_merge_srv_conf(), it's
* still dependent on the module order and unreliable.
*/
ngx_conf_init_msec_value(us->resolver_timeout, 30000);
if (resolve
&& (us->resolver == NULL
|| us->resolver->connections.nelts == 0))
{
ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
"no resolver defined to resolve names"
" at run time in upstream \"%V\" in %s:%ui",
&us->host, us->file_name, us->line);
return NGX_ERROR;
}
} else if (resolve) {
ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
"resolving names at run time requires"
" upstream \"%V\" in %s:%ui"
" to be in shared memory",
&us->host, us->file_name, us->line);
return NGX_ERROR;
}
#endif
if (n + r == 0) {
ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
"no servers in upstream \"%V\" in %s:%ui",
&us->host, us->file_name, us->line);
@ -77,7 +148,8 @@ ngx_stream_upstream_init_round_robin(ngx_conf_t *cf,
return NGX_ERROR;
}
peer = ngx_pcalloc(cf->pool, sizeof(ngx_stream_upstream_rr_peer_t) * n);
peer = ngx_pcalloc(cf->pool, sizeof(ngx_stream_upstream_rr_peer_t)
* (n + r));
if (peer == NULL) {
return NGX_ERROR;
}
@ -92,11 +164,45 @@ ngx_stream_upstream_init_round_robin(ngx_conf_t *cf,
n = 0;
peerp = &peers->peer;
#if (NGX_STREAM_UPSTREAM_ZONE)
rpeerp = &peers->resolve;
#endif
for (i = 0; i < us->servers->nelts; i++) {
if (server[i].backup) {
continue;
}
#if (NGX_STREAM_UPSTREAM_ZONE)
if (server[i].host.len) {
peer[n].host = ngx_pcalloc(cf->pool,
sizeof(ngx_stream_upstream_host_t));
if (peer[n].host == NULL) {
return NGX_ERROR;
}
peer[n].host->name = server[i].host;
peer[n].sockaddr = server[i].addrs[0].sockaddr;
peer[n].socklen = server[i].addrs[0].socklen;
peer[n].name = server[i].addrs[0].name;
peer[n].weight = server[i].weight;
peer[n].effective_weight = server[i].weight;
peer[n].current_weight = 0;
peer[n].max_conns = server[i].max_conns;
peer[n].max_fails = server[i].max_fails;
peer[n].fail_timeout = server[i].fail_timeout;
peer[n].down = server[i].down;
peer[n].server = server[i].name;
*rpeerp = &peer[n];
rpeerp = &peer[n].next;
n++;
continue;
}
#endif
for (j = 0; j < server[i].naddrs; j++) {
peer[n].sockaddr = server[i].addrs[j].sockaddr;
peer[n].socklen = server[i].addrs[j].socklen;
@ -121,6 +227,7 @@ ngx_stream_upstream_init_round_robin(ngx_conf_t *cf,
/* backup servers */
n = 0;
r = 0;
w = 0;
t = 0;
@ -129,6 +236,13 @@ ngx_stream_upstream_init_round_robin(ngx_conf_t *cf,
continue;
}
#if (NGX_STREAM_UPSTREAM_ZONE)
if (server[i].host.len) {
r++;
continue;
}
#endif
n += server[i].naddrs;
w += server[i].naddrs * server[i].weight;
@ -137,7 +251,7 @@ ngx_stream_upstream_init_round_robin(ngx_conf_t *cf,
}
}
if (n == 0) {
if (n + r == 0) {
return NGX_OK;
}
@ -146,12 +260,16 @@ ngx_stream_upstream_init_round_robin(ngx_conf_t *cf,
return NGX_ERROR;
}
peer = ngx_pcalloc(cf->pool, sizeof(ngx_stream_upstream_rr_peer_t) * n);
peer = ngx_pcalloc(cf->pool, sizeof(ngx_stream_upstream_rr_peer_t)
* (n + r));
if (peer == NULL) {
return NGX_ERROR;
}
if (n > 0) {
peers->single = 0;
}
backup->single = 0;
backup->number = n;
backup->weighted = (w != n);
@ -162,11 +280,45 @@ ngx_stream_upstream_init_round_robin(ngx_conf_t *cf,
n = 0;
peerp = &backup->peer;
#if (NGX_STREAM_UPSTREAM_ZONE)
rpeerp = &backup->resolve;
#endif
for (i = 0; i < us->servers->nelts; i++) {
if (!server[i].backup) {
continue;
}
#if (NGX_STREAM_UPSTREAM_ZONE)
if (server[i].host.len) {
peer[n].host = ngx_pcalloc(cf->pool,
sizeof(ngx_stream_upstream_host_t));
if (peer[n].host == NULL) {
return NGX_ERROR;
}
peer[n].host->name = server[i].host;
peer[n].sockaddr = server[i].addrs[0].sockaddr;
peer[n].socklen = server[i].addrs[0].socklen;
peer[n].name = server[i].addrs[0].name;
peer[n].weight = server[i].weight;
peer[n].effective_weight = server[i].weight;
peer[n].current_weight = 0;
peer[n].max_conns = server[i].max_conns;
peer[n].max_fails = server[i].max_fails;
peer[n].fail_timeout = server[i].fail_timeout;
peer[n].down = server[i].down;
peer[n].server = server[i].name;
*rpeerp = &peer[n];
rpeerp = &peer[n].next;
n++;
continue;
}
#endif
for (j = 0; j < server[i].naddrs; j++) {
peer[n].sockaddr = server[i].addrs[j].sockaddr;
peer[n].socklen = server[i].addrs[j].socklen;
@ -280,7 +432,12 @@ ngx_stream_upstream_init_round_robin_peer(ngx_stream_session_t *s,
rrp->peers = us->peer.data;
rrp->current = NULL;
rrp->config = 0;
ngx_stream_upstream_rr_peers_rlock(rrp->peers);
#if (NGX_STREAM_UPSTREAM_ZONE)
rrp->config = rrp->peers->config ? *rrp->peers->config : 0;
#endif
n = rrp->peers->number;
@ -288,6 +445,10 @@ ngx_stream_upstream_init_round_robin_peer(ngx_stream_session_t *s,
n = rrp->peers->next->number;
}
s->upstream->peer.tries = ngx_stream_upstream_tries(rrp->peers);
ngx_stream_upstream_rr_peers_unlock(rrp->peers);
if (n <= 8 * sizeof(uintptr_t)) {
rrp->tried = &rrp->data;
rrp->data = 0;
@ -304,7 +465,7 @@ ngx_stream_upstream_init_round_robin_peer(ngx_stream_session_t *s,
s->upstream->peer.get = ngx_stream_upstream_get_round_robin_peer;
s->upstream->peer.free = ngx_stream_upstream_free_round_robin_peer;
s->upstream->peer.notify = ngx_stream_upstream_notify_round_robin_peer;
s->upstream->peer.tries = ngx_stream_upstream_tries(rrp->peers);
#if (NGX_STREAM_SSL)
s->upstream->peer.set_session =
ngx_stream_upstream_set_round_robin_peer_session;
@ -455,6 +616,12 @@ ngx_stream_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data)
peers = rrp->peers;
ngx_stream_upstream_rr_peers_wlock(peers);
#if (NGX_STREAM_UPSTREAM_ZONE)
if (peers->config && rrp->config != *peers->config) {
goto busy;
}
#endif
if (peers->single) {
peer = peers->peer;
@ -467,6 +634,7 @@ ngx_stream_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data)
}
rrp->current = peer;
ngx_stream_upstream_rr_peer_ref(peers, peer);
} else {
@ -519,6 +687,10 @@ failed:
ngx_stream_upstream_rr_peers_wlock(peers);
}
#if (NGX_STREAM_UPSTREAM_ZONE)
busy:
#endif
ngx_stream_upstream_rr_peers_unlock(peers);
pc->name = peers->name;
@ -589,6 +761,7 @@ ngx_stream_upstream_get_peer(ngx_stream_upstream_rr_peer_data_t *rrp)
}
rrp->current = best;
ngx_stream_upstream_rr_peer_ref(rrp->peers, best);
n = p / (8 * sizeof(uintptr_t));
m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t));
@ -623,9 +796,17 @@ ngx_stream_upstream_free_round_robin_peer(ngx_peer_connection_t *pc, void *data,
ngx_stream_upstream_rr_peer_lock(rrp->peers, peer);
if (rrp->peers->single) {
if (peer->fails) {
peer->fails = 0;
}
peer->conns--;
if (ngx_stream_upstream_rr_peer_unref(rrp->peers, peer) == NGX_OK) {
ngx_stream_upstream_rr_peer_unlock(rrp->peers, peer);
}
ngx_stream_upstream_rr_peers_unlock(rrp->peers);
pc->tries = 0;
@ -667,7 +848,10 @@ ngx_stream_upstream_free_round_robin_peer(ngx_peer_connection_t *pc, void *data,
peer->conns--;
if (ngx_stream_upstream_rr_peer_unref(rrp->peers, peer) == NGX_OK) {
ngx_stream_upstream_rr_peer_unlock(rrp->peers, peer);
}
ngx_stream_upstream_rr_peers_unlock(rrp->peers);
if (pc->tries) {

86
src/stream/ngx_stream_upstream_round_robin.h
View File

@ -14,8 +14,23 @@
#include <ngx_stream.h>
typedef struct ngx_stream_upstream_rr_peers_s ngx_stream_upstream_rr_peers_t;
typedef struct ngx_stream_upstream_rr_peer_s ngx_stream_upstream_rr_peer_t;
#if (NGX_STREAM_UPSTREAM_ZONE)
typedef struct {
ngx_event_t event; /* must be first */
ngx_uint_t worker;
ngx_str_t name;
ngx_stream_upstream_rr_peers_t *peers;
ngx_stream_upstream_rr_peer_t *peer;
} ngx_stream_upstream_host_t;
#endif
struct ngx_stream_upstream_rr_peer_s {
struct sockaddr *sockaddr;
socklen_t socklen;
@ -44,24 +59,28 @@ struct ngx_stream_upstream_rr_peer_s {
int ssl_session_len;
#if (NGX_STREAM_UPSTREAM_ZONE)
unsigned zombie:1;
ngx_atomic_t lock;
ngx_uint_t refs;
ngx_stream_upstream_host_t *host;
#endif
ngx_stream_upstream_rr_peer_t *next;
NGX_COMPAT_BEGIN(25)
NGX_COMPAT_BEGIN(14)
NGX_COMPAT_END
};
typedef struct ngx_stream_upstream_rr_peers_s ngx_stream_upstream_rr_peers_t;
struct ngx_stream_upstream_rr_peers_s {
ngx_uint_t number;
#if (NGX_STREAM_UPSTREAM_ZONE)
ngx_slab_pool_t *shpool;
ngx_atomic_t rwlock;
ngx_uint_t *config;
ngx_stream_upstream_rr_peer_t *resolve;
ngx_stream_upstream_rr_peers_t *zone_next;
#endif
@ -112,6 +131,65 @@ struct ngx_stream_upstream_rr_peers_s {
ngx_rwlock_unlock(&peer->lock); \
}
#define ngx_stream_upstream_rr_peer_ref(peers, peer) \
(peer)->refs++;
static ngx_inline void
ngx_stream_upstream_rr_peer_free_locked(ngx_stream_upstream_rr_peers_t *peers,
ngx_stream_upstream_rr_peer_t *peer)
{
if (peer->refs) {
peer->zombie = 1;
return;
}
ngx_slab_free_locked(peers->shpool, peer->sockaddr);
ngx_slab_free_locked(peers->shpool, peer->name.data);
if (peer->server.data && (peer->host == NULL || peer->host->peer == peer)) {
ngx_slab_free_locked(peers->shpool, peer->server.data);
}
#if (NGX_STREAM_SSL)
if (peer->ssl_session) {
ngx_slab_free_locked(peers->shpool, peer->ssl_session);
}
#endif
ngx_slab_free_locked(peers->shpool, peer);
}
static ngx_inline void
ngx_stream_upstream_rr_peer_free(ngx_stream_upstream_rr_peers_t *peers,
ngx_stream_upstream_rr_peer_t *peer)
{
ngx_shmtx_lock(&peers->shpool->mutex);
ngx_stream_upstream_rr_peer_free_locked(peers, peer);
ngx_shmtx_unlock(&peers->shpool->mutex);
}
static ngx_inline ngx_int_t
ngx_stream_upstream_rr_peer_unref(ngx_stream_upstream_rr_peers_t *peers,
ngx_stream_upstream_rr_peer_t *peer)
{
peer->refs--;
if (peers->shpool == NULL) {
return NGX_OK;
}
if (peer->refs == 0 && peer->zombie) {
ngx_stream_upstream_rr_peer_free(peers, peer);
return NGX_DONE;
}
return NGX_OK;
}
#else
#define ngx_stream_upstream_rr_peers_rlock(peers)
@ -119,6 +197,8 @@ struct ngx_stream_upstream_rr_peers_s {
#define ngx_stream_upstream_rr_peers_unlock(peers)
#define ngx_stream_upstream_rr_peer_lock(peers, peer)
#define ngx_stream_upstream_rr_peer_unlock(peers, peer)
#define ngx_stream_upstream_rr_peer_ref(peers, peer)
#define ngx_stream_upstream_rr_peer_unref(peers, peer) NGX_OK
#endif

362
src/stream/ngx_stream_upstream_zone_module.c
View File

@ -18,6 +18,13 @@ static ngx_stream_upstream_rr_peers_t *ngx_stream_upstream_zone_copy_peers(
ngx_slab_pool_t *shpool, ngx_stream_upstream_srv_conf_t *uscf);
static ngx_stream_upstream_rr_peer_t *ngx_stream_upstream_zone_copy_peer(
ngx_stream_upstream_rr_peers_t *peers, ngx_stream_upstream_rr_peer_t *src);
static void ngx_stream_upstream_zone_set_single(
ngx_stream_upstream_srv_conf_t *uscf);
static void ngx_stream_upstream_zone_remove_peer_locked(
ngx_stream_upstream_rr_peers_t *peers, ngx_stream_upstream_rr_peer_t *peer);
static ngx_int_t ngx_stream_upstream_zone_init_worker(ngx_cycle_t *cycle);
static void ngx_stream_upstream_zone_resolve_timer(ngx_event_t *event);
static void ngx_stream_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx);
static ngx_command_t ngx_stream_upstream_zone_commands[] = {
@ -52,7 +59,7 @@ ngx_module_t ngx_stream_upstream_zone_module = {
NGX_STREAM_MODULE, /* module type */
NULL, /* init master */
NULL, /* init module */
NULL, /* init process */
ngx_stream_upstream_zone_init_worker, /* init process */
NULL, /* init thread */
NULL, /* exit thread */
NULL, /* exit process */
@ -185,9 +192,15 @@ ngx_stream_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,
ngx_stream_upstream_srv_conf_t *uscf)
{
ngx_str_t *name;
ngx_uint_t *config;
ngx_stream_upstream_rr_peer_t *peer, **peerp;
ngx_stream_upstream_rr_peers_t *peers, *backup;
config = ngx_slab_calloc(shpool, sizeof(ngx_uint_t));
if (config == NULL) {
return NULL;
}
peers = ngx_slab_alloc(shpool, sizeof(ngx_stream_upstream_rr_peers_t));
if (peers == NULL) {
return NULL;
@ -211,6 +224,7 @@ ngx_stream_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,
peers->name = name;
peers->shpool = shpool;
peers->config = config;
for (peerp = &peers->peer; *peerp; peerp = &peer->next) {
/* pool is unlocked */
@ -220,6 +234,17 @@ ngx_stream_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,
}
*peerp = peer;
(*peers->config)++;
}
for (peerp = &peers->resolve; *peerp; peerp = &peer->next) {
peer = ngx_stream_upstream_zone_copy_peer(peers, *peerp);
if (peer == NULL) {
return NULL;
}
*peerp = peer;
(*peers->config)++;
}
if (peers->next == NULL) {
@ -236,6 +261,7 @@ ngx_stream_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,
backup->name = name;
backup->shpool = shpool;
backup->config = config;
for (peerp = &backup->peer; *peerp; peerp = &peer->next) {
/* pool is unlocked */
@ -245,6 +271,17 @@ ngx_stream_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,
}
*peerp = peer;
(*backup->config)++;
}
for (peerp = &backup->resolve; *peerp; peerp = &peer->next) {
peer = ngx_stream_upstream_zone_copy_peer(backup, *peerp);
if (peer == NULL) {
return NULL;
}
*peerp = peer;
(*backup->config)++;
}
peers->next = backup;
@ -276,6 +313,7 @@ ngx_stream_upstream_zone_copy_peer(ngx_stream_upstream_rr_peers_t *peers,
dst->sockaddr = NULL;
dst->name.data = NULL;
dst->server.data = NULL;
dst->host = NULL;
}
dst->sockaddr = ngx_slab_calloc_locked(pool, sizeof(ngx_sockaddr_t));
@ -298,12 +336,37 @@ ngx_stream_upstream_zone_copy_peer(ngx_stream_upstream_rr_peers_t *peers,
}
ngx_memcpy(dst->server.data, src->server.data, src->server.len);
if (src->host) {
dst->host = ngx_slab_calloc_locked(pool,
sizeof(ngx_stream_upstream_host_t));
if (dst->host == NULL) {
goto failed;
}
dst->host->name.data = ngx_slab_alloc_locked(pool,
src->host->name.len);
if (dst->host->name.data == NULL) {
goto failed;
}
dst->host->peers = peers;
dst->host->peer = dst;
dst->host->name.len = src->host->name.len;
ngx_memcpy(dst->host->name.data, src->host->name.data,
src->host->name.len);
}
}
return dst;
failed:
if (dst->host) {
ngx_slab_free_locked(pool, dst->host);
}
if (dst->server.data) {
ngx_slab_free_locked(pool, dst->server.data);
}
@ -320,3 +383,300 @@ failed:
return NULL;
}
static void
ngx_stream_upstream_zone_set_single(ngx_stream_upstream_srv_conf_t *uscf)
{
ngx_stream_upstream_rr_peers_t *peers;
peers = uscf->peer.data;
if (peers->number == 1
&& (peers->next == NULL || peers->next->number == 0))
{
peers->single = 1;
} else {
peers->single = 0;
}
}
static void
ngx_stream_upstream_zone_remove_peer_locked(
ngx_stream_upstream_rr_peers_t *peers, ngx_stream_upstream_rr_peer_t *peer)
{
peers->total_weight -= peer->weight;
peers->number--;
peers->tries -= (peer->down == 0);
(*peers->config)++;
peers->weighted = (peers->total_weight != peers->number);
ngx_stream_upstream_rr_peer_free(peers, peer);
}
static ngx_int_t
ngx_stream_upstream_zone_init_worker(ngx_cycle_t *cycle)
{
ngx_uint_t i;
ngx_event_t *event;
ngx_stream_upstream_rr_peer_t *peer;
ngx_stream_upstream_rr_peers_t *peers;
ngx_stream_upstream_srv_conf_t *uscf, **uscfp;
ngx_stream_upstream_main_conf_t *umcf;
if (ngx_process != NGX_PROCESS_WORKER
&& ngx_process != NGX_PROCESS_SINGLE)
{
return NGX_OK;
}
umcf = ngx_stream_cycle_get_module_main_conf(cycle,
ngx_stream_upstream_module);
if (umcf == NULL) {
return NGX_OK;
}
uscfp = umcf->upstreams.elts;
for (i = 0; i < umcf->upstreams.nelts; i++) {
uscf = uscfp[i];
if (uscf->shm_zone == NULL) {
continue;
}
peers = uscf->peer.data;
do {
ngx_stream_upstream_rr_peers_wlock(peers);
for (peer = peers->resolve; peer; peer = peer->next) {
if (peer->host->worker != ngx_worker) {
continue;
}
event = &peer->host->event;
ngx_memzero(event, sizeof(ngx_event_t));
event->data = uscf;
event->handler = ngx_stream_upstream_zone_resolve_timer;
event->log = cycle->log;
event->cancelable = 1;
ngx_add_timer(event, 1);
}
ngx_stream_upstream_rr_peers_unlock(peers);
peers = peers->next;
} while (peers);
}
return NGX_OK;
}
static void
ngx_stream_upstream_zone_resolve_timer(ngx_event_t *event)
{
ngx_resolver_ctx_t *ctx;
ngx_stream_upstream_host_t *host;
ngx_stream_upstream_srv_conf_t *uscf;
host = (ngx_stream_upstream_host_t *) event;
uscf = event->data;
ctx = ngx_resolve_start(uscf->resolver, NULL);
if (ctx == NULL) {
goto retry;
}
if (ctx == NGX_NO_RESOLVER) {
ngx_log_error(NGX_LOG_ERR, event->log, 0,
"no resolver defined to resolve %V", &host->name);
return;
}
ctx->name = host->name;
ctx->handler = ngx_stream_upstream_zone_resolve_handler;
ctx->data = host;
ctx->timeout = uscf->resolver_timeout;
ctx->cancelable = 1;
if (ngx_resolve_name(ctx) == NGX_OK) {
return;
}
retry:
ngx_add_timer(event, ngx_max(uscf->resolver_timeout, 1000));
}
static void
ngx_stream_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx)
{
time_t now;
in_port_t port;
ngx_msec_t timer;
ngx_uint_t i, j;
ngx_event_t *event;
ngx_resolver_addr_t *addr;
ngx_stream_upstream_host_t *host;
ngx_stream_upstream_rr_peer_t *peer, *template, **peerp;
ngx_stream_upstream_rr_peers_t *peers;
ngx_stream_upstream_srv_conf_t *uscf;
host = ctx->data;
event = &host->event;
uscf = event->data;
peers = host->peers;
template = host->peer;
ngx_stream_upstream_rr_peers_wlock(peers);
now = ngx_time();
if (ctx->state) {
ngx_log_error(NGX_LOG_ERR, event->log, 0,
"%V could not be resolved (%i: %s)",
&ctx->name, ctx->state,
ngx_resolver_strerror(ctx->state));
if (ctx->state != NGX_RESOLVE_NXDOMAIN) {
ngx_stream_upstream_rr_peers_unlock(peers);
ngx_resolve_name_done(ctx);
ngx_add_timer(event, ngx_max(uscf->resolver_timeout, 1000));
return;
}
/* NGX_RESOLVE_NXDOMAIN */
ctx->naddrs = 0;
}
#if (NGX_DEBUG)
{
u_char text[NGX_SOCKADDR_STRLEN];
size_t len;
for (i = 0; i < ctx->naddrs; i++) {
len = ngx_sock_ntop(ctx->addrs[i].sockaddr, ctx->addrs[i].socklen,
text, NGX_SOCKADDR_STRLEN, 0);
ngx_log_debug3(NGX_LOG_DEBUG_STREAM, event->log, 0,
"name %V was resolved to %*s", &host->name, len, text);
}
}
#endif
for (peerp = &peers->peer; *peerp; /* void */ ) {
peer = *peerp;
if (peer->host != host) {
goto next;
}
for (j = 0; j < ctx->naddrs; j++) {
addr = &ctx->addrs[j];
if (addr->name.len == 0
&& ngx_cmp_sockaddr(peer->sockaddr, peer->socklen,
addr->sockaddr, addr->socklen, 0)
== NGX_OK)
{
addr->name.len = 1;
goto next;
}
}
*peerp = peer->next;
ngx_stream_upstream_zone_remove_peer_locked(peers, peer);
ngx_stream_upstream_zone_set_single(uscf);
continue;
next:
peerp = &peer->next;
}
for (i = 0; i < ctx->naddrs; i++) {
addr = &ctx->addrs[i];
if (addr->name.len == 1) {
addr->name.len = 0;
continue;
}
ngx_shmtx_lock(&peers->shpool->mutex);
peer = ngx_stream_upstream_zone_copy_peer(peers, NULL);
ngx_shmtx_unlock(&peers->shpool->mutex);
if (peer == NULL) {
ngx_log_error(NGX_LOG_ERR, event->log, 0,
"cannot add new server to upstream \"%V\", "
"memory exhausted", peers->name);
break;
}
ngx_memcpy(peer->sockaddr, addr->sockaddr, addr->socklen);
port = ((struct sockaddr_in *) template->sockaddr)->sin_port;
switch (peer->sockaddr->sa_family) {
#if (NGX_HAVE_INET6)
case AF_INET6:
((struct sockaddr_in6 *) peer->sockaddr)->sin6_port = port;
break;
#endif
default: /* AF_INET */
((struct sockaddr_in *) peer->sockaddr)->sin_port = port;
}
peer->socklen = addr->socklen;
peer->name.len = ngx_sock_ntop(peer->sockaddr, peer->socklen,
peer->name.data, NGX_SOCKADDR_STRLEN, 1);
peer->host = template->host;
peer->server = template->server;
peer->weight = template->weight;
peer->effective_weight = peer->weight;
peer->max_conns = template->max_conns;
peer->max_fails = template->max_fails;
peer->fail_timeout = template->fail_timeout;
peer->down = template->down;
*peerp = peer;
peerp = &peer->next;
peers->number++;
peers->tries += (peer->down == 0);
peers->total_weight += peer->weight;
peers->weighted = (peers->total_weight != peers->number);
(*peers->config)++;
ngx_stream_upstream_zone_set_single(uscf);
}
ngx_stream_upstream_rr_peers_unlock(peers);
timer = (ngx_msec_t) 1000 * (ctx->valid > now ? ctx->valid - now + 1 : 1);
ngx_resolve_name_done(ctx);
ngx_add_timer(event, timer);
}