mirror/nginx
1
0
Fork 0
mirror of https://github.com/nginx/nginx.git synced 2024-11-23 20:19:02 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Upstream: pre-resolve servers on reload.

Browse Source 
After configuration is reloaded, it may take some time for the
re-resolvable upstream servers to resolve and become available
as peers.  During this time, client requests might get dropped.

Such servers are now pre-resolved using the "cache" of already
resolved peers from the old shared memory zone.
This commit is contained in:
Ruslan Ermilov 2017-11-03 22:22:23 +03:00 committed by Aleksei Bavshin
parent 1524c5e3fc
commit 5ebe7a4122
2 changed files with 388 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

201
src/http/modules/ngx_http_upstream_zone_module.c
View File

@ -15,9 +15,15 @@ static char *ngx_http_upstream_zone(ngx_conf_t *cf, ngx_command_t *cmd,
static ngx_int_t ngx_http_upstream_init_zone(ngx_shm_zone_t *shm_zone,
void *data);
static ngx_http_upstream_rr_peers_t *ngx_http_upstream_zone_copy_peers(
ngx_slab_pool_t *shpool, ngx_http_upstream_srv_conf_t *uscf);
ngx_slab_pool_t *shpool, ngx_http_upstream_srv_conf_t *uscf,
ngx_http_upstream_srv_conf_t *ouscf);
static ngx_http_upstream_rr_peer_t *ngx_http_upstream_zone_copy_peer(
ngx_http_upstream_rr_peers_t *peers, ngx_http_upstream_rr_peer_t *src);
static ngx_int_t ngx_http_upstream_zone_preresolve(
ngx_http_upstream_rr_peer_t *resolve,
ngx_http_upstream_rr_peers_t *peers,
ngx_http_upstream_rr_peer_t *oresolve,
ngx_http_upstream_rr_peers_t *opeers);
static void ngx_http_upstream_zone_set_single(
ngx_http_upstream_srv_conf_t *uscf);
static void ngx_http_upstream_zone_remove_peer_locked(
@ -128,11 +134,11 @@ static ngx_int_t
ngx_http_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data)
{
size_t len;
ngx_uint_t i;
ngx_uint_t i, j;
ngx_slab_pool_t *shpool;
ngx_http_upstream_rr_peers_t *peers, **peersp;
ngx_http_upstream_srv_conf_t *uscf, **uscfp;
ngx_http_upstream_main_conf_t *umcf;
ngx_http_upstream_srv_conf_t *uscf, *ouscf, **uscfp, **ouscfp;
ngx_http_upstream_main_conf_t *umcf, *oumcf;
shpool = (ngx_slab_pool_t *) shm_zone->shm.addr;
umcf = shm_zone->data;
@ -169,6 +175,7 @@ ngx_http_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data)
/* copy peers to shared memory */
peersp = (ngx_http_upstream_rr_peers_t **) (void *) &shpool->data;
oumcf = data;
for (i = 0; i < umcf->upstreams.nelts; i++) {
uscf = uscfp[i];
@ -177,7 +184,38 @@ ngx_http_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data)
continue;
}
peers = ngx_http_upstream_zone_copy_peers(shpool, uscf);
ouscf = NULL;
if (oumcf) {
ouscfp = oumcf->upstreams.elts;
for (j = 0; j < oumcf->upstreams.nelts; j++) {
if (ouscfp[j]->shm_zone == NULL) {
continue;
}
if (ouscfp[j]->shm_zone->shm.name.len != shm_zone->shm.name.len
|| ngx_memcmp(ouscfp[j]->shm_zone->shm.name.data,
shm_zone->shm.name.data,
shm_zone->shm.name.len)
!= 0)
{
continue;
}
if (ouscfp[j]->host.len == uscf->host.len
&& ngx_memcmp(ouscfp[j]->host.data, uscf->host.data,
uscf->host.len)
== 0)
{
ouscf = ouscfp[j];
break;
}
}
}
peers = ngx_http_upstream_zone_copy_peers(shpool, uscf, ouscf);
if (peers == NULL) {
return NGX_ERROR;
}
@ -192,12 +230,14 @@ ngx_http_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data)
static ngx_http_upstream_rr_peers_t *
ngx_http_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,
ngx_http_upstream_srv_conf_t *uscf)
ngx_http_upstream_srv_conf_t *uscf, ngx_http_upstream_srv_conf_t *ouscf)
{
ngx_str_t *name;
ngx_uint_t *config;
ngx_http_upstream_rr_peer_t *peer, **peerp;
ngx_http_upstream_rr_peers_t *peers, *backup;
ngx_http_upstream_rr_peers_t *peers, *opeers, *backup;
opeers = (ouscf ? ouscf->peer.data : NULL);
config = ngx_slab_calloc(shpool, sizeof(ngx_uint_t));
if (config == NULL) {
@ -250,6 +290,16 @@ ngx_http_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,
(*peers->config)++;
}
if (opeers) {
if (ngx_http_upstream_zone_preresolve(peers->resolve, peers,
opeers->resolve, opeers)
!= NGX_OK)
{
return NULL;
}
}
if (peers->next == NULL) {
goto done;
}
@ -289,10 +339,30 @@ ngx_http_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,
peers->next = backup;
if (opeers && opeers->next) {
if (ngx_http_upstream_zone_preresolve(peers->resolve, backup,
opeers->resolve, opeers->next)
!= NGX_OK)
{
return NULL;
}
if (ngx_http_upstream_zone_preresolve(backup->resolve, backup,
opeers->next->resolve,
opeers->next)
!= NGX_OK)
{
return NULL;
}
}
done:
uscf->peer.data = peers;
ngx_http_upstream_zone_set_single(uscf);
return peers;
}
@ -404,6 +474,123 @@ failed:
}
static ngx_int_t
ngx_http_upstream_zone_preresolve(ngx_http_upstream_rr_peer_t *resolve,
ngx_http_upstream_rr_peers_t *peers,
ngx_http_upstream_rr_peer_t *oresolve,
ngx_http_upstream_rr_peers_t *opeers)
{
in_port_t port;
ngx_str_t *server;
ngx_http_upstream_host_t *host;
ngx_http_upstream_rr_peer_t *peer, *template, *opeer, **peerp;
if (resolve == NULL || oresolve == NULL) {
return NGX_OK;
}
for (peerp = &peers->peer; *peerp; peerp = &(*peerp)->next) {
/* void */
}
ngx_http_upstream_rr_peers_rlock(opeers);
for (template = resolve; template; template = template->next) {
for (opeer = oresolve; opeer; opeer = opeer->next) {
if (opeer->host->name.len != template->host->name.len
|| ngx_memcmp(opeer->host->name.data,
template->host->name.data,
template->host->name.len)
!= 0)
{
continue;
}
if (opeer->host->service.len != template->host->service.len
|| ngx_memcmp(opeer->host->service.data,
template->host->service.data,
template->host->service.len)
!= 0)
{
continue;
}
host = opeer->host;
for (opeer = opeers->peer; opeer; opeer = opeer->next) {
if (opeer->host != host) {
continue;
}
peer = ngx_http_upstream_zone_copy_peer(peers, NULL);
if (peer == NULL) {
ngx_http_upstream_rr_peers_unlock(opeers);
return NGX_ERROR;
}
ngx_memcpy(peer->sockaddr, opeer->sockaddr, opeer->socklen);
if (template->host->service.len == 0) {
port = ngx_inet_get_port(template->sockaddr);
ngx_inet_set_port(peer->sockaddr, port);
}
peer->socklen = opeer->socklen;
peer->name.len = ngx_sock_ntop(peer->sockaddr, peer->socklen,
peer->name.data,
NGX_SOCKADDR_STRLEN, 1);
peer->host = template->host;
server = template->host->service.len ? &opeer->server
: &template->server;
peer->server.data = ngx_slab_alloc(peers->shpool, server->len);
if (peer->server.data == NULL) {
ngx_http_upstream_rr_peers_unlock(opeers);
return NGX_ERROR;
}
ngx_memcpy(peer->server.data, server->data, server->len);
peer->server.len = server->len;
if (host->service.len == 0) {
peer->weight = template->weight;
} else {
peer->weight = (template->weight != 1 ? template->weight
: opeer->weight);
}
peer->effective_weight = peer->weight;
peer->max_conns = template->max_conns;
peer->max_fails = template->max_fails;
peer->fail_timeout = template->fail_timeout;
peer->down = template->down;
(*peers->config)++;
*peerp = peer;
peerp = &peer->next;
peers->number++;
peers->tries += (peer->down == 0);
peers->total_weight += peer->weight;
peers->weighted = (peers->total_weight != peers->number);
}
break;
}
}
ngx_http_upstream_rr_peers_unlock(opeers);
return NGX_OK;
}
static void
ngx_http_upstream_zone_set_single(ngx_http_upstream_srv_conf_t *uscf)
{

201
src/stream/ngx_stream_upstream_zone_module.c
View File

@ -15,9 +15,15 @@ static char *ngx_stream_upstream_zone(ngx_conf_t *cf, ngx_command_t *cmd,
static ngx_int_t ngx_stream_upstream_init_zone(ngx_shm_zone_t *shm_zone,
void *data);
static ngx_stream_upstream_rr_peers_t *ngx_stream_upstream_zone_copy_peers(
ngx_slab_pool_t *shpool, ngx_stream_upstream_srv_conf_t *uscf);
ngx_slab_pool_t *shpool, ngx_stream_upstream_srv_conf_t *uscf,
ngx_stream_upstream_srv_conf_t *ouscf);
static ngx_stream_upstream_rr_peer_t *ngx_stream_upstream_zone_copy_peer(
ngx_stream_upstream_rr_peers_t *peers, ngx_stream_upstream_rr_peer_t *src);
static ngx_int_t ngx_stream_upstream_zone_preresolve(
ngx_stream_upstream_rr_peer_t *resolve,
ngx_stream_upstream_rr_peers_t *peers,
ngx_stream_upstream_rr_peer_t *oresolve,
ngx_stream_upstream_rr_peers_t *opeers);
static void ngx_stream_upstream_zone_set_single(
ngx_stream_upstream_srv_conf_t *uscf);
static void ngx_stream_upstream_zone_remove_peer_locked(
@ -125,11 +131,11 @@ static ngx_int_t
ngx_stream_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data)
{
size_t len;
ngx_uint_t i;
ngx_uint_t i, j;
ngx_slab_pool_t *shpool;
ngx_stream_upstream_rr_peers_t *peers, **peersp;
ngx_stream_upstream_srv_conf_t *uscf, **uscfp;
ngx_stream_upstream_main_conf_t *umcf;
ngx_stream_upstream_srv_conf_t *uscf, *ouscf, **uscfp, **ouscfp;
ngx_stream_upstream_main_conf_t *umcf, *oumcf;
shpool = (ngx_slab_pool_t *) shm_zone->shm.addr;
umcf = shm_zone->data;
@ -166,6 +172,7 @@ ngx_stream_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data)
/* copy peers to shared memory */
peersp = (ngx_stream_upstream_rr_peers_t **) (void *) &shpool->data;
oumcf = data;
for (i = 0; i < umcf->upstreams.nelts; i++) {
uscf = uscfp[i];
@ -174,7 +181,38 @@ ngx_stream_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data)
continue;
}
peers = ngx_stream_upstream_zone_copy_peers(shpool, uscf);
ouscf = NULL;
if (oumcf) {
ouscfp = oumcf->upstreams.elts;
for (j = 0; j < oumcf->upstreams.nelts; j++) {
if (ouscfp[j]->shm_zone == NULL) {
continue;
}
if (ouscfp[j]->shm_zone->shm.name.len != shm_zone->shm.name.len
|| ngx_memcmp(ouscfp[j]->shm_zone->shm.name.data,
shm_zone->shm.name.data,
shm_zone->shm.name.len)
!= 0)
{
continue;
}
if (ouscfp[j]->host.len == uscf->host.len
&& ngx_memcmp(ouscfp[j]->host.data, uscf->host.data,
uscf->host.len)
== 0)
{
ouscf = ouscfp[j];
break;
}
}
}
peers = ngx_stream_upstream_zone_copy_peers(shpool, uscf, ouscf);
if (peers == NULL) {
return NGX_ERROR;
}
@ -189,12 +227,14 @@ ngx_stream_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data)
static ngx_stream_upstream_rr_peers_t *
ngx_stream_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,
ngx_stream_upstream_srv_conf_t *uscf)
ngx_stream_upstream_srv_conf_t *uscf, ngx_stream_upstream_srv_conf_t *ouscf)
{
ngx_str_t *name;
ngx_uint_t *config;
ngx_stream_upstream_rr_peer_t *peer, **peerp;
ngx_stream_upstream_rr_peers_t *peers, *backup;
ngx_stream_upstream_rr_peers_t *peers, *opeers, *backup;
opeers = (ouscf ? ouscf->peer.data : NULL);
config = ngx_slab_calloc(shpool, sizeof(ngx_uint_t));
if (config == NULL) {
@ -247,6 +287,16 @@ ngx_stream_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,
(*peers->config)++;
}
if (opeers) {
if (ngx_stream_upstream_zone_preresolve(peers->resolve, peers,
opeers->resolve, opeers)
!= NGX_OK)
{
return NULL;
}
}
if (peers->next == NULL) {
goto done;
}
@ -286,10 +336,30 @@ ngx_stream_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,
peers->next = backup;
if (opeers && opeers->next) {
if (ngx_stream_upstream_zone_preresolve(peers->resolve, backup,
opeers->resolve, opeers->next)
!= NGX_OK)
{
return NULL;
}
if (ngx_stream_upstream_zone_preresolve(backup->resolve, backup,
opeers->next->resolve,
opeers->next)
!= NGX_OK)
{
return NULL;
}
}
done:
uscf->peer.data = peers;
ngx_stream_upstream_zone_set_single(uscf);
return peers;
}
@ -401,6 +471,123 @@ failed:
}
static ngx_int_t
ngx_stream_upstream_zone_preresolve(ngx_stream_upstream_rr_peer_t *resolve,
ngx_stream_upstream_rr_peers_t *peers,
ngx_stream_upstream_rr_peer_t *oresolve,
ngx_stream_upstream_rr_peers_t *opeers)
{
in_port_t port;
ngx_str_t *server;
ngx_stream_upstream_host_t *host;
ngx_stream_upstream_rr_peer_t *peer, *template, *opeer, **peerp;
if (resolve == NULL || oresolve == NULL) {
return NGX_OK;
}
for (peerp = &peers->peer; *peerp; peerp = &(*peerp)->next) {
/* void */
}
ngx_stream_upstream_rr_peers_rlock(opeers);
for (template = resolve; template; template = template->next) {
for (opeer = oresolve; opeer; opeer = opeer->next) {
if (opeer->host->name.len != template->host->name.len
|| ngx_memcmp(opeer->host->name.data,
template->host->name.data,
template->host->name.len)
!= 0)
{
continue;
}
if (opeer->host->service.len != template->host->service.len
|| ngx_memcmp(opeer->host->service.data,
template->host->service.data,
template->host->service.len)
!= 0)
{
continue;
}
host = opeer->host;
for (opeer = opeers->peer; opeer; opeer = opeer->next) {
if (opeer->host != host) {
continue;
}
peer = ngx_stream_upstream_zone_copy_peer(peers, NULL);
if (peer == NULL) {
ngx_stream_upstream_rr_peers_unlock(opeers);
return NGX_ERROR;
}
ngx_memcpy(peer->sockaddr, opeer->sockaddr, opeer->socklen);
if (template->host->service.len == 0) {
port = ngx_inet_get_port(template->sockaddr);
ngx_inet_set_port(peer->sockaddr, port);
}
peer->socklen = opeer->socklen;
peer->name.len = ngx_sock_ntop(peer->sockaddr, peer->socklen,
peer->name.data,
NGX_SOCKADDR_STRLEN, 1);
peer->host = template->host;
server = template->host->service.len ? &opeer->server
: &template->server;
peer->server.data = ngx_slab_alloc(peers->shpool, server->len);
if (peer->server.data == NULL) {
ngx_stream_upstream_rr_peers_unlock(opeers);
return NGX_ERROR;
}
ngx_memcpy(peer->server.data, server->data, server->len);
peer->server.len = server->len;
if (host->service.len == 0) {
peer->weight = template->weight;
} else {
peer->weight = (template->weight != 1 ? template->weight
: opeer->weight);
}
peer->effective_weight = peer->weight;
peer->max_conns = template->max_conns;
peer->max_fails = template->max_fails;
peer->fail_timeout = template->fail_timeout;
peer->down = template->down;
(*peers->config)++;
*peerp = peer;
peerp = &peer->next;
peers->number++;
peers->tries += (peer->down == 0);
peers->total_weight += peer->weight;
peers->weighted = (peers->total_weight != peers->number);
}
break;
}
}
ngx_stream_upstream_rr_peers_unlock(opeers);
return NGX_OK;
}
static void
ngx_stream_upstream_zone_set_single(ngx_stream_upstream_srv_conf_t *uscf)
{