mirror/nginx
1
0
Fork 0
mirror of https://github.com/nginx/nginx.git synced 2024-11-27 15:39:01 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge of r4655, r4656, r4657, r4695, r4696: upstream changes.

Browse Source 
*) Upstream: least_conn balancer module.

*) Upstream: weights and IPv6 support in ip_hash balancer.

*) Upstream keepalive: "single" parameter deprecated.
This commit is contained in:
Maxim Dounin 2012-07-02 16:41:13 +00:00
parent bb612f1336
commit eaf3544e15
8 changed files with 478 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
auto/modules
View File

@ -345,6 +345,11 @@ if [ $HTTP_UPSTREAM_IP_HASH = YES ]; then
HTTP_SRCS="$HTTP_SRCS $HTTP_UPSTREAM_IP_HASH_SRCS"
fi
if [ $HTTP_UPSTREAM_LEAST_CONN = YES ]; then
HTTP_MODULES="$HTTP_MODULES $HTTP_UPSTREAM_LEAST_CONN_MODULE"
HTTP_SRCS="$HTTP_SRCS $HTTP_UPSTREAM_LEAST_CONN_SRCS"
fi
if [ $HTTP_UPSTREAM_KEEPALIVE = YES ]; then
HTTP_MODULES="$HTTP_MODULES $HTTP_UPSTREAM_KEEPALIVE_MODULE"
HTTP_SRCS="$HTTP_SRCS $HTTP_UPSTREAM_KEEPALIVE_SRCS"

3
auto/options
View File

@ -96,6 +96,7 @@ HTTP_FLV=NO
HTTP_MP4=NO
HTTP_GZIP_STATIC=NO
HTTP_UPSTREAM_IP_HASH=YES
HTTP_UPSTREAM_LEAST_CONN=YES
HTTP_UPSTREAM_KEEPALIVE=YES
# STUB
@ -243,6 +244,8 @@ use the \"--without-http_limit_conn_module\" option instead"
--without-http_empty_gif_module) HTTP_EMPTY_GIF=NO ;;
--without-http_browser_module) HTTP_BROWSER=NO ;;
--without-http_upstream_ip_hash_module) HTTP_UPSTREAM_IP_HASH=NO ;;
--without-http_upstream_least_conn_module)
HTTP_UPSTREAM_LEAST_CONN=NO ;;
--without-http_upstream_keepalive_module) HTTP_UPSTREAM_KEEPALIVE=NO ;;
--with-http_perl_module) HTTP_PERL=YES ;;

5
auto/sources
View File

@ -479,6 +479,11 @@ HTTP_UPSTREAM_IP_HASH_MODULE=ngx_http_upstream_ip_hash_module
HTTP_UPSTREAM_IP_HASH_SRCS=src/http/modules/ngx_http_upstream_ip_hash_module.c
HTTP_UPSTREAM_LEAST_CONN_MODULE=ngx_http_upstream_least_conn_module
HTTP_UPSTREAM_LEAST_CONN_SRCS=" \
src/http/modules/ngx_http_upstream_least_conn_module.c"
HTTP_UPSTREAM_KEEPALIVE_MODULE=ngx_http_upstream_keepalive_module
HTTP_UPSTREAM_KEEPALIVE_SRCS=" \
src/http/modules/ngx_http_upstream_keepalive_module.c"

59
src/http/modules/ngx_http_upstream_ip_hash_module.c
View File

@ -16,7 +16,8 @@ typedef struct {
ngx_uint_t hash;
u_char addr[3];
u_char addrlen;
u_char *addr;
u_char tries;
@ -76,7 +77,10 @@ ngx_module_t ngx_http_upstream_ip_hash_module = {
};
ngx_int_t
static u_char ngx_http_upstream_ip_hash_pseudo_addr[3];
static ngx_int_t
ngx_http_upstream_init_ip_hash(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us)
{
if (ngx_http_upstream_init_round_robin(cf, us) != NGX_OK) {
@ -93,8 +97,10 @@ static ngx_int_t
ngx_http_upstream_init_ip_hash_peer(ngx_http_request_t *r,
ngx_http_upstream_srv_conf_t *us)
{
u_char *p;
struct sockaddr_in *sin;
#if (NGX_HAVE_INET6)
struct sockaddr_in6 *sin6;
#endif
ngx_http_upstream_ip_hash_peer_data_t *iphp;
iphp = ngx_palloc(r->pool, sizeof(ngx_http_upstream_ip_hash_peer_data_t));
@ -110,20 +116,25 @@ ngx_http_upstream_init_ip_hash_peer(ngx_http_request_t *r,
r->upstream->peer.get = ngx_http_upstream_get_ip_hash_peer;
/* AF_INET only */
if (r->connection->sockaddr->sa_family == AF_INET) {
switch (r->connection->sockaddr->sa_family) {
case AF_INET:
sin = (struct sockaddr_in *) r->connection->sockaddr;
p = (u_char *) &sin->sin_addr.s_addr;
iphp->addr[0] = p[0];
iphp->addr[1] = p[1];
iphp->addr[2] = p[2];
iphp->addr = (u_char *) &sin->sin_addr.s_addr;
iphp->addrlen = 3;
break;
} else {
iphp->addr[0] = 0;
iphp->addr[1] = 0;
iphp->addr[2] = 0;
#if (NGX_HAVE_INET6)
case AF_INET6:
sin6 = (struct sockaddr_in6 *) r->connection->sockaddr;
iphp->addr = (u_char *) &sin6->sin6_addr.s6_addr;
iphp->addrlen = 16;
break;
#endif
default:
iphp->addr = ngx_http_upstream_ip_hash_pseudo_addr;
iphp->addrlen = 3;
}
iphp->hash = 89;
@ -140,6 +151,7 @@ ngx_http_upstream_get_ip_hash_peer(ngx_peer_connection_t *pc, void *data)
ngx_http_upstream_ip_hash_peer_data_t *iphp = data;
time_t now;
ngx_int_t w;
uintptr_t m;
ngx_uint_t i, n, p, hash;
ngx_http_upstream_rr_peer_t *peer;
@ -162,11 +174,25 @@ ngx_http_upstream_get_ip_hash_peer(ngx_peer_connection_t *pc, void *data)
for ( ;; ) {
for (i = 0; i < 3; i++) {
for (i = 0; i < iphp->addrlen; i++) {
hash = (hash * 113 + iphp->addr[i]) % 6271;
}
p = hash % iphp->rrp.peers->number;
if (!iphp->rrp.peers->weighted) {
p = hash % iphp->rrp.peers->number;
} else {
w = hash % iphp->rrp.peers->total_weight;
for (i = 0; i < iphp->rrp.peers->number; i++) {
w -= iphp->rrp.peers->peer[i].weight;
if (w < 0) {
break;
}
}
p = i;
}
n = p / (8 * sizeof(uintptr_t));
m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t));
@ -229,6 +255,7 @@ ngx_http_upstream_ip_hash(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
uscf->peer.init_upstream = ngx_http_upstream_init_ip_hash;
uscf->flags = NGX_HTTP_UPSTREAM_CREATE
|NGX_HTTP_UPSTREAM_WEIGHT
|NGX_HTTP_UPSTREAM_MAX_FAILS
|NGX_HTTP_UPSTREAM_FAIL_TIMEOUT
|NGX_HTTP_UPSTREAM_DOWN;

33
src/http/modules/ngx_http_upstream_keepalive_module.c
View File

@ -12,7 +12,6 @@
typedef struct {
ngx_uint_t max_cached;
ngx_uint_t single; /* unsigned:1 */
ngx_queue_t cache;
ngx_queue_t free;
@ -223,36 +222,11 @@ ngx_http_upstream_get_keepalive_peer(ngx_peer_connection_t *pc, void *data)
kp->failed = 0;
/* single pool of cached connections */
if (kp->conf->single && !ngx_queue_empty(&kp->conf->cache)) {
q = ngx_queue_head(&kp->conf->cache);
item = ngx_queue_data(q, ngx_http_upstream_keepalive_cache_t, queue);
c = item->connection;
ngx_queue_remove(q);
ngx_queue_insert_head(&kp->conf->free, q);
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"get keepalive peer: using connection %p", c);
c->idle = 0;
c->log = pc->log;
c->read->log = pc->log;
c->write->log = pc->log;
c->pool->log = pc->log;
pc->connection = c;
pc->cached = 1;
return NGX_DONE;
}
/* ask balancer */
rc = kp->original_get_peer(pc, kp->data);
if (kp->conf->single || rc != NGX_OK) {
if (rc != NGX_OK) {
return rc;
}
@ -552,7 +526,8 @@ ngx_http_upstream_keepalive(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
for (i = 2; i < cf->args->nelts; i++) {
if (ngx_strcmp(value[i].data, "single") == 0) {
kcf->single = 1;
ngx_conf_log_error(NGX_LOG_WARN, cf, 0,
"the \"single\" parameter is deprecated");
continue;
}

402
src/http/modules/ngx_http_upstream_least_conn_module.c Normal file
View File

@ -0,0 +1,402 @@
/*
* Copyright (C) Maxim Dounin
* Copyright (C) Nginx, Inc.
*/
#include <ngx_config.h>
#include <ngx_core.h>
#include <ngx_http.h>
typedef struct {
ngx_uint_t *conns;
} ngx_http_upstream_least_conn_conf_t;
typedef struct {
/* the round robin data must be first */
ngx_http_upstream_rr_peer_data_t rrp;
ngx_uint_t *conns;
ngx_event_get_peer_pt get_rr_peer;
ngx_event_free_peer_pt free_rr_peer;
} ngx_http_upstream_lc_peer_data_t;
static ngx_int_t ngx_http_upstream_init_least_conn_peer(ngx_http_request_t *r,
ngx_http_upstream_srv_conf_t *us);
static ngx_int_t ngx_http_upstream_get_least_conn_peer(
ngx_peer_connection_t *pc, void *data);
static void ngx_http_upstream_free_least_conn_peer(ngx_peer_connection_t *pc,
void *data, ngx_uint_t state);
static void *ngx_http_upstream_least_conn_create_conf(ngx_conf_t *cf);
static char *ngx_http_upstream_least_conn(ngx_conf_t *cf, ngx_command_t *cmd,
void *conf);
static ngx_command_t ngx_http_upstream_least_conn_commands[] = {
{ ngx_string("least_conn"),
NGX_HTTP_UPS_CONF|NGX_CONF_NOARGS,
ngx_http_upstream_least_conn,
0,
0,
NULL },
ngx_null_command
};
static ngx_http_module_t ngx_http_upstream_least_conn_module_ctx = {
NULL, /* preconfiguration */
NULL, /* postconfiguration */
NULL, /* create main configuration */
NULL, /* init main configuration */
ngx_http_upstream_least_conn_create_conf, /* create server configuration */
NULL, /* merge server configuration */
NULL, /* create location configuration */
NULL /* merge location configuration */
};
ngx_module_t ngx_http_upstream_least_conn_module = {
NGX_MODULE_V1,
&ngx_http_upstream_least_conn_module_ctx, /* module context */
ngx_http_upstream_least_conn_commands, /* module directives */
NGX_HTTP_MODULE, /* module type */
NULL, /* init master */
NULL, /* init module */
NULL, /* init process */
NULL, /* init thread */
NULL, /* exit thread */
NULL, /* exit process */
NULL, /* exit master */
NGX_MODULE_V1_PADDING
};
ngx_int_t
ngx_http_upstream_init_least_conn(ngx_conf_t *cf,
ngx_http_upstream_srv_conf_t *us)
{
ngx_uint_t n;
ngx_http_upstream_rr_peers_t *peers;
ngx_http_upstream_least_conn_conf_t *lcf;
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, cf->log, 0,
"init least conn");
if (ngx_http_upstream_init_round_robin(cf, us) != NGX_OK) {
return NGX_ERROR;
}
peers = us->peer.data;
n = peers->number;
if (peers->next) {
n += peers->next->number;
}
lcf = ngx_http_conf_upstream_srv_conf(us,
ngx_http_upstream_least_conn_module);
lcf->conns = ngx_pcalloc(cf->pool, sizeof(ngx_uint_t) * n);
if (lcf->conns == NULL) {
return NGX_ERROR;
}
us->peer.init = ngx_http_upstream_init_least_conn_peer;
return NGX_OK;
}
static ngx_int_t
ngx_http_upstream_init_least_conn_peer(ngx_http_request_t *r,
ngx_http_upstream_srv_conf_t *us)
{
ngx_http_upstream_lc_peer_data_t *lcp;
ngx_http_upstream_least_conn_conf_t *lcf;
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"init least conn peer");
lcf = ngx_http_conf_upstream_srv_conf(us,
ngx_http_upstream_least_conn_module);
lcp = ngx_palloc(r->pool, sizeof(ngx_http_upstream_lc_peer_data_t));
if (lcp == NULL) {
return NGX_ERROR;
}
lcp->conns = lcf->conns;
r->upstream->peer.data = &lcp->rrp;
if (ngx_http_upstream_init_round_robin_peer(r, us) != NGX_OK) {
return NGX_ERROR;
}
r->upstream->peer.get = ngx_http_upstream_get_least_conn_peer;
r->upstream->peer.free = ngx_http_upstream_free_least_conn_peer;
lcp->get_rr_peer = ngx_http_upstream_get_round_robin_peer;
lcp->free_rr_peer = ngx_http_upstream_free_round_robin_peer;
return NGX_OK;
}
static ngx_int_t
ngx_http_upstream_get_least_conn_peer(ngx_peer_connection_t *pc, void *data)
{
ngx_http_upstream_lc_peer_data_t *lcp = data;
time_t now;
uintptr_t m;
ngx_int_t rc, total;
ngx_uint_t i, n, p, many;
ngx_http_upstream_rr_peer_t *peer, *best;
ngx_http_upstream_rr_peers_t *peers;
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"get least conn peer, try: %ui", pc->tries);
if (lcp->rrp.peers->single) {
return lcp->get_rr_peer(pc, &lcp->rrp);
}
pc->cached = 0;
pc->connection = NULL;
now = ngx_time();
peers = lcp->rrp.peers;
best = NULL;
total = 0;
#if (NGX_SUPPRESS_WARN)
many = 0;
p = 0;
#endif
for (i = 0; i < peers->number; i++) {
n = i / (8 * sizeof(uintptr_t));
m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t));
if (lcp->rrp.tried[n] & m) {
continue;
}
peer = &peers->peer[i];
if (peer->down) {
continue;
}
if (peer->max_fails
&& peer->fails >= peer->max_fails
&& now - peer->checked <= peer->fail_timeout)
{
continue;
}
/*
* select peer with least number of connections; if there are
* multiple peers with the same number of connections, select
* based on round-robin
*/
if (best == NULL
|| lcp->conns[i] * best->weight < lcp->conns[p] * peer->weight)
{
best = peer;
many = 0;
p = i;
} else if (lcp->conns[i] * best->weight
== lcp->conns[p] * peer->weight)
{
many = 1;
}
}
if (best == NULL) {
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"get least conn peer, no peer found");
goto failed;
}
if (many) {
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"get least conn peer, many");
for (i = p; i < peers->number; i++) {
n = i / (8 * sizeof(uintptr_t));
m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t));
if (lcp->rrp.tried[n] & m) {
continue;
}
peer = &peers->peer[i];
if (peer->down) {
continue;
}
if (lcp->conns[i] * best->weight != lcp->conns[p] * peer->weight) {
continue;
}
if (peer->max_fails
&& peer->fails >= peer->max_fails
&& now - peer->checked <= peer->fail_timeout)
{
continue;
}
peer->current_weight += peer->effective_weight;
total += peer->effective_weight;
if (peer->effective_weight < peer->weight) {
peer->effective_weight++;
}
if (peer->current_weight > best->current_weight) {
best = peer;
p = i;
}
}
}
best->current_weight -= total;
best->checked = now;
pc->sockaddr = best->sockaddr;
pc->socklen = best->socklen;
pc->name = &best->name;
lcp->rrp.current = p;
n = p / (8 * sizeof(uintptr_t));
m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t));
lcp->rrp.tried[n] |= m;
lcp->conns[p]++;
if (pc->tries == 1 && peers->next) {
pc->tries += peers->next->number;
}
return NGX_OK;
failed:
if (peers->next) {
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"get least conn peer, backup servers");
lcp->conns += peers->number;
lcp->rrp.peers = peers->next;
pc->tries = lcp->rrp.peers->number;
n = lcp->rrp.peers->number / (8 * sizeof(uintptr_t)) + 1;
for (i = 0; i < n; i++) {
lcp->rrp.tried[i] = 0;
}
rc = ngx_http_upstream_get_least_conn_peer(pc, lcp);
if (rc != NGX_BUSY) {
return rc;
}
}
/* all peers failed, mark them as live for quick recovery */
for (i = 0; i < peers->number; i++) {
peers->peer[i].fails = 0;
}
pc->name = peers->name;
return NGX_BUSY;
}
static void
ngx_http_upstream_free_least_conn_peer(ngx_peer_connection_t *pc,
void *data, ngx_uint_t state)
{
ngx_http_upstream_lc_peer_data_t *lcp = data;
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"free least conn peer %ui %ui", pc->tries, state);
if (lcp->rrp.peers->single) {
lcp->free_rr_peer(pc, &lcp->rrp, state);
return;
}
if (state == 0 && pc->tries == 0) {
return;
}
lcp->conns[lcp->rrp.current]--;
lcp->free_rr_peer(pc, &lcp->rrp, state);
}
static void *
ngx_http_upstream_least_conn_create_conf(ngx_conf_t *cf)
{
ngx_http_upstream_least_conn_conf_t *conf;
conf = ngx_pcalloc(cf->pool,
sizeof(ngx_http_upstream_least_conn_conf_t));
if (conf == NULL) {
return NULL;
}
/*
* set by ngx_pcalloc():
*
* conf->conns = NULL;
*/
return conf;
}
static char *
ngx_http_upstream_least_conn(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
{
ngx_http_upstream_srv_conf_t *uscf;
uscf = ngx_http_conf_get_module_srv_conf(cf, ngx_http_upstream_module);
uscf->peer.init_upstream = ngx_http_upstream_init_least_conn;
uscf->flags = NGX_HTTP_UPSTREAM_CREATE
|NGX_HTTP_UPSTREAM_WEIGHT
|NGX_HTTP_UPSTREAM_MAX_FAILS
|NGX_HTTP_UPSTREAM_FAIL_TIMEOUT
|NGX_HTTP_UPSTREAM_DOWN
|NGX_HTTP_UPSTREAM_BACKUP;
return NGX_CONF_OK;
}

12
src/http/ngx_http_upstream_round_robin.c
View File

@ -30,7 +30,7 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
ngx_http_upstream_srv_conf_t *us)
{
ngx_url_t u;
ngx_uint_t i, j, n;
ngx_uint_t i, j, n, w;
ngx_http_upstream_server_t *server;
ngx_http_upstream_rr_peers_t *peers, *backup;
@ -40,6 +40,7 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
server = us->servers->elts;
n = 0;
w = 0;
for (i = 0; i < us->servers->nelts; i++) {
if (server[i].backup) {
@ -47,6 +48,7 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
}
n += server[i].naddrs;
w += server[i].naddrs * server[i].weight;
}
if (n == 0) {
@ -64,6 +66,8 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
peers->single = (n == 1);
peers->number = n;
peers->weighted = (w != n);
peers->total_weight = w;
peers->name = &us->host;
n = 0;
@ -96,6 +100,7 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
/* backup servers */
n = 0;
w = 0;
for (i = 0; i < us->servers->nelts; i++) {
if (!server[i].backup) {
@ -103,6 +108,7 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
}
n += server[i].naddrs;
w += server[i].naddrs * server[i].weight;
}
if (n == 0) {
@ -118,6 +124,8 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
peers->single = 0;
backup->single = 0;
backup->number = n;
backup->weighted = (w != n);
backup->total_weight = w;
backup->name = &us->host;
n = 0;
@ -185,6 +193,8 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
peers->single = (n == 1);
peers->number = n;
peers->weighted = 0;
peers->total_weight = n;
peers->name = &us->host;
for (i = 0; i < u.naddrs; i++) {

6
src/http/ngx_http_upstream_round_robin.h
View File

@ -41,13 +41,17 @@ typedef struct {
typedef struct ngx_http_upstream_rr_peers_s ngx_http_upstream_rr_peers_t;
struct ngx_http_upstream_rr_peers_s {
ngx_uint_t single; /* unsigned single:1; */
ngx_uint_t number;
ngx_uint_t last_cached;
/* ngx_mutex_t *mutex; */
ngx_connection_t **cached;
ngx_uint_t total_weight;
unsigned single:1;
unsigned weighted:1;
ngx_str_t *name;
ngx_http_upstream_rr_peers_t *next;