diff --git a/src/http/modules/ngx_http_upstream_hash_module.c b/src/http/modules/ngx_http_upstream_hash_module.c index 249733f72..56ec38ae0 100644 --- a/src/http/modules/ngx_http_upstream_hash_module.c +++ b/src/http/modules/ngx_http_upstream_hash_module.c @@ -176,7 +176,10 @@ ngx_http_upstream_get_hash_peer(ngx_peer_connection_t *pc, void *data) ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, "get hash peer, try: %ui", pc->tries); + ngx_http_upstream_rr_peers_wlock(hp->rrp.peers); + if (hp->tries > 20 || hp->rrp.peers->single) { + ngx_http_upstream_rr_peers_unlock(hp->rrp.peers); return hp->get_rr_peer(pc, &hp->rrp); } @@ -258,6 +261,7 @@ ngx_http_upstream_get_hash_peer(ngx_peer_connection_t *pc, void *data) next: if (++hp->tries > 20) { + ngx_http_upstream_rr_peers_unlock(hp->rrp.peers); return hp->get_rr_peer(pc, &hp->rrp); } } @@ -274,6 +278,8 @@ ngx_http_upstream_get_hash_peer(ngx_peer_connection_t *pc, void *data) peer->checked = now; } + ngx_http_upstream_rr_peers_unlock(hp->rrp.peers); + hp->rrp.tried[n] |= m; return NGX_OK; @@ -465,8 +471,13 @@ ngx_http_upstream_init_chash_peer(ngx_http_request_t *r, hcf = ngx_http_conf_upstream_srv_conf(us, ngx_http_upstream_hash_module); hash = ngx_crc32_long(hp->key.data, hp->key.len); + + ngx_http_upstream_rr_peers_rlock(hp->rrp.peers); + hp->hash = ngx_http_upstream_find_chash_point(hcf->points, hash); + ngx_http_upstream_rr_peers_unlock(hp->rrp.peers); + return NGX_OK; } @@ -489,6 +500,8 @@ ngx_http_upstream_get_chash_peer(ngx_peer_connection_t *pc, void *data) ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, "get consistent hash peer, try: %ui", pc->tries); + ngx_http_upstream_rr_peers_wlock(hp->rrp.peers); + pc->cached = 0; pc->connection = NULL; @@ -561,6 +574,7 @@ ngx_http_upstream_get_chash_peer(ngx_peer_connection_t *pc, void *data) hp->tries++; if (hp->tries >= points->number) { + ngx_http_upstream_rr_peers_unlock(hp->rrp.peers); return NGX_BUSY; } } @@ -579,6 +593,8 @@ found: best->checked = now; } + ngx_http_upstream_rr_peers_unlock(hp->rrp.peers); + n = best_i / (8 * sizeof(uintptr_t)); m = (uintptr_t) 1 << best_i % (8 * sizeof(uintptr_t)); diff --git a/src/http/modules/ngx_http_upstream_ip_hash_module.c b/src/http/modules/ngx_http_upstream_ip_hash_module.c index 2c6506d6f..debc0d8e5 100644 --- a/src/http/modules/ngx_http_upstream_ip_hash_module.c +++ b/src/http/modules/ngx_http_upstream_ip_hash_module.c @@ -161,7 +161,10 @@ ngx_http_upstream_get_ip_hash_peer(ngx_peer_connection_t *pc, void *data) /* TODO: cached */ + ngx_http_upstream_rr_peers_wlock(iphp->rrp.peers); + if (iphp->tries > 20 || iphp->rrp.peers->single) { + ngx_http_upstream_rr_peers_unlock(iphp->rrp.peers); return iphp->get_rr_peer(pc, &iphp->rrp); } @@ -212,8 +215,6 @@ ngx_http_upstream_get_ip_hash_peer(ngx_peer_connection_t *pc, void *data) ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0, "get ip hash peer, hash: %ui %04XA", p, m); - /* ngx_lock_mutex(iphp->rrp.peers->mutex); */ - if (peer->down) { goto next_try; } @@ -230,14 +231,12 @@ ngx_http_upstream_get_ip_hash_peer(ngx_peer_connection_t *pc, void *data) next_try: iphp->rrp.tried[n] |= m; - - /* ngx_unlock_mutex(iphp->rrp.peers->mutex); */ - pc->tries--; next: if (++iphp->tries > 20) { + ngx_http_upstream_rr_peers_unlock(iphp->rrp.peers); return iphp->get_rr_peer(pc, &iphp->rrp); } } @@ -254,7 +253,7 @@ ngx_http_upstream_get_ip_hash_peer(ngx_peer_connection_t *pc, void *data) peer->checked = now; } - /* ngx_unlock_mutex(iphp->rrp.peers->mutex); */ + ngx_http_upstream_rr_peers_unlock(iphp->rrp.peers); iphp->rrp.tried[n] |= m; iphp->hash = hash; diff --git a/src/http/modules/ngx_http_upstream_least_conn_module.c b/src/http/modules/ngx_http_upstream_least_conn_module.c index 61b538631..92951bd20 100644 --- a/src/http/modules/ngx_http_upstream_least_conn_module.c +++ b/src/http/modules/ngx_http_upstream_least_conn_module.c @@ -122,6 +122,8 @@ ngx_http_upstream_get_least_conn_peer(ngx_peer_connection_t *pc, void *data) peers = rrp->peers; + ngx_http_upstream_rr_peers_wlock(peers); + best = NULL; total = 0; @@ -241,6 +243,8 @@ ngx_http_upstream_get_least_conn_peer(ngx_peer_connection_t *pc, void *data) rrp->tried[n] |= m; + ngx_http_upstream_rr_peers_unlock(peers); + return NGX_OK; failed: @@ -258,11 +262,15 @@ failed: rrp->tried[i] = 0; } + ngx_http_upstream_rr_peers_unlock(peers); + rc = ngx_http_upstream_get_least_conn_peer(pc, rrp); if (rc != NGX_BUSY) { return rc; } + + ngx_http_upstream_rr_peers_wlock(peers); } /* all peers failed, mark them as live for quick recovery */ @@ -271,6 +279,8 @@ failed: peer->fails = 0; } + ngx_http_upstream_rr_peers_unlock(peers); + pc->name = peers->name; return NGX_BUSY; diff --git a/src/http/ngx_http_upstream_round_robin.c b/src/http/ngx_http_upstream_round_robin.c index 8d7046ade..9a970dcba 100644 --- a/src/http/ngx_http_upstream_round_robin.c +++ b/src/http/ngx_http_upstream_round_robin.c @@ -432,8 +432,7 @@ ngx_http_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data) pc->connection = NULL; peers = rrp->peers; - - /* ngx_lock_mutex(peers->mutex); */ + ngx_http_upstream_rr_peers_wlock(peers); if (peers->single) { peer = peers->peer; @@ -465,7 +464,7 @@ ngx_http_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data) peer->conns++; - /* ngx_unlock_mutex(peers->mutex); */ + ngx_http_upstream_rr_peers_unlock(peers); return NGX_OK; @@ -473,8 +472,6 @@ failed: if (peers->next) { - /* ngx_unlock_mutex(peers->mutex); */ - ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, "backup servers"); rrp->peers = peers->next; @@ -486,13 +483,15 @@ failed: rrp->tried[i] = 0; } + ngx_http_upstream_rr_peers_unlock(peers); + rc = ngx_http_upstream_get_round_robin_peer(pc, rrp); if (rc != NGX_BUSY) { return rc; } - /* ngx_lock_mutex(peers->mutex); */ + ngx_http_upstream_rr_peers_wlock(peers); } /* all peers failed, mark them as live for quick recovery */ @@ -501,7 +500,7 @@ failed: peer->fails = 0; } - /* ngx_unlock_mutex(peers->mutex); */ + ngx_http_upstream_rr_peers_unlock(peers); pc->name = peers->name; @@ -608,11 +607,12 @@ ngx_http_upstream_free_round_robin_peer(ngx_peer_connection_t *pc, void *data, return; } + ngx_http_upstream_rr_peers_rlock(rrp->peers); + ngx_http_upstream_rr_peer_lock(rrp->peers, peer); + if (state & NGX_PEER_FAILED) { now = ngx_time(); - /* ngx_lock_mutex(rrp->peers->mutex); */ - peer->fails++; peer->accessed = now; peer->checked = now; @@ -629,8 +629,6 @@ ngx_http_upstream_free_round_robin_peer(ngx_peer_connection_t *pc, void *data, peer->effective_weight = 0; } - /* ngx_unlock_mutex(rrp->peers->mutex); */ - } else { /* mark peer live if check passed */ @@ -642,11 +640,12 @@ ngx_http_upstream_free_round_robin_peer(ngx_peer_connection_t *pc, void *data, peer->conns--; + ngx_http_upstream_rr_peer_unlock(rrp->peers, peer); + ngx_http_upstream_rr_peers_unlock(rrp->peers); + if (pc->tries) { pc->tries--; } - - /* ngx_unlock_mutex(rrp->peers->mutex); */ } @@ -664,9 +663,6 @@ ngx_http_upstream_set_round_robin_peer_session(ngx_peer_connection_t *pc, peer = rrp->current; - /* TODO: threads only mutex */ - /* ngx_lock_mutex(rrp->peers->mutex); */ - ssl_session = peer->ssl_session; rc = ngx_ssl_set_session(pc->connection, ssl_session); @@ -674,8 +670,6 @@ ngx_http_upstream_set_round_robin_peer_session(ngx_peer_connection_t *pc, ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, "set session: %p", ssl_session); - /* ngx_unlock_mutex(rrp->peers->mutex); */ - return rc; } @@ -700,14 +694,9 @@ ngx_http_upstream_save_round_robin_peer_session(ngx_peer_connection_t *pc, peer = rrp->current; - /* TODO: threads only mutex */ - /* ngx_lock_mutex(rrp->peers->mutex); */ - old_ssl_session = peer->ssl_session; peer->ssl_session = ssl_session; - /* ngx_unlock_mutex(rrp->peers->mutex); */ - if (old_ssl_session) { ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, diff --git a/src/http/ngx_http_upstream_round_robin.h b/src/http/ngx_http_upstream_round_robin.h index 53beac8e3..0dca40e18 100644 --- a/src/http/ngx_http_upstream_round_robin.h +++ b/src/http/ngx_http_upstream_round_robin.h @@ -63,6 +63,13 @@ struct ngx_http_upstream_rr_peers_s { }; +#define ngx_http_upstream_rr_peers_rlock(peers) +#define ngx_http_upstream_rr_peers_wlock(peers) +#define ngx_http_upstream_rr_peers_unlock(peers) +#define ngx_http_upstream_rr_peer_lock(peers, peer) +#define ngx_http_upstream_rr_peer_unlock(peers, peer) + + typedef struct { ngx_http_upstream_rr_peers_t *peers; ngx_http_upstream_rr_peer_t *current;