2004-06-17 03:36:07 +08:00
|
|
|
|
2004-09-28 16:34:51 +08:00
|
|
|
/*
|
2004-09-30 00:00:49 +08:00
|
|
|
* Copyright (C) Igor Sysoev
|
2012-01-18 23:07:43 +08:00
|
|
|
* Copyright (C) Nginx, Inc.
|
2004-09-28 16:34:51 +08:00
|
|
|
*/
|
2004-06-17 03:36:07 +08:00
|
|
|
|
2006-11-20 16:51:45 +08:00
|
|
|
#include <ngx_config.h>
|
|
|
|
#include <ngx_core.h>
|
2004-06-17 03:36:07 +08:00
|
|
|
|
2006-11-20 16:51:45 +08:00
|
|
|
|
|
|
|
#define NGX_SLAB_PAGE_MASK 3
|
|
|
|
#define NGX_SLAB_PAGE 0
|
|
|
|
#define NGX_SLAB_BIG 1
|
|
|
|
#define NGX_SLAB_EXACT 2
|
|
|
|
#define NGX_SLAB_SMALL 3
|
|
|
|
|
|
|
|
#if (NGX_PTR_SIZE == 4)
|
|
|
|
|
|
|
|
#define NGX_SLAB_PAGE_FREE 0
|
|
|
|
#define NGX_SLAB_PAGE_BUSY 0xffffffff
|
|
|
|
#define NGX_SLAB_PAGE_START 0x80000000
|
|
|
|
|
|
|
|
#define NGX_SLAB_SHIFT_MASK 0x0000000f
|
|
|
|
#define NGX_SLAB_MAP_MASK 0xffff0000
|
|
|
|
#define NGX_SLAB_MAP_SHIFT 16
|
|
|
|
|
|
|
|
#define NGX_SLAB_BUSY 0xffffffff
|
|
|
|
|
|
|
|
#else /* (NGX_PTR_SIZE == 8) */
|
|
|
|
|
|
|
|
#define NGX_SLAB_PAGE_FREE 0
|
|
|
|
#define NGX_SLAB_PAGE_BUSY 0xffffffffffffffff
|
|
|
|
#define NGX_SLAB_PAGE_START 0x8000000000000000
|
|
|
|
|
|
|
|
#define NGX_SLAB_SHIFT_MASK 0x000000000000000f
|
|
|
|
#define NGX_SLAB_MAP_MASK 0xffffffff00000000
|
|
|
|
#define NGX_SLAB_MAP_SHIFT 32
|
|
|
|
|
|
|
|
#define NGX_SLAB_BUSY 0xffffffffffffffff
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
|
|
#if (NGX_DEBUG_MALLOC)
|
2007-12-22 19:06:53 +08:00
|
|
|
|
2011-10-24 23:50:19 +08:00
|
|
|
#define ngx_slab_junk(p, size) ngx_memset(p, 0xA5, size)
|
2007-12-22 19:06:53 +08:00
|
|
|
|
2012-07-26 22:47:42 +08:00
|
|
|
#elif (NGX_HAVE_DEBUG_MALLOC)
|
2007-12-22 19:06:53 +08:00
|
|
|
|
|
|
|
#define ngx_slab_junk(p, size) \
|
2011-10-24 23:50:19 +08:00
|
|
|
if (ngx_debug_malloc) ngx_memset(p, 0xA5, size)
|
2007-12-22 19:06:53 +08:00
|
|
|
|
2006-11-20 16:51:45 +08:00
|
|
|
#else
|
2007-12-22 19:06:53 +08:00
|
|
|
|
2006-11-20 16:51:45 +08:00
|
|
|
#define ngx_slab_junk(p, size)
|
2007-12-22 19:06:53 +08:00
|
|
|
|
|
|
|
#endif
|
|
|
|
|
2006-11-20 16:51:45 +08:00
|
|
|
static ngx_slab_page_t *ngx_slab_alloc_pages(ngx_slab_pool_t *pool,
|
|
|
|
ngx_uint_t pages);
|
|
|
|
static void ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page,
|
|
|
|
ngx_uint_t pages);
|
2009-03-28 01:00:42 +08:00
|
|
|
static void ngx_slab_error(ngx_slab_pool_t *pool, ngx_uint_t level,
|
|
|
|
char *text);
|
2006-11-20 16:51:45 +08:00
|
|
|
|
|
|
|
|
|
|
|
static ngx_uint_t ngx_slab_max_size;
|
|
|
|
static ngx_uint_t ngx_slab_exact_size;
|
|
|
|
static ngx_uint_t ngx_slab_exact_shift;
|
|
|
|
|
|
|
|
|
|
|
|
void
|
|
|
|
ngx_slab_init(ngx_slab_pool_t *pool)
|
|
|
|
{
|
|
|
|
u_char *p;
|
|
|
|
size_t size;
|
|
|
|
ngx_int_t m;
|
|
|
|
ngx_uint_t i, n, pages;
|
|
|
|
ngx_slab_page_t *slots;
|
|
|
|
|
|
|
|
/* STUB */
|
|
|
|
if (ngx_slab_max_size == 0) {
|
|
|
|
ngx_slab_max_size = ngx_pagesize / 2;
|
|
|
|
ngx_slab_exact_size = ngx_pagesize / (8 * sizeof(uintptr_t));
|
|
|
|
for (n = ngx_slab_exact_size; n >>= 1; ngx_slab_exact_shift++) {
|
|
|
|
/* void */
|
|
|
|
}
|
|
|
|
}
|
|
|
|
/**/
|
|
|
|
|
|
|
|
pool->min_size = 1 << pool->min_shift;
|
|
|
|
|
|
|
|
p = (u_char *) pool + sizeof(ngx_slab_pool_t);
|
|
|
|
size = pool->end - p;
|
|
|
|
|
|
|
|
ngx_slab_junk(p, size);
|
|
|
|
|
|
|
|
slots = (ngx_slab_page_t *) p;
|
|
|
|
n = ngx_pagesize_shift - pool->min_shift;
|
|
|
|
|
|
|
|
for (i = 0; i < n; i++) {
|
|
|
|
slots[i].slab = 0;
|
|
|
|
slots[i].next = &slots[i];
|
|
|
|
slots[i].prev = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
p += n * sizeof(ngx_slab_page_t);
|
|
|
|
|
2007-11-17 02:15:04 +08:00
|
|
|
pages = (ngx_uint_t) (size / (ngx_pagesize + sizeof(ngx_slab_page_t)));
|
2006-11-20 16:51:45 +08:00
|
|
|
|
|
|
|
ngx_memzero(p, pages * sizeof(ngx_slab_page_t));
|
|
|
|
|
|
|
|
pool->pages = (ngx_slab_page_t *) p;
|
|
|
|
|
|
|
|
pool->free.prev = 0;
|
|
|
|
pool->free.next = (ngx_slab_page_t *) p;
|
|
|
|
|
|
|
|
pool->pages->slab = pages;
|
|
|
|
pool->pages->next = &pool->free;
|
|
|
|
pool->pages->prev = (uintptr_t) &pool->free;
|
|
|
|
|
|
|
|
pool->start = (u_char *)
|
2007-01-12 02:58:40 +08:00
|
|
|
ngx_align_ptr((uintptr_t) p + pages * sizeof(ngx_slab_page_t),
|
2006-11-20 16:51:45 +08:00
|
|
|
ngx_pagesize);
|
|
|
|
|
|
|
|
m = pages - (pool->end - pool->start) / ngx_pagesize;
|
|
|
|
if (m > 0) {
|
|
|
|
pages -= m;
|
|
|
|
pool->pages->slab = pages;
|
|
|
|
}
|
2009-03-28 01:00:42 +08:00
|
|
|
|
2014-06-03 21:53:03 +08:00
|
|
|
pool->last = pool->pages + pages;
|
|
|
|
|
2014-04-01 01:38:30 +08:00
|
|
|
pool->log_nomem = 1;
|
2009-03-28 01:00:42 +08:00
|
|
|
pool->log_ctx = &pool->zero;
|
|
|
|
pool->zero = '\0';
|
2006-11-20 16:51:45 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
void *
|
|
|
|
ngx_slab_alloc(ngx_slab_pool_t *pool, size_t size)
|
2007-01-03 07:10:42 +08:00
|
|
|
{
|
|
|
|
void *p;
|
|
|
|
|
|
|
|
ngx_shmtx_lock(&pool->mutex);
|
|
|
|
|
|
|
|
p = ngx_slab_alloc_locked(pool, size);
|
|
|
|
|
|
|
|
ngx_shmtx_unlock(&pool->mutex);
|
|
|
|
|
|
|
|
return p;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
void *
|
|
|
|
ngx_slab_alloc_locked(ngx_slab_pool_t *pool, size_t size)
|
2006-11-20 16:51:45 +08:00
|
|
|
{
|
|
|
|
size_t s;
|
2006-11-28 04:47:55 +08:00
|
|
|
uintptr_t p, n, m, mask, *bitmap;
|
|
|
|
ngx_uint_t i, slot, shift, map;
|
2006-11-20 16:51:45 +08:00
|
|
|
ngx_slab_page_t *page, *prev, *slots;
|
|
|
|
|
2014-06-18 16:39:59 +08:00
|
|
|
if (size > ngx_slab_max_size) {
|
2007-01-03 07:10:42 +08:00
|
|
|
|
|
|
|
ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
|
2007-01-03 07:12:24 +08:00
|
|
|
"slab alloc: %uz", size);
|
2007-01-03 07:10:42 +08:00
|
|
|
|
2012-08-30 23:09:21 +08:00
|
|
|
page = ngx_slab_alloc_pages(pool, (size >> ngx_pagesize_shift)
|
|
|
|
+ ((size % ngx_pagesize) ? 1 : 0));
|
2006-11-20 16:51:45 +08:00
|
|
|
if (page) {
|
|
|
|
p = (page - pool->pages) << ngx_pagesize_shift;
|
|
|
|
p += (uintptr_t) pool->start;
|
|
|
|
|
|
|
|
} else {
|
|
|
|
p = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (size > pool->min_size) {
|
|
|
|
shift = 1;
|
|
|
|
for (s = size - 1; s >>= 1; shift++) { /* void */ }
|
|
|
|
slot = shift - pool->min_shift;
|
|
|
|
|
|
|
|
} else {
|
|
|
|
size = pool->min_size;
|
|
|
|
shift = pool->min_shift;
|
|
|
|
slot = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
ngx_log_debug2(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
|
|
|
|
"slab alloc: %uz slot: %ui", size, slot);
|
|
|
|
|
|
|
|
slots = (ngx_slab_page_t *) ((u_char *) pool + sizeof(ngx_slab_pool_t));
|
|
|
|
page = slots[slot].next;
|
|
|
|
|
|
|
|
if (page->next != page) {
|
|
|
|
|
2007-01-03 07:10:42 +08:00
|
|
|
if (shift < ngx_slab_exact_shift) {
|
2006-11-20 16:51:45 +08:00
|
|
|
|
|
|
|
do {
|
|
|
|
p = (page - pool->pages) << ngx_pagesize_shift;
|
|
|
|
bitmap = (uintptr_t *) (pool->start + p);
|
|
|
|
|
|
|
|
map = (1 << (ngx_pagesize_shift - shift))
|
|
|
|
/ (sizeof(uintptr_t) * 8);
|
|
|
|
|
|
|
|
for (n = 0; n < map; n++) {
|
|
|
|
|
|
|
|
if (bitmap[n] != NGX_SLAB_BUSY) {
|
|
|
|
|
|
|
|
for (m = 1, i = 0; m; m <<= 1, i++) {
|
|
|
|
if ((bitmap[n] & m)) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
bitmap[n] |= m;
|
2007-01-03 07:10:42 +08:00
|
|
|
|
|
|
|
i = ((n * sizeof(uintptr_t) * 8) << shift)
|
|
|
|
+ (i << shift);
|
2006-11-20 16:51:45 +08:00
|
|
|
|
|
|
|
if (bitmap[n] == NGX_SLAB_BUSY) {
|
|
|
|
for (n = n + 1; n < map; n++) {
|
2016-03-30 16:52:16 +08:00
|
|
|
if (bitmap[n] != NGX_SLAB_BUSY) {
|
|
|
|
p = (uintptr_t) bitmap + i;
|
2006-11-20 16:51:45 +08:00
|
|
|
|
2016-03-30 16:52:16 +08:00
|
|
|
goto done;
|
|
|
|
}
|
2006-11-20 16:51:45 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
prev = (ngx_slab_page_t *)
|
|
|
|
(page->prev & ~NGX_SLAB_PAGE_MASK);
|
|
|
|
prev->next = page->next;
|
|
|
|
page->next->prev = page->prev;
|
|
|
|
|
|
|
|
page->next = NULL;
|
|
|
|
page->prev = NGX_SLAB_SMALL;
|
|
|
|
}
|
|
|
|
|
|
|
|
p = (uintptr_t) bitmap + i;
|
|
|
|
|
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
page = page->next;
|
|
|
|
|
|
|
|
} while (page);
|
|
|
|
|
2007-01-03 07:10:42 +08:00
|
|
|
} else if (shift == ngx_slab_exact_shift) {
|
2006-11-20 16:51:45 +08:00
|
|
|
|
|
|
|
do {
|
|
|
|
if (page->slab != NGX_SLAB_BUSY) {
|
|
|
|
|
|
|
|
for (m = 1, i = 0; m; m <<= 1, i++) {
|
|
|
|
if ((page->slab & m)) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
page->slab |= m;
|
|
|
|
|
|
|
|
if (page->slab == NGX_SLAB_BUSY) {
|
|
|
|
prev = (ngx_slab_page_t *)
|
|
|
|
(page->prev & ~NGX_SLAB_PAGE_MASK);
|
|
|
|
prev->next = page->next;
|
|
|
|
page->next->prev = page->prev;
|
|
|
|
|
|
|
|
page->next = NULL;
|
|
|
|
page->prev = NGX_SLAB_EXACT;
|
|
|
|
}
|
|
|
|
|
|
|
|
p = (page - pool->pages) << ngx_pagesize_shift;
|
|
|
|
p += i << shift;
|
|
|
|
p += (uintptr_t) pool->start;
|
|
|
|
|
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
page = page->next;
|
|
|
|
|
|
|
|
} while (page);
|
|
|
|
|
2007-01-03 07:10:42 +08:00
|
|
|
} else { /* shift > ngx_slab_exact_shift */
|
2006-11-20 16:51:45 +08:00
|
|
|
|
|
|
|
n = ngx_pagesize_shift - (page->slab & NGX_SLAB_SHIFT_MASK);
|
|
|
|
n = 1 << n;
|
2007-01-12 02:58:40 +08:00
|
|
|
n = ((uintptr_t) 1 << n) - 1;
|
2006-11-20 16:51:45 +08:00
|
|
|
mask = n << NGX_SLAB_MAP_SHIFT;
|
|
|
|
|
|
|
|
do {
|
|
|
|
if ((page->slab & NGX_SLAB_MAP_MASK) != mask) {
|
|
|
|
|
2006-11-28 04:47:55 +08:00
|
|
|
for (m = (uintptr_t) 1 << NGX_SLAB_MAP_SHIFT, i = 0;
|
2006-11-20 16:51:45 +08:00
|
|
|
m & mask;
|
|
|
|
m <<= 1, i++)
|
|
|
|
{
|
|
|
|
if ((page->slab & m)) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
page->slab |= m;
|
|
|
|
|
|
|
|
if ((page->slab & NGX_SLAB_MAP_MASK) == mask) {
|
|
|
|
prev = (ngx_slab_page_t *)
|
|
|
|
(page->prev & ~NGX_SLAB_PAGE_MASK);
|
|
|
|
prev->next = page->next;
|
|
|
|
page->next->prev = page->prev;
|
|
|
|
|
|
|
|
page->next = NULL;
|
|
|
|
page->prev = NGX_SLAB_BIG;
|
|
|
|
}
|
|
|
|
|
|
|
|
p = (page - pool->pages) << ngx_pagesize_shift;
|
|
|
|
p += i << shift;
|
|
|
|
p += (uintptr_t) pool->start;
|
|
|
|
|
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
page = page->next;
|
|
|
|
|
|
|
|
} while (page);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
page = ngx_slab_alloc_pages(pool, 1);
|
|
|
|
|
|
|
|
if (page) {
|
2007-01-03 07:10:42 +08:00
|
|
|
if (shift < ngx_slab_exact_shift) {
|
2006-11-20 16:51:45 +08:00
|
|
|
p = (page - pool->pages) << ngx_pagesize_shift;
|
|
|
|
bitmap = (uintptr_t *) (pool->start + p);
|
|
|
|
|
|
|
|
s = 1 << shift;
|
|
|
|
n = (1 << (ngx_pagesize_shift - shift)) / 8 / s;
|
|
|
|
|
|
|
|
if (n == 0) {
|
|
|
|
n = 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
bitmap[0] = (2 << n) - 1;
|
|
|
|
|
|
|
|
map = (1 << (ngx_pagesize_shift - shift)) / (sizeof(uintptr_t) * 8);
|
|
|
|
|
|
|
|
for (i = 1; i < map; i++) {
|
|
|
|
bitmap[i] = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
page->slab = shift;
|
|
|
|
page->next = &slots[slot];
|
|
|
|
page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL;
|
|
|
|
|
|
|
|
slots[slot].next = page;
|
|
|
|
|
|
|
|
p = ((page - pool->pages) << ngx_pagesize_shift) + s * n;
|
|
|
|
p += (uintptr_t) pool->start;
|
|
|
|
|
|
|
|
goto done;
|
|
|
|
|
2007-01-03 07:10:42 +08:00
|
|
|
} else if (shift == ngx_slab_exact_shift) {
|
2006-11-20 16:51:45 +08:00
|
|
|
|
|
|
|
page->slab = 1;
|
|
|
|
page->next = &slots[slot];
|
|
|
|
page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT;
|
|
|
|
|
|
|
|
slots[slot].next = page;
|
|
|
|
|
|
|
|
p = (page - pool->pages) << ngx_pagesize_shift;
|
|
|
|
p += (uintptr_t) pool->start;
|
2004-06-17 03:36:07 +08:00
|
|
|
|
2006-11-20 16:51:45 +08:00
|
|
|
goto done;
|
|
|
|
|
2007-01-03 07:10:42 +08:00
|
|
|
} else { /* shift > ngx_slab_exact_shift */
|
2006-11-20 16:51:45 +08:00
|
|
|
|
2006-11-28 04:47:55 +08:00
|
|
|
page->slab = ((uintptr_t) 1 << NGX_SLAB_MAP_SHIFT) | shift;
|
2006-11-20 16:51:45 +08:00
|
|
|
page->next = &slots[slot];
|
|
|
|
page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG;
|
|
|
|
|
|
|
|
slots[slot].next = page;
|
|
|
|
|
|
|
|
p = (page - pool->pages) << ngx_pagesize_shift;
|
|
|
|
p += (uintptr_t) pool->start;
|
|
|
|
|
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
p = 0;
|
|
|
|
|
|
|
|
done:
|
|
|
|
|
2016-03-31 07:33:57 +08:00
|
|
|
ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
|
|
|
|
"slab alloc: %p", (void *) p);
|
2006-11-20 16:51:45 +08:00
|
|
|
|
|
|
|
return (void *) p;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2014-06-04 19:09:19 +08:00
|
|
|
void *
|
|
|
|
ngx_slab_calloc(ngx_slab_pool_t *pool, size_t size)
|
|
|
|
{
|
|
|
|
void *p;
|
|
|
|
|
|
|
|
ngx_shmtx_lock(&pool->mutex);
|
|
|
|
|
|
|
|
p = ngx_slab_calloc_locked(pool, size);
|
|
|
|
|
|
|
|
ngx_shmtx_unlock(&pool->mutex);
|
|
|
|
|
|
|
|
return p;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
void *
|
|
|
|
ngx_slab_calloc_locked(ngx_slab_pool_t *pool, size_t size)
|
|
|
|
{
|
|
|
|
void *p;
|
|
|
|
|
|
|
|
p = ngx_slab_alloc_locked(pool, size);
|
|
|
|
if (p) {
|
|
|
|
ngx_memzero(p, size);
|
|
|
|
}
|
|
|
|
|
|
|
|
return p;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2006-11-20 16:51:45 +08:00
|
|
|
void
|
|
|
|
ngx_slab_free(ngx_slab_pool_t *pool, void *p)
|
2007-01-03 07:10:42 +08:00
|
|
|
{
|
|
|
|
ngx_shmtx_lock(&pool->mutex);
|
|
|
|
|
|
|
|
ngx_slab_free_locked(pool, p);
|
|
|
|
|
|
|
|
ngx_shmtx_unlock(&pool->mutex);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
void
|
|
|
|
ngx_slab_free_locked(ngx_slab_pool_t *pool, void *p)
|
2004-09-28 16:34:51 +08:00
|
|
|
{
|
2006-11-20 16:51:45 +08:00
|
|
|
size_t size;
|
2007-01-12 03:13:46 +08:00
|
|
|
uintptr_t slab, m, *bitmap;
|
|
|
|
ngx_uint_t n, type, slot, shift, map;
|
2006-11-20 16:51:45 +08:00
|
|
|
ngx_slab_page_t *slots, *page;
|
|
|
|
|
|
|
|
ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, "slab free: %p", p);
|
|
|
|
|
|
|
|
if ((u_char *) p < pool->start || (u_char *) p > pool->end) {
|
2009-03-28 01:00:42 +08:00
|
|
|
ngx_slab_error(pool, NGX_LOG_ALERT, "ngx_slab_free(): outside of pool");
|
2006-11-20 16:51:45 +08:00
|
|
|
goto fail;
|
|
|
|
}
|
|
|
|
|
|
|
|
n = ((u_char *) p - pool->start) >> ngx_pagesize_shift;
|
|
|
|
page = &pool->pages[n];
|
|
|
|
slab = page->slab;
|
|
|
|
type = page->prev & NGX_SLAB_PAGE_MASK;
|
|
|
|
|
|
|
|
switch (type) {
|
|
|
|
|
|
|
|
case NGX_SLAB_SMALL:
|
|
|
|
|
|
|
|
shift = slab & NGX_SLAB_SHIFT_MASK;
|
|
|
|
size = 1 << shift;
|
|
|
|
|
|
|
|
if ((uintptr_t) p & (size - 1)) {
|
|
|
|
goto wrong_chunk;
|
|
|
|
}
|
|
|
|
|
|
|
|
n = ((uintptr_t) p & (ngx_pagesize - 1)) >> shift;
|
2007-01-12 02:58:40 +08:00
|
|
|
m = (uintptr_t) 1 << (n & (sizeof(uintptr_t) * 8 - 1));
|
2006-11-20 16:51:45 +08:00
|
|
|
n /= (sizeof(uintptr_t) * 8);
|
2013-09-05 01:16:59 +08:00
|
|
|
bitmap = (uintptr_t *)
|
|
|
|
((uintptr_t) p & ~((uintptr_t) ngx_pagesize - 1));
|
2006-11-20 16:51:45 +08:00
|
|
|
|
|
|
|
if (bitmap[n] & m) {
|
|
|
|
|
|
|
|
if (page->next == NULL) {
|
|
|
|
slots = (ngx_slab_page_t *)
|
|
|
|
((u_char *) pool + sizeof(ngx_slab_pool_t));
|
|
|
|
slot = shift - pool->min_shift;
|
|
|
|
|
|
|
|
page->next = slots[slot].next;
|
|
|
|
slots[slot].next = page;
|
|
|
|
|
|
|
|
page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL;
|
|
|
|
page->next->prev = (uintptr_t) page | NGX_SLAB_SMALL;
|
|
|
|
}
|
|
|
|
|
|
|
|
bitmap[n] &= ~m;
|
|
|
|
|
|
|
|
n = (1 << (ngx_pagesize_shift - shift)) / 8 / (1 << shift);
|
|
|
|
|
|
|
|
if (n == 0) {
|
|
|
|
n = 1;
|
|
|
|
}
|
|
|
|
|
2007-01-12 02:58:40 +08:00
|
|
|
if (bitmap[0] & ~(((uintptr_t) 1 << n) - 1)) {
|
2006-11-20 16:51:45 +08:00
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
|
|
|
|
map = (1 << (ngx_pagesize_shift - shift)) / (sizeof(uintptr_t) * 8);
|
|
|
|
|
|
|
|
for (n = 1; n < map; n++) {
|
|
|
|
if (bitmap[n]) {
|
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
ngx_slab_free_pages(pool, page, 1);
|
|
|
|
|
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
|
|
|
|
goto chunk_already_free;
|
|
|
|
|
|
|
|
case NGX_SLAB_EXACT:
|
|
|
|
|
2007-01-12 02:58:40 +08:00
|
|
|
m = (uintptr_t) 1 <<
|
|
|
|
(((uintptr_t) p & (ngx_pagesize - 1)) >> ngx_slab_exact_shift);
|
2006-11-20 16:51:45 +08:00
|
|
|
size = ngx_slab_exact_size;
|
|
|
|
|
|
|
|
if ((uintptr_t) p & (size - 1)) {
|
|
|
|
goto wrong_chunk;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (slab & m) {
|
|
|
|
if (slab == NGX_SLAB_BUSY) {
|
|
|
|
slots = (ngx_slab_page_t *)
|
|
|
|
((u_char *) pool + sizeof(ngx_slab_pool_t));
|
|
|
|
slot = ngx_slab_exact_shift - pool->min_shift;
|
|
|
|
|
|
|
|
page->next = slots[slot].next;
|
|
|
|
slots[slot].next = page;
|
|
|
|
|
|
|
|
page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT;
|
|
|
|
page->next->prev = (uintptr_t) page | NGX_SLAB_EXACT;
|
|
|
|
}
|
|
|
|
|
|
|
|
page->slab &= ~m;
|
|
|
|
|
|
|
|
if (page->slab) {
|
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
|
|
|
|
ngx_slab_free_pages(pool, page, 1);
|
|
|
|
|
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
|
|
|
|
goto chunk_already_free;
|
|
|
|
|
|
|
|
case NGX_SLAB_BIG:
|
|
|
|
|
|
|
|
shift = slab & NGX_SLAB_SHIFT_MASK;
|
|
|
|
size = 1 << shift;
|
|
|
|
|
|
|
|
if ((uintptr_t) p & (size - 1)) {
|
|
|
|
goto wrong_chunk;
|
|
|
|
}
|
|
|
|
|
2007-01-12 02:58:40 +08:00
|
|
|
m = (uintptr_t) 1 << ((((uintptr_t) p & (ngx_pagesize - 1)) >> shift)
|
|
|
|
+ NGX_SLAB_MAP_SHIFT);
|
2006-11-20 16:51:45 +08:00
|
|
|
|
|
|
|
if (slab & m) {
|
|
|
|
|
|
|
|
if (page->next == NULL) {
|
|
|
|
slots = (ngx_slab_page_t *)
|
|
|
|
((u_char *) pool + sizeof(ngx_slab_pool_t));
|
|
|
|
slot = shift - pool->min_shift;
|
|
|
|
|
|
|
|
page->next = slots[slot].next;
|
|
|
|
slots[slot].next = page;
|
|
|
|
|
|
|
|
page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG;
|
|
|
|
page->next->prev = (uintptr_t) page | NGX_SLAB_BIG;
|
|
|
|
}
|
|
|
|
|
|
|
|
page->slab &= ~m;
|
|
|
|
|
|
|
|
if (page->slab & NGX_SLAB_MAP_MASK) {
|
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
|
|
|
|
ngx_slab_free_pages(pool, page, 1);
|
|
|
|
|
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
|
|
|
|
goto chunk_already_free;
|
|
|
|
|
|
|
|
case NGX_SLAB_PAGE:
|
|
|
|
|
|
|
|
if ((uintptr_t) p & (ngx_pagesize - 1)) {
|
|
|
|
goto wrong_chunk;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (slab == NGX_SLAB_PAGE_FREE) {
|
2009-03-28 01:00:42 +08:00
|
|
|
ngx_slab_error(pool, NGX_LOG_ALERT,
|
|
|
|
"ngx_slab_free(): page is already free");
|
2006-11-20 16:51:45 +08:00
|
|
|
goto fail;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (slab == NGX_SLAB_PAGE_BUSY) {
|
2009-03-28 01:00:42 +08:00
|
|
|
ngx_slab_error(pool, NGX_LOG_ALERT,
|
|
|
|
"ngx_slab_free(): pointer to wrong page");
|
2006-11-20 16:51:45 +08:00
|
|
|
goto fail;
|
|
|
|
}
|
|
|
|
|
|
|
|
n = ((u_char *) p - pool->start) >> ngx_pagesize_shift;
|
|
|
|
size = slab & ~NGX_SLAB_PAGE_START;
|
|
|
|
|
|
|
|
ngx_slab_free_pages(pool, &pool->pages[n], size);
|
|
|
|
|
2009-04-16 03:44:41 +08:00
|
|
|
ngx_slab_junk(p, size << ngx_pagesize_shift);
|
2006-11-20 16:51:45 +08:00
|
|
|
|
2009-04-16 03:44:41 +08:00
|
|
|
return;
|
2006-11-20 16:51:45 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/* not reached */
|
|
|
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
done:
|
|
|
|
|
|
|
|
ngx_slab_junk(p, size);
|
|
|
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
wrong_chunk:
|
|
|
|
|
2009-03-28 01:00:42 +08:00
|
|
|
ngx_slab_error(pool, NGX_LOG_ALERT,
|
|
|
|
"ngx_slab_free(): pointer to wrong chunk");
|
2006-11-20 16:51:45 +08:00
|
|
|
|
|
|
|
goto fail;
|
|
|
|
|
|
|
|
chunk_already_free:
|
|
|
|
|
2009-03-28 01:00:42 +08:00
|
|
|
ngx_slab_error(pool, NGX_LOG_ALERT,
|
|
|
|
"ngx_slab_free(): chunk is already free");
|
2006-11-20 16:51:45 +08:00
|
|
|
|
|
|
|
fail:
|
|
|
|
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
static ngx_slab_page_t *
|
|
|
|
ngx_slab_alloc_pages(ngx_slab_pool_t *pool, ngx_uint_t pages)
|
|
|
|
{
|
|
|
|
ngx_slab_page_t *page, *p;
|
|
|
|
|
|
|
|
for (page = pool->free.next; page != &pool->free; page = page->next) {
|
|
|
|
|
|
|
|
if (page->slab >= pages) {
|
|
|
|
|
|
|
|
if (page->slab > pages) {
|
2014-06-03 21:53:03 +08:00
|
|
|
page[page->slab - 1].prev = (uintptr_t) &page[pages];
|
|
|
|
|
2006-11-20 16:51:45 +08:00
|
|
|
page[pages].slab = page->slab - pages;
|
|
|
|
page[pages].next = page->next;
|
|
|
|
page[pages].prev = page->prev;
|
|
|
|
|
|
|
|
p = (ngx_slab_page_t *) page->prev;
|
|
|
|
p->next = &page[pages];
|
|
|
|
page->next->prev = (uintptr_t) &page[pages];
|
|
|
|
|
|
|
|
} else {
|
|
|
|
p = (ngx_slab_page_t *) page->prev;
|
|
|
|
p->next = page->next;
|
|
|
|
page->next->prev = page->prev;
|
|
|
|
}
|
|
|
|
|
|
|
|
page->slab = pages | NGX_SLAB_PAGE_START;
|
|
|
|
page->next = NULL;
|
|
|
|
page->prev = NGX_SLAB_PAGE;
|
|
|
|
|
|
|
|
if (--pages == 0) {
|
|
|
|
return page;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (p = page + 1; pages; pages--) {
|
|
|
|
p->slab = NGX_SLAB_PAGE_BUSY;
|
|
|
|
p->next = NULL;
|
|
|
|
p->prev = NGX_SLAB_PAGE;
|
|
|
|
p++;
|
|
|
|
}
|
|
|
|
|
|
|
|
return page;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-04-01 01:38:30 +08:00
|
|
|
if (pool->log_nomem) {
|
|
|
|
ngx_slab_error(pool, NGX_LOG_CRIT,
|
|
|
|
"ngx_slab_alloc() failed: no memory");
|
|
|
|
}
|
2007-01-03 07:10:42 +08:00
|
|
|
|
2006-11-20 16:51:45 +08:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
static void
|
|
|
|
ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page,
|
|
|
|
ngx_uint_t pages)
|
|
|
|
{
|
2014-06-03 21:53:03 +08:00
|
|
|
ngx_uint_t type;
|
|
|
|
ngx_slab_page_t *prev, *join;
|
2006-11-20 16:51:45 +08:00
|
|
|
|
|
|
|
page->slab = pages--;
|
|
|
|
|
|
|
|
if (pages) {
|
|
|
|
ngx_memzero(&page[1], pages * sizeof(ngx_slab_page_t));
|
|
|
|
}
|
|
|
|
|
2007-01-03 07:10:42 +08:00
|
|
|
if (page->next) {
|
|
|
|
prev = (ngx_slab_page_t *) (page->prev & ~NGX_SLAB_PAGE_MASK);
|
|
|
|
prev->next = page->next;
|
|
|
|
page->next->prev = page->prev;
|
|
|
|
}
|
2006-11-20 16:51:45 +08:00
|
|
|
|
2014-06-03 21:53:03 +08:00
|
|
|
join = page + page->slab;
|
|
|
|
|
|
|
|
if (join < pool->last) {
|
|
|
|
type = join->prev & NGX_SLAB_PAGE_MASK;
|
|
|
|
|
|
|
|
if (type == NGX_SLAB_PAGE) {
|
|
|
|
|
|
|
|
if (join->next != NULL) {
|
|
|
|
pages += join->slab;
|
|
|
|
page->slab += join->slab;
|
|
|
|
|
|
|
|
prev = (ngx_slab_page_t *) (join->prev & ~NGX_SLAB_PAGE_MASK);
|
|
|
|
prev->next = join->next;
|
|
|
|
join->next->prev = join->prev;
|
|
|
|
|
|
|
|
join->slab = NGX_SLAB_PAGE_FREE;
|
|
|
|
join->next = NULL;
|
|
|
|
join->prev = NGX_SLAB_PAGE;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (page > pool->pages) {
|
|
|
|
join = page - 1;
|
|
|
|
type = join->prev & NGX_SLAB_PAGE_MASK;
|
|
|
|
|
|
|
|
if (type == NGX_SLAB_PAGE) {
|
|
|
|
|
|
|
|
if (join->slab == NGX_SLAB_PAGE_FREE) {
|
|
|
|
join = (ngx_slab_page_t *) (join->prev & ~NGX_SLAB_PAGE_MASK);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (join->next != NULL) {
|
|
|
|
pages += join->slab;
|
|
|
|
join->slab += page->slab;
|
|
|
|
|
|
|
|
prev = (ngx_slab_page_t *) (join->prev & ~NGX_SLAB_PAGE_MASK);
|
|
|
|
prev->next = join->next;
|
|
|
|
join->next->prev = join->prev;
|
|
|
|
|
|
|
|
page->slab = NGX_SLAB_PAGE_FREE;
|
|
|
|
page->next = NULL;
|
|
|
|
page->prev = NGX_SLAB_PAGE;
|
|
|
|
|
|
|
|
page = join;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (pages) {
|
|
|
|
page[pages].prev = (uintptr_t) page;
|
|
|
|
}
|
|
|
|
|
2007-01-03 07:10:42 +08:00
|
|
|
page->prev = (uintptr_t) &pool->free;
|
2006-11-20 16:51:45 +08:00
|
|
|
page->next = pool->free.next;
|
|
|
|
|
|
|
|
page->next->prev = (uintptr_t) page;
|
2007-01-03 07:10:42 +08:00
|
|
|
|
|
|
|
pool->free.next = page;
|
2004-06-17 03:36:07 +08:00
|
|
|
}
|
2009-03-28 01:00:42 +08:00
|
|
|
|
|
|
|
|
|
|
|
static void
|
|
|
|
ngx_slab_error(ngx_slab_pool_t *pool, ngx_uint_t level, char *text)
|
|
|
|
{
|
|
|
|
ngx_log_error(level, ngx_cycle->log, 0, "%s%s", text, pool->log_ctx);
|
|
|
|
}
|