Add align to struct mg_iobuf

This commit is contained in:
Sergey Lyubka 2022-08-01 12:53:25 +01:00
parent a468f58128
commit 17e2fdd4cb
12 changed files with 92 additions and 74 deletions

View File

@ -3493,6 +3493,7 @@ struct mg_iobuf {
unsigned char *buf; // Pointer to stored data
size_t size; // Total size available
size_t len; // Current number of bytes
size_t align; // Alignment during allocation
};
```
@ -3504,7 +3505,7 @@ by `buf`, and `len` specifies number of bytes currently stored.
### mg\_iobuf\_init()
```c
int mg_iobuf_init(struct mg_iobuf *io, size_t size);
int mg_iobuf_init(struct mg_iobuf *io, size_t size, size_t align);
```
Initialize IO buffer, allocate `size` bytes.
@ -3512,6 +3513,7 @@ Initialize IO buffer, allocate `size` bytes.
Parameters:
- `io` - Pointer to `mg_iobuf` structure to initialize
- `size` - Amount of bytes to allocate
- `align` - Align `size` to the `align` mem boundary. `0` means no alignment
Return value: 1 on success, 0 on allocation failure
@ -3519,7 +3521,7 @@ Usage example:
```c
struct mg_iobuf io;
if (mg_iobuf_init(&io)) {
if (mg_iobuf_init(&io, 0, 64)) {
// io successfully initialized
}
```
@ -3533,6 +3535,9 @@ int mg_iobuf_resize(struct mg_iobuf *io, size_t size);
Resize IO buffer, set the new size to `size`. The `io->buf` pointer could
change after this, for example if the buffer grows. If `size` is 0, then the
`io->buf` is freed and set to NULL, and both `size` and `len` are set to 0.
The resulting `io->size` is always aligned to the `io->align` byte boundary;
therefore, to avoid memory fragmentation and frequent reallocations, set
`io->align` to a higher value.
Parameters:
- `io` - iobuf to resize
@ -3544,11 +3549,10 @@ Usage example:
```c
struct mg_iobuf io;
// IO buffer initialization
// ...
mg_iobuf_init(&io, 0, 10); // An empty buffer with 10-byte alignment
if (mg_iobuf_resize(&io, 1024)) {
// New io size is 1024 bytes
if (mg_iobuf_resize(&io, 1)) {
// New io size is 10
}
```
@ -3579,11 +3583,11 @@ mg_iobuf_free(&io);
### mg\_iobuf\_add()
```c
size_t mg_iobuf_add(struct mg_iobuf *io, size_t offset, const void *buf, size_t len, size_t align);
size_t mg_iobuf_add(struct mg_iobuf *io, size_t offset, const void *buf, size_t len);
```
Insert data buffer `buf`, `len` at offset `offset`. The iobuf is expanded
if required. The resulting `io->size` is always aligned to the `align` byte boundary; therefore,
if required. The resulting `io->size` is always aligned to the `io->align` byte boundary; therefore,
to avoid memory fragmentation and frequent reallocations, set `align` to a higher value.
Parameters:
@ -3591,21 +3595,20 @@ Parameters:
- `offset` - Offset to add data
- `buf` - Data to add
- `len` - Data length
- `align` - Align boundary
Return value: new `io` length
Usage example:
```c
struct mg_iobuf io; // Declare buffer
mg_iobuf_init(&io, 0); // Initialise buffer to have 0 allocated bytes
struct mg_iobuf io; // Declare buffer
mg_iobuf_init(&io, 0, 16); // Initialise empty buffer with 16 byte alignment
```
<img src="images/mg_iobuf_add1.svg" alt="Function mg_iobuf_init()" />
```c
mg_iobuf_add(&io, io.len, "hello", 5, 16); // Append "hello"
mg_iobuf_add(&io, io.len, "hello", 5); // Append "hello"
```
<img src="images/mg_iobuf_add2.svg" alt="Function mg_iobuf_add()" />
@ -3631,11 +3634,9 @@ Usage example:
```c
struct mg_iobuf io;
mg_iobuf_init(&io, 0); // Empty buffer
mg_iobuf_add(&io, 0, "hi", 2, 512); // io->len is 2, io->size is 512
// ...
mg_iobuf_del(&io, 0, "hi", 2, 512); // io->len is 0, io->size is still 512
mg_iobuf_init(&io, 0, 512); // Empty buffer
mg_iobuf_add(&io, 0, "hi", 2); // io->len is 2, io->size is 512
mg_iobuf_del(&io, 0, "hi", 2); // io->len is 0, io->size is still 512
```
<img src="images/mg_iobug_del.png" alt="Function mg_iobuf_del()" />

View File

@ -799,7 +799,7 @@ bool mg_send(struct mg_connection *c, const void *buf, size_t len) {
res = true;
} else {
// tx_tdp(ifp, ifp->ip, c->loc.port, c->rem.ip, c->rem.port, buf, len);
return mg_iobuf_add(&c->send, c->send.len, buf, len, MG_IO_SIZE);
return mg_iobuf_add(&c->send, c->send.len, buf, len);
}
return res;
}

View File

@ -487,7 +487,7 @@ static void mg_putchar_iobuf_static(char ch, void *param) {
void mg_pfn_iobuf(char ch, void *param) {
struct mg_iobuf *io = (struct mg_iobuf *) param;
if (io->len + 2 > io->size) mg_iobuf_resize(io, io->size + 64);
if (io->len + 2 > io->size) mg_iobuf_resize(io, io->len + 2);
if (io->len + 2 <= io->size) {
io->buf[io->len++] = (uint8_t) ch;
io->buf[io->len] = 0;
@ -509,9 +509,10 @@ void mg_pfn_realloc(char ch, void *param) {
}
size_t mg_vsnprintf(char *buf, size_t len, const char *fmt, va_list *ap) {
struct mg_iobuf io = {(uint8_t *) buf, len, 0};
struct mg_iobuf io = {(uint8_t *) buf, len, 0, 0};
size_t n = mg_vrprintf(mg_putchar_iobuf_static, &io, fmt, ap);
if (n < len) buf[n] = '\0';
// if (len > 0) buf[len - 1] = '\0'; // Guarantee to 0-terminate
return n;
}
@ -2376,8 +2377,13 @@ static void zeromem(volatile unsigned char *buf, size_t len) {
}
}
static size_t roundup(size_t size, size_t align) {
return align == 0 ? size : (size + align - 1) / align * align;
}
int mg_iobuf_resize(struct mg_iobuf *io, size_t new_size) {
int ok = 1;
new_size = roundup(new_size, io->align);
if (new_size == 0) {
zeromem(io->buf, io->size);
free(io->buf);
@ -2402,21 +2408,18 @@ int mg_iobuf_resize(struct mg_iobuf *io, size_t new_size) {
return ok;
}
int mg_iobuf_init(struct mg_iobuf *io, size_t size) {
int mg_iobuf_init(struct mg_iobuf *io, size_t size, size_t align) {
io->buf = NULL;
io->align = align;
io->size = io->len = 0;
return mg_iobuf_resize(io, size);
}
size_t mg_iobuf_add(struct mg_iobuf *io, size_t ofs, const void *buf,
size_t len, size_t chunk_size) {
size_t new_size = io->len + len;
if (new_size > io->size) {
new_size += chunk_size; // Make sure that io->size
new_size -= new_size % chunk_size; // is aligned by chunk_size boundary
mg_iobuf_resize(io, new_size); // Attempt to realloc
if (new_size != io->size) len = 0; // Realloc failure, append nothing
}
size_t len) {
size_t new_size = roundup(io->len + len, io->align);
mg_iobuf_resize(io, new_size); // Attempt to resize
if (new_size != io->size) len = 0; // Resize failure, append nothing
if (ofs < io->len) memmove(io->buf + ofs + len, io->buf + ofs, io->len - ofs);
if (buf != NULL) memmove(io->buf + ofs, buf, len);
if (ofs > io->len) io->len += ofs - io->len;
@ -3426,6 +3429,7 @@ struct mg_connection *mg_alloc_conn(struct mg_mgr *mgr) {
(struct mg_connection *) calloc(1, sizeof(*c) + mgr->extraconnsize);
if (c != NULL) {
c->mgr = mgr;
c->send.align = c->recv.align = MG_IO_SIZE;
c->id = ++mgr->nextid;
}
return c;
@ -4106,7 +4110,7 @@ bool mg_send(struct mg_connection *c, const void *buf, size_t len) {
iolog(c, (char *) buf, n, false);
return n > 0;
} else {
return mg_iobuf_add(&c->send, c->send.len, buf, len, MG_IO_SIZE);
return mg_iobuf_add(&c->send, c->send.len, buf, len);
}
}
@ -4593,12 +4597,12 @@ void mg_mgr_poll(struct mg_mgr *mgr, int ms) {
#if MG_ENABLE_SSI
static char *mg_ssi(const char *path, const char *root, int depth) {
struct mg_iobuf b = {NULL, 0, 0};
struct mg_iobuf b = {NULL, 0, 0, MG_IO_SIZE};
FILE *fp = fopen(path, "rb");
if (fp != NULL) {
char buf[MG_SSI_BUFSIZ], arg[sizeof(buf)];
int ch, intag = 0;
size_t len = 0, align = MG_IO_SIZE;
size_t len = 0;
buf[0] = arg[0] = '\0';
while ((ch = fgetc(fp)) != EOF) {
if (intag && ch == '>' && buf[len - 1] == '-' && buf[len - 2] == '-') {
@ -4611,7 +4615,7 @@ static char *mg_ssi(const char *path, const char *root, int depth) {
mg_snprintf(tmp, sizeof(tmp), "%.*s%s", (int) (p - path), path, arg);
if (depth < MG_MAX_SSI_DEPTH &&
(data = mg_ssi(tmp, root, depth + 1)) != NULL) {
mg_iobuf_add(&b, b.len, data, strlen(data), align);
mg_iobuf_add(&b, b.len, data, strlen(data));
free(data);
} else {
MG_ERROR(("%s: file=%s error or too deep", path, arg));
@ -4621,7 +4625,7 @@ static char *mg_ssi(const char *path, const char *root, int depth) {
mg_snprintf(tmp, sizeof(tmp), "%s%s", root, arg);
if (depth < MG_MAX_SSI_DEPTH &&
(data = mg_ssi(tmp, root, depth + 1)) != NULL) {
mg_iobuf_add(&b, b.len, data, strlen(data), align);
mg_iobuf_add(&b, b.len, data, strlen(data));
free(data);
} else {
MG_ERROR(("%s: virtual=%s error or too deep", path, arg));
@ -4629,13 +4633,13 @@ static char *mg_ssi(const char *path, const char *root, int depth) {
} else {
// Unknown SSI tag
MG_ERROR(("Unknown SSI tag: %.*s", (int) len, buf));
mg_iobuf_add(&b, b.len, buf, len, align);
mg_iobuf_add(&b, b.len, buf, len);
}
intag = 0;
len = 0;
} else if (ch == '<') {
intag = 1;
if (len > 0) mg_iobuf_add(&b, b.len, buf, len, align);
if (len > 0) mg_iobuf_add(&b, b.len, buf, len);
len = 0;
buf[len++] = (char) (ch & 0xff);
} else if (intag) {
@ -4649,13 +4653,13 @@ static char *mg_ssi(const char *path, const char *root, int depth) {
} else {
buf[len++] = (char) (ch & 0xff);
if (len >= sizeof(buf)) {
mg_iobuf_add(&b, b.len, buf, len, align);
mg_iobuf_add(&b, b.len, buf, len);
len = 0;
}
}
}
if (len > 0) mg_iobuf_add(&b, b.len, buf, len, align);
if (b.len > 0) mg_iobuf_add(&b, b.len, "", 1, align); // nul-terminate
if (len > 0) mg_iobuf_add(&b, b.len, buf, len);
if (b.len > 0) mg_iobuf_add(&b, b.len, "", 1); // nul-terminate
fclose(fp);
}
(void) depth;
@ -5887,7 +5891,7 @@ size_t mg_ws_wrap(struct mg_connection *c, size_t len, int op) {
size_t header_len = mkhdr(len, op, c->is_client, header);
// NOTE: order of operations is important!
mg_iobuf_add(&c->send, c->send.len, NULL, header_len, MG_IO_SIZE);
mg_iobuf_add(&c->send, c->send.len, NULL, header_len);
p = &c->send.buf[c->send.len - len]; // p points to data
memmove(p, p - header_len, len); // Shift data
memcpy(p - header_len, header, header_len); // Prepend header
@ -6853,7 +6857,7 @@ bool mg_send(struct mg_connection *c, const void *buf, size_t len) {
res = true;
} else {
// tx_tdp(ifp, ifp->ip, c->loc.port, c->rem.ip, c->rem.port, buf, len);
return mg_iobuf_add(&c->send, c->send.len, buf, len, MG_IO_SIZE);
return mg_iobuf_add(&c->send, c->send.len, buf, len);
}
return res;
}

View File

@ -913,12 +913,13 @@ struct mg_iobuf {
unsigned char *buf; // Pointer to stored data
size_t size; // Total size available
size_t len; // Current number of bytes
size_t align; // Alignment during allocation
};
int mg_iobuf_init(struct mg_iobuf *, size_t);
int mg_iobuf_init(struct mg_iobuf *, size_t, size_t);
int mg_iobuf_resize(struct mg_iobuf *, size_t);
void mg_iobuf_free(struct mg_iobuf *);
size_t mg_iobuf_add(struct mg_iobuf *, size_t, const void *, size_t, size_t);
size_t mg_iobuf_add(struct mg_iobuf *, size_t, const void *, size_t);
size_t mg_iobuf_del(struct mg_iobuf *, size_t ofs, size_t len);
int mg_base64_update(unsigned char p, char *to, int len);

View File

@ -75,7 +75,7 @@ static void mg_putchar_iobuf_static(char ch, void *param) {
void mg_pfn_iobuf(char ch, void *param) {
struct mg_iobuf *io = (struct mg_iobuf *) param;
if (io->len + 2 > io->size) mg_iobuf_resize(io, io->size + 64);
if (io->len + 2 > io->size) mg_iobuf_resize(io, io->len + 2);
if (io->len + 2 <= io->size) {
io->buf[io->len++] = (uint8_t) ch;
io->buf[io->len] = 0;
@ -97,9 +97,10 @@ void mg_pfn_realloc(char ch, void *param) {
}
size_t mg_vsnprintf(char *buf, size_t len, const char *fmt, va_list *ap) {
struct mg_iobuf io = {(uint8_t *) buf, len, 0};
struct mg_iobuf io = {(uint8_t *) buf, len, 0, 0};
size_t n = mg_vrprintf(mg_putchar_iobuf_static, &io, fmt, ap);
if (n < len) buf[n] = '\0';
// if (len > 0) buf[len - 1] = '\0'; // Guarantee to 0-terminate
return n;
}

View File

@ -10,8 +10,13 @@ static void zeromem(volatile unsigned char *buf, size_t len) {
}
}
static size_t roundup(size_t size, size_t align) {
return align == 0 ? size : (size + align - 1) / align * align;
}
int mg_iobuf_resize(struct mg_iobuf *io, size_t new_size) {
int ok = 1;
new_size = roundup(new_size, io->align);
if (new_size == 0) {
zeromem(io->buf, io->size);
free(io->buf);
@ -36,21 +41,18 @@ int mg_iobuf_resize(struct mg_iobuf *io, size_t new_size) {
return ok;
}
int mg_iobuf_init(struct mg_iobuf *io, size_t size) {
int mg_iobuf_init(struct mg_iobuf *io, size_t size, size_t align) {
io->buf = NULL;
io->align = align;
io->size = io->len = 0;
return mg_iobuf_resize(io, size);
}
size_t mg_iobuf_add(struct mg_iobuf *io, size_t ofs, const void *buf,
size_t len, size_t chunk_size) {
size_t new_size = io->len + len;
if (new_size > io->size) {
new_size += chunk_size; // Make sure that io->size
new_size -= new_size % chunk_size; // is aligned by chunk_size boundary
mg_iobuf_resize(io, new_size); // Attempt to realloc
if (new_size != io->size) len = 0; // Realloc failure, append nothing
}
size_t len) {
size_t new_size = roundup(io->len + len, io->align);
mg_iobuf_resize(io, new_size); // Attempt to resize
if (new_size != io->size) len = 0; // Resize failure, append nothing
if (ofs < io->len) memmove(io->buf + ofs + len, io->buf + ofs, io->len - ofs);
if (buf != NULL) memmove(io->buf + ofs, buf, len);
if (ofs > io->len) io->len += ofs - io->len;

View File

@ -6,10 +6,11 @@ struct mg_iobuf {
unsigned char *buf; // Pointer to stored data
size_t size; // Total size available
size_t len; // Current number of bytes
size_t align; // Alignment during allocation
};
int mg_iobuf_init(struct mg_iobuf *, size_t);
int mg_iobuf_init(struct mg_iobuf *, size_t, size_t);
int mg_iobuf_resize(struct mg_iobuf *, size_t);
void mg_iobuf_free(struct mg_iobuf *);
size_t mg_iobuf_add(struct mg_iobuf *, size_t, const void *, size_t, size_t);
size_t mg_iobuf_add(struct mg_iobuf *, size_t, const void *, size_t);
size_t mg_iobuf_del(struct mg_iobuf *, size_t ofs, size_t len);

View File

@ -143,6 +143,7 @@ struct mg_connection *mg_alloc_conn(struct mg_mgr *mgr) {
(struct mg_connection *) calloc(1, sizeof(*c) + mgr->extraconnsize);
if (c != NULL) {
c->mgr = mgr;
c->send.align = c->recv.align = MG_IO_SIZE;
c->id = ++mgr->nextid;
}
return c;

View File

@ -172,7 +172,7 @@ bool mg_send(struct mg_connection *c, const void *buf, size_t len) {
iolog(c, (char *) buf, n, false);
return n > 0;
} else {
return mg_iobuf_add(&c->send, c->send.len, buf, len, MG_IO_SIZE);
return mg_iobuf_add(&c->send, c->send.len, buf, len);
}
}

View File

@ -12,12 +12,12 @@
#if MG_ENABLE_SSI
static char *mg_ssi(const char *path, const char *root, int depth) {
struct mg_iobuf b = {NULL, 0, 0};
struct mg_iobuf b = {NULL, 0, 0, MG_IO_SIZE};
FILE *fp = fopen(path, "rb");
if (fp != NULL) {
char buf[MG_SSI_BUFSIZ], arg[sizeof(buf)];
int ch, intag = 0;
size_t len = 0, align = MG_IO_SIZE;
size_t len = 0;
buf[0] = arg[0] = '\0';
while ((ch = fgetc(fp)) != EOF) {
if (intag && ch == '>' && buf[len - 1] == '-' && buf[len - 2] == '-') {
@ -30,7 +30,7 @@ static char *mg_ssi(const char *path, const char *root, int depth) {
mg_snprintf(tmp, sizeof(tmp), "%.*s%s", (int) (p - path), path, arg);
if (depth < MG_MAX_SSI_DEPTH &&
(data = mg_ssi(tmp, root, depth + 1)) != NULL) {
mg_iobuf_add(&b, b.len, data, strlen(data), align);
mg_iobuf_add(&b, b.len, data, strlen(data));
free(data);
} else {
MG_ERROR(("%s: file=%s error or too deep", path, arg));
@ -40,7 +40,7 @@ static char *mg_ssi(const char *path, const char *root, int depth) {
mg_snprintf(tmp, sizeof(tmp), "%s%s", root, arg);
if (depth < MG_MAX_SSI_DEPTH &&
(data = mg_ssi(tmp, root, depth + 1)) != NULL) {
mg_iobuf_add(&b, b.len, data, strlen(data), align);
mg_iobuf_add(&b, b.len, data, strlen(data));
free(data);
} else {
MG_ERROR(("%s: virtual=%s error or too deep", path, arg));
@ -48,13 +48,13 @@ static char *mg_ssi(const char *path, const char *root, int depth) {
} else {
// Unknown SSI tag
MG_ERROR(("Unknown SSI tag: %.*s", (int) len, buf));
mg_iobuf_add(&b, b.len, buf, len, align);
mg_iobuf_add(&b, b.len, buf, len);
}
intag = 0;
len = 0;
} else if (ch == '<') {
intag = 1;
if (len > 0) mg_iobuf_add(&b, b.len, buf, len, align);
if (len > 0) mg_iobuf_add(&b, b.len, buf, len);
len = 0;
buf[len++] = (char) (ch & 0xff);
} else if (intag) {
@ -68,13 +68,13 @@ static char *mg_ssi(const char *path, const char *root, int depth) {
} else {
buf[len++] = (char) (ch & 0xff);
if (len >= sizeof(buf)) {
mg_iobuf_add(&b, b.len, buf, len, align);
mg_iobuf_add(&b, b.len, buf, len);
len = 0;
}
}
}
if (len > 0) mg_iobuf_add(&b, b.len, buf, len, align);
if (b.len > 0) mg_iobuf_add(&b, b.len, "", 1, align); // nul-terminate
if (len > 0) mg_iobuf_add(&b, b.len, buf, len);
if (b.len > 0) mg_iobuf_add(&b, b.len, "", 1); // nul-terminate
fclose(fp);
}
(void) depth;

View File

@ -290,7 +290,7 @@ size_t mg_ws_wrap(struct mg_connection *c, size_t len, int op) {
size_t header_len = mkhdr(len, op, c->is_client, header);
// NOTE: order of operations is important!
mg_iobuf_add(&c->send, c->send.len, NULL, header_len, MG_IO_SIZE);
mg_iobuf_add(&c->send, c->send.len, NULL, header_len);
p = &c->send.buf[c->send.len - len]; // p points to data
memmove(p, p - header_len, len); // Shift data
memcpy(p - header_len, header, header_len); // Prepend header

View File

@ -265,18 +265,18 @@ static void test_base64(void) {
}
static void test_iobuf(void) {
struct mg_iobuf io = {0, 0, 0};
struct mg_iobuf io = {0, 0, 0, 10};
ASSERT(io.buf == NULL && io.size == 0 && io.len == 0);
mg_iobuf_resize(&io, 1);
ASSERT(io.buf != NULL && io.size == 1 && io.len == 0);
ASSERT(io.buf != NULL && io.size == 10 && io.len == 0);
ASSERT(memcmp(io.buf, "\x00", 1) == 0);
mg_iobuf_add(&io, 3, "hi", 2, 10);
mg_iobuf_add(&io, 3, "hi", 2);
ASSERT(io.buf != NULL && io.size == 10 && io.len == 5);
ASSERT(memcmp(io.buf, "\x00\x00\x00hi", 5) == 0);
mg_iobuf_add(&io, io.len, "!", 1, 10);
mg_iobuf_add(&io, io.len, "!", 1);
ASSERT(io.buf != NULL && io.size == 10 && io.len == 6);
ASSERT(memcmp(io.buf, "\x00\x00\x00hi!", 6) == 0);
mg_iobuf_add(&io, 0, "x", 1, 10);
mg_iobuf_add(&io, 0, "x", 1);
ASSERT(memcmp(io.buf, "x\x00\x00\x00hi!", 7) == 0);
ASSERT(io.buf != NULL && io.size == 10 && io.len == 7);
mg_iobuf_del(&io, 1, 3);
@ -1585,6 +1585,13 @@ static void test_str(void) {
ASSERT(mg_snprintf(tmp, sizeof(tmp), "%H", 9, s) == 20);
ASSERT(strcmp(tmp, expected) == 0);
}
{
char tmp[3];
ASSERT(mg_snprintf(tmp, sizeof(tmp), "%s", "0123456789") == 10);
ASSERT(strcmp(tmp, "01") == 0);
ASSERT(tmp[2] == '\0');
}
}
static void fn1(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
@ -2023,8 +2030,8 @@ static void eh11(struct mg_connection *c, int ev, void *ev_data,
struct stream_status *status = (struct stream_status *) fn_data;
if (ev == MG_EV_CONNECT) {
size_t len = MG_MAX_RECV_SIZE * 2;
struct mg_iobuf buf = {NULL, 0, 0};
mg_iobuf_init(&buf, len);
struct mg_iobuf buf = {NULL, 0, 0, 0};
mg_iobuf_init(&buf, len, 0);
mg_random(buf.buf, buf.size);
buf.len = buf.size;
mg_send(c, buf.buf, buf.len);