diff --git a/include/memory.h b/include/memory.h index 858bc6c..102ddf9 100644 --- a/include/memory.h +++ b/include/memory.h @@ -22,30 +22,12 @@ #ifndef r3__memory_h #define r3__memory_h -#ifdef __sun__ -#include -#endif -#include -#include -#include #include -#include -#include #ifdef __cplusplus extern "C" { #endif -#define R3_STRUCT_FROM_MEMBER(s, m, p) ((s *)((char *)(p)-offsetof(s, m))) - -#if __GNUC__ >= 3 -#define R3_LIKELY(x) __builtin_expect(!!(x), 1) -#define R3_UNLIKELY(x) __builtin_expect(!!(x), 0) -#else -#define R3_LIKELY(x) (x) -#define R3_UNLIKELY(x) (x) -#endif - #ifdef __GNUC__ #define R3_GNUC_VERSION ((__GNUC__ << 16) | (__GNUC_MINOR__ << 8) | __GNUC_PATCHLEVEL__) #else @@ -68,8 +50,6 @@ extern "C" { #define R3_RETURNS_NONNULL #endif -typedef struct st_r3_buffer_prototype_t r3_buffer_prototype_t; - /** * buffer structure compatible with iovec */ @@ -78,77 +58,15 @@ typedef struct st_r3_iovec_t { unsigned int len; } r3_iovec_t; -typedef struct st_r3_mem_recycle_t { - unsigned int max; - unsigned int cnt; - struct st_r3_mem_recycle_chunk_t *_link; -} r3_mem_recycle_t; - -struct st_r3_mem_pool_shared_entry_t { - unsigned int refcnt; - void (*dispose)(void *); - char bytes[1]; -}; - -/** - * the memory pool - */ -typedef struct st_r3_mem_pool_t { - struct st_r3_mem_pool_chunk_t *chunks; - unsigned int chunk_offset; - struct st_r3_mem_pool_shared_ref_t *shared_refs; - struct st_r3_mem_pool_direct_t *directs; -} r3_mem_pool_t; - -/** - * buffer used to store incoming / outgoing octets - */ -typedef struct st_r3_buffer_t { - /** - * capacity of the buffer (or minimum initial capacity in case of a prototype (i.e. bytes == NULL)) - */ - unsigned int capacity; - /** - * amount of the data available - */ - unsigned int size; - /** - * pointer to the start of the data (or NULL if is pointing to a prototype) - */ - char *bytes; - /** - * prototype (or NULL if the instance is part of the prototype (i.e. bytes == NULL)) - */ - r3_buffer_prototype_t *_prototype; - /** - * file descriptor (if not -1, used to store the buffer) - */ - int _fd; - char _buf[1]; -} r3_buffer_t; - -typedef struct st_r3_buffer_mmap_settings_t { - unsigned int threshold; - char fn_template[FILENAME_MAX]; -} r3_buffer_mmap_settings_t; - -struct st_r3_buffer_prototype_t { - r3_mem_recycle_t allocator; - r3_buffer_t _initial_buf; - r3_buffer_mmap_settings_t *mmap_settings; -}; - -#define R3_VECTOR(type) \ - struct { \ - type *entries; \ - unsigned int size; \ - unsigned int capacity; \ +#define R3_VECTOR(type) \ + struct { \ + type *entries; \ + unsigned int size; \ + unsigned int capacity; \ } typedef R3_VECTOR(void) r3_vector_t; -extern void *(*r3_mem__set_secure)(void *, int, size_t); - /** * prints an error message and aborts */ @@ -158,97 +76,19 @@ R3_NORETURN void r3_fatal(const char *msg); * constructor for r3_iovec_t */ static r3_iovec_t r3_iovec_init(const void *base, unsigned int len); + /** * wrapper of malloc; allocates given size of memory or dies if impossible */ R3_RETURNS_NONNULL static void *r3_mem_alloc(unsigned int sz); + /** - * warpper of realloc; reallocs the given chunk or dies if impossible + * wrapper of realloc; reallocs the given chunk or dies if impossible */ static void *r3_mem_realloc(void *oldp, unsigned int sz); -/** - * allocates memory using the reusing allocator - */ -void *r3_mem_alloc_recycle(r3_mem_recycle_t *allocator, unsigned int sz); -/** - * returns the memory to the reusing allocator - */ -void r3_mem_free_recycle(r3_mem_recycle_t *allocator, void *p); - -/** - * initializes the memory pool. - */ -void r3_mem_init_pool(r3_mem_pool_t *pool); -/** - * clears the memory pool. - * Applications may dispose the pool after calling the function or reuse it without calling r3_mem_init_pool. - */ -void r3_mem_clear_pool(r3_mem_pool_t *pool); -/** - * allocates given size of memory from the memory pool, or dies if impossible - */ -void *r3_mem_alloc_pool(r3_mem_pool_t *pool, unsigned int sz); -/** - * allocates a ref-counted chunk of given size from the memory pool, or dies if impossible. - * The ref-count of the returned chunk is 1 regardless of whether or not the chunk is linked to a pool. - * @param pool pool to which the allocated chunk should be linked (or NULL to allocate an orphan chunk) - */ -void *r3_mem_alloc_shared(r3_mem_pool_t *pool, unsigned int sz, void (*dispose)(void *)); -/** - * links a ref-counted chunk to a memory pool. - * The ref-count of the chunk will be decremented when the pool is cleared. - * It is permitted to link a chunk more than once to a single pool. - */ -void r3_mem_link_shared(r3_mem_pool_t *pool, void *p); -/** - * increments the reference count of a ref-counted chunk. - */ -static void r3_mem_addref_shared(void *p); -/** - * decrements the reference count of a ref-counted chunk. - * The chunk gets freed when the ref-count reaches zero. - */ -static int r3_mem_release_shared(void *p); -/** - * initialize the buffer using given prototype. - */ -static void r3_buffer_init(r3_buffer_t **buffer, r3_buffer_prototype_t *prototype); -/** - * - */ -void r3_buffer__do_free(r3_buffer_t *buffer); -/** - * disposes of the buffer - */ -static void r3_buffer_dispose(r3_buffer_t **buffer); -/** - * allocates a buffer. - * @param inbuf - pointer to a pointer pointing to the structure (set *inbuf to NULL to allocate a new buffer) - * @param min_guarantee minimum number of bytes to reserve - * @return buffer to which the next data should be stored - * @note When called against a new buffer, the function returns a buffer twice the size of requested guarantee. The function uses - * exponential backoff for already-allocated buffers. - */ -r3_iovec_t r3_buffer_reserve(r3_buffer_t **inbuf, unsigned int min_guarantee); -/** - * throws away given size of the data from the buffer. - * @param delta number of octets to be drained from the buffer - */ -void r3_buffer_consume(r3_buffer_t **inbuf, unsigned int delta); -/** - * resets the buffer prototype - */ -static void r3_buffer_set_prototype(r3_buffer_t **buffer, r3_buffer_prototype_t *prototype); -/** - * registers a buffer to memory pool, so that it would be freed when the pool is flushed. Note that the buffer cannot be resized - * after it is linked. - */ -static void r3_buffer_link_to_pool(r3_buffer_t *buffer, r3_mem_pool_t *pool); -void r3_buffer__dispose_linked(void *p); /** * grows the vector so that it could store at least new_capacity elements of given size (or dies if impossible). - * @param pool memory pool that the vector is using * @param vector the vector * @param element_size size of the elements stored in the vector * @param new_capacity the capacity of the buffer after the function returns @@ -258,31 +98,6 @@ void r3_buffer__dispose_linked(void *p); static void r3_vector__reserve(r3_vector_t *vector, unsigned int element_size, unsigned int new_capacity); void r3_vector__expand(r3_vector_t *vector, unsigned int element_size, unsigned int new_capacity); -/** - * tests if target chunk (target_len bytes long) is equal to test chunk (test_len bytes long) - */ -static int r3_memis(const void *target, unsigned int target_len, const void *test, unsigned int test_len); - -/** - * secure memset - */ -static void *r3_mem_set_secure(void *b, int c, unsigned int len); - -/** - * swaps contents of memory - */ -void r3_mem_swap(void *x, void *y, unsigned int len); - -/** - * emits hexdump of given buffer to fp - */ -void r3_dump_memory(FILE *fp, const char *buf, unsigned int len); - -/** - * appends an element to a NULL-terminated list allocated using malloc - */ -void r3_append_to_null_terminated_list(void ***list, void *element); - /* inline defs */ inline r3_iovec_t r3_iovec_init(const void *base, unsigned int len) @@ -312,52 +127,6 @@ inline void *r3_mem_realloc(void *oldp, unsigned int sz) return newp; } -inline void r3_mem_addref_shared(void *p) -{ - struct st_r3_mem_pool_shared_entry_t *entry = R3_STRUCT_FROM_MEMBER(struct st_r3_mem_pool_shared_entry_t, bytes, p); - assert(entry->refcnt != 0); - ++entry->refcnt; -} - -inline int r3_mem_release_shared(void *p) -{ - struct st_r3_mem_pool_shared_entry_t *entry = R3_STRUCT_FROM_MEMBER(struct st_r3_mem_pool_shared_entry_t, bytes, p); - if (--entry->refcnt == 0) { - if (entry->dispose != NULL) - entry->dispose(entry->bytes); - free(entry); - return 1; - } - return 0; -} - -inline void r3_buffer_init(r3_buffer_t **buffer, r3_buffer_prototype_t *prototype) -{ - *buffer = &prototype->_initial_buf; -} - -inline void r3_buffer_dispose(r3_buffer_t **_buffer) -{ - r3_buffer_t *buffer = *_buffer; - *_buffer = NULL; - if (buffer->bytes != NULL) - r3_buffer__do_free(buffer); -} - -inline void r3_buffer_set_prototype(r3_buffer_t **buffer, r3_buffer_prototype_t *prototype) -{ - if ((*buffer)->_prototype != NULL) - (*buffer)->_prototype = prototype; - else - *buffer = &prototype->_initial_buf; -} - -inline void r3_buffer_link_to_pool(r3_buffer_t *buffer, r3_mem_pool_t *pool) -{ - r3_buffer_t **slot = (r3_buffer_t **)r3_mem_alloc_shared(pool, sizeof(*slot), r3_buffer__dispose_linked); - *slot = buffer; -} - inline void r3_vector__reserve(r3_vector_t *vector, unsigned int element_size, unsigned int new_capacity) { if (vector->capacity < new_capacity) { @@ -365,23 +134,6 @@ inline void r3_vector__reserve(r3_vector_t *vector, unsigned int element_size, u } } -inline int r3_memis(const void *_target, unsigned int target_len, const void *_test, unsigned int test_len) -{ - const char *target = (const char *)_target, *test = (const char *)_test; - if (target_len != test_len) - return 0; - if (target_len == 0) - return 1; - if (target[0] != test[0]) - return 0; - return memcmp(target + 1, test + 1, test_len - 1) == 0; -} - -inline void *r3_mem_set_secure(void *b, int c, unsigned int len) -{ - return r3_mem__set_secure(b, c, len); -} - #ifdef __cplusplus } #endif diff --git a/src/memory.c b/src/memory.c index 438f260..8d122ef 100644 --- a/src/memory.c +++ b/src/memory.c @@ -20,302 +20,18 @@ * IN THE SOFTWARE. */ -#include +#include "config.h" #include -#include -#include -#include #include -#include #include -#include -#include -#include #include "memory.h" -struct st_r3_mem_recycle_chunk_t { - struct st_r3_mem_recycle_chunk_t *next; -}; - -struct st_r3_mem_pool_chunk_t { - struct st_r3_mem_pool_chunk_t *next; - unsigned int _dummy; /* align to 2*sizeof(void*) */ - char bytes[4096 - sizeof(void *) * 2]; -}; - -struct st_r3_mem_pool_direct_t { - struct st_r3_mem_pool_direct_t *next; - unsigned int _dummy; /* align to 2*sizeof(void*) */ - char bytes[1]; -}; - -struct st_r3_mem_pool_shared_ref_t { - struct st_r3_mem_pool_shared_ref_t *next; - struct st_r3_mem_pool_shared_entry_t *entry; -}; - -void *(*r3_mem__set_secure)(void *, int, size_t) = memset; - -static __thread r3_mem_recycle_t mempool_allocator = {16}; - void r3_fatal(const char *msg) { fprintf(stderr, "fatal:%s\n", msg); abort(); } -void *r3_mem_alloc_recycle(r3_mem_recycle_t *allocator, unsigned int sz) -{ - struct st_r3_mem_recycle_chunk_t *chunk; - if (allocator->cnt == 0) - return r3_mem_alloc(sz); - /* detach and return the pooled pointer */ - chunk = allocator->_link; - assert(chunk != NULL); - allocator->_link = chunk->next; - --allocator->cnt; - return chunk; -} - -void r3_mem_free_recycle(r3_mem_recycle_t *allocator, void *p) -{ - struct st_r3_mem_recycle_chunk_t *chunk; - if (allocator->cnt == allocator->max) { - free(p); - return; - } - /* register the pointer to the pool */ - chunk = p; - chunk->next = allocator->_link; - allocator->_link = chunk; - ++allocator->cnt; -} - -void r3_mem_init_pool(r3_mem_pool_t *pool) -{ - pool->chunks = NULL; - pool->chunk_offset = sizeof(pool->chunks->bytes); - pool->directs = NULL; - pool->shared_refs = NULL; -} - -void r3_mem_clear_pool(r3_mem_pool_t *pool) -{ - /* release the refcounted chunks */ - if (pool->shared_refs != NULL) { - struct st_r3_mem_pool_shared_ref_t *ref = pool->shared_refs; - do { - r3_mem_release_shared(ref->entry->bytes); - } while ((ref = ref->next) != NULL); - pool->shared_refs = NULL; - } - /* release the direct chunks */ - if (pool->directs != NULL) { - struct st_r3_mem_pool_direct_t *direct = pool->directs, *next; - do { - next = direct->next; - free(direct); - } while ((direct = next) != NULL); - pool->directs = NULL; - } - /* free chunks, and reset the first chunk */ - while (pool->chunks != NULL) { - struct st_r3_mem_pool_chunk_t *next = pool->chunks->next; - r3_mem_free_recycle(&mempool_allocator, pool->chunks); - pool->chunks = next; - } - pool->chunk_offset = sizeof(pool->chunks->bytes); -} - -void *r3_mem_alloc_pool(r3_mem_pool_t *pool, unsigned int sz) -{ - void *ret; - - if (sz >= sizeof(pool->chunks->bytes) / 4) { - /* allocate large requests directly */ - struct st_r3_mem_pool_direct_t *newp = r3_mem_alloc(offsetof(struct st_r3_mem_pool_direct_t, bytes) + sz); - newp->next = pool->directs; - pool->directs = newp; - return newp->bytes; - } - - /* 16-bytes rounding */ - sz = (sz + 15) & ~15; - if (sizeof(pool->chunks->bytes) - pool->chunk_offset < sz) { - /* allocate new chunk */ - struct st_r3_mem_pool_chunk_t *newp = r3_mem_alloc_recycle(&mempool_allocator, sizeof(*newp)); - newp->next = pool->chunks; - pool->chunks = newp; - pool->chunk_offset = 0; - } - - ret = pool->chunks->bytes + pool->chunk_offset; - pool->chunk_offset += sz; - return ret; -} - -static void link_shared(r3_mem_pool_t *pool, struct st_r3_mem_pool_shared_entry_t *entry) -{ - struct st_r3_mem_pool_shared_ref_t *ref = r3_mem_alloc_pool(pool, sizeof(struct st_r3_mem_pool_shared_ref_t)); - ref->entry = entry; - ref->next = pool->shared_refs; - pool->shared_refs = ref; -} - -void *r3_mem_alloc_shared(r3_mem_pool_t *pool, unsigned int sz, void (*dispose)(void *)) -{ - struct st_r3_mem_pool_shared_entry_t *entry = r3_mem_alloc(offsetof(struct st_r3_mem_pool_shared_entry_t, bytes) + sz); - entry->refcnt = 1; - entry->dispose = dispose; - if (pool != NULL) - link_shared(pool, entry); - return entry->bytes; -} - -void r3_mem_link_shared(r3_mem_pool_t *pool, void *p) -{ - r3_mem_addref_shared(p); - link_shared(pool, R3_STRUCT_FROM_MEMBER(struct st_r3_mem_pool_shared_entry_t, bytes, p)); -} - -static unsigned int topagesize(unsigned int capacity) -{ - unsigned int pagesize = getpagesize(); - return (offsetof(r3_buffer_t, _buf) + capacity + pagesize - 1) / pagesize * pagesize; -} - -void r3_buffer__do_free(r3_buffer_t *buffer) -{ - /* caller should assert that the buffer is not part of the prototype */ - if (buffer->capacity == buffer->_prototype->_initial_buf.capacity) { - r3_mem_free_recycle(&buffer->_prototype->allocator, buffer); - } else if (buffer->_fd != -1) { - close(buffer->_fd); - munmap((void *)buffer, topagesize(buffer->capacity)); - } else { - free(buffer); - } -} - -r3_iovec_t r3_buffer_reserve(r3_buffer_t **_inbuf, unsigned int min_guarantee) -{ - r3_buffer_t *inbuf = *_inbuf; - r3_iovec_t ret; - - if (inbuf->bytes == NULL) { - r3_buffer_prototype_t *prototype = R3_STRUCT_FROM_MEMBER(r3_buffer_prototype_t, _initial_buf, inbuf); - if (min_guarantee <= prototype->_initial_buf.capacity) { - min_guarantee = prototype->_initial_buf.capacity; - inbuf = r3_mem_alloc_recycle(&prototype->allocator, offsetof(r3_buffer_t, _buf) + min_guarantee); - } else { - inbuf = r3_mem_alloc(offsetof(r3_buffer_t, _buf) + min_guarantee); - } - *_inbuf = inbuf; - inbuf->size = 0; - inbuf->bytes = inbuf->_buf; - inbuf->capacity = min_guarantee; - inbuf->_prototype = prototype; - inbuf->_fd = -1; - } else { - if (min_guarantee <= inbuf->capacity - inbuf->size - (inbuf->bytes - inbuf->_buf)) { - /* ok */ - } else if ((inbuf->size + min_guarantee) * 2 <= inbuf->capacity) { - /* the capacity should be less than or equal to 2 times of: size + guarantee */ - memmove(inbuf->_buf, inbuf->bytes, inbuf->size); - inbuf->bytes = inbuf->_buf; - } else { - unsigned int new_capacity = inbuf->capacity; - do { - new_capacity *= 2; - } while (new_capacity - inbuf->size < min_guarantee); - if (inbuf->_prototype->mmap_settings != NULL && inbuf->_prototype->mmap_settings->threshold <= new_capacity) { - unsigned int new_allocsize = topagesize(new_capacity); - int fd; - r3_buffer_t *newp; - if (inbuf->_fd == -1) { - char *tmpfn = alloca(strlen(inbuf->_prototype->mmap_settings->fn_template) + 1); - strcpy(tmpfn, inbuf->_prototype->mmap_settings->fn_template); - if ((fd = mkstemp(tmpfn)) == -1) { - fprintf(stderr, "failed to create temporary file:%s:%s\n", tmpfn, strerror(errno)); - goto MapError; - } - unlink(tmpfn); - } else { - fd = inbuf->_fd; - } - if (ftruncate(fd, new_allocsize) != 0) { - perror("failed to resize temporary file"); - goto MapError; - } - if ((newp = (void *)mmap(NULL, new_allocsize, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0)) == MAP_FAILED) { - perror("mmap failed"); - goto MapError; - } - if (inbuf->_fd == -1) { - /* copy data (moving from malloc to mmap) */ - newp->size = inbuf->size; - newp->bytes = newp->_buf; - newp->capacity = new_capacity; - newp->_prototype = inbuf->_prototype; - newp->_fd = fd; - memcpy(newp->_buf, inbuf->bytes, inbuf->size); - r3_buffer__do_free(inbuf); - *_inbuf = inbuf = newp; - } else { - /* munmap */ - unsigned int offset = inbuf->bytes - inbuf->_buf; - munmap((void *)inbuf, topagesize(inbuf->capacity)); - *_inbuf = inbuf = newp; - inbuf->capacity = new_capacity; - inbuf->bytes = newp->_buf + offset; - } - } else { - r3_buffer_t *newp = r3_mem_alloc(offsetof(r3_buffer_t, _buf) + new_capacity); - newp->size = inbuf->size; - newp->bytes = newp->_buf; - newp->capacity = new_capacity; - newp->_prototype = inbuf->_prototype; - newp->_fd = -1; - memcpy(newp->_buf, inbuf->bytes, inbuf->size); - r3_buffer__do_free(inbuf); - *_inbuf = inbuf = newp; - } - } - } - - ret.base = inbuf->bytes + inbuf->size; - ret.len = inbuf->_buf + inbuf->capacity - ret.base; - - return ret; - -MapError: - ret.base = NULL; - ret.len = 0; - return ret; -} - -void r3_buffer_consume(r3_buffer_t **_inbuf, unsigned int delta) -{ - r3_buffer_t *inbuf = *_inbuf; - - if (delta != 0) { - assert(inbuf->bytes != NULL); - if (inbuf->size == delta) { - *_inbuf = &inbuf->_prototype->_initial_buf; - r3_buffer__do_free(inbuf); - } else { - inbuf->size -= delta; - inbuf->bytes += delta; - } - } -} - -void r3_buffer__dispose_linked(void *p) -{ - r3_buffer_t **buf = p; - r3_buffer_dispose(buf); -} - void r3_vector__expand(r3_vector_t *vector, unsigned int element_size, unsigned int new_capacity) { void *new_entries; @@ -327,49 +43,3 @@ void r3_vector__expand(r3_vector_t *vector, unsigned int element_size, unsigned new_entries = r3_mem_realloc(vector->entries, element_size * vector->capacity); vector->entries = new_entries; } - -void r3_mem_swap(void *_x, void *_y, unsigned int len) -{ - char *x = _x, *y = _y; - char buf[256]; - - while (len != 0) { - unsigned int blocksz = len < sizeof(buf) ? len : sizeof(buf); - memcpy(buf, x, blocksz); - memcpy(x, y, blocksz); - memcpy(y, buf, blocksz); - len -= blocksz; - } -} - -void r3_dump_memory(FILE *fp, const char *buf, unsigned int len) -{ - unsigned int i, j; - - for (i = 0; i < len; i += 16) { - fprintf(fp, "%08x", i); - for (j = 0; j != 16; ++j) { - if (i + j < len) - fprintf(fp, " %02x", (int)(unsigned char)buf[i + j]); - else - fprintf(fp, " "); - } - fprintf(fp, " "); - for (j = 0; j != 16 && i + j < len; ++j) { - int ch = buf[i + j]; - fputc(' ' <= ch && ch < 0x7f ? ch : '.', fp); - } - fprintf(fp, "\n"); - } -} - -void r3_append_to_null_terminated_list(void ***list, void *element) -{ - unsigned int cnt; - - for (cnt = 0; (*list)[cnt] != NULL; ++cnt) - ; - *list = r3_mem_realloc(*list, (cnt + 2) * sizeof(void *)); - (*list)[cnt++] = element; - (*list)[cnt] = NULL; -}