From ad8faeb787ff875f94060f02700f8abff0d70c09 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 01:41:45 +0800 Subject: [PATCH 01/56] Update test results --- tests/bench_str.csv | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/bench_str.csv b/tests/bench_str.csv index 1b3d4f7..b0e5c9f 100644 --- a/tests/bench_str.csv +++ b/tests/bench_str.csv @@ -432,3 +432,9 @@ 1400606491,14030387.83 1400606523,13157029.51 1400606567,13999364.95 +1400607671,13906437.74 +1400607682,13828950.20 +1400607689,13932349.19 +1400607695,13583203.56 +1400607698,13630627.45 +1400607700,13972490.11 From 9c191682daa83dadaadd3f8175991ccdbc4040d6 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 20 May 2014 20:50:15 +0300 Subject: [PATCH 02/56] Added zmalloc from redis in order to allow different allocators. --- Makefile.am | 3 + include/zmalloc.h | 85 +++++++++++ src/CMakeLists.txt | 2 +- src/Makefile.am | 3 +- src/edge.c | 8 +- src/gvc.c | 7 +- src/list.c | 55 +++---- src/node.c | 55 +++---- src/str.c | 13 +- src/token.c | 12 +- src/zmalloc.c | 351 +++++++++++++++++++++++++++++++++++++++++++++ tests/check_tree.c | 12 +- 12 files changed, 524 insertions(+), 82 deletions(-) create mode 100644 include/zmalloc.h create mode 100644 src/zmalloc.c diff --git a/Makefile.am b/Makefile.am index f44568f..816fe55 100644 --- a/Makefile.am +++ b/Makefile.am @@ -5,8 +5,11 @@ ACLOCAL_AMFLAGS=-I m4 r3_includedir = $(includedir)/r3 r3_include_HEADERS = \ include/r3.h \ + include/r3_define.h \ + include/r3_list.h \ include/r3_str.h \ include/str_array.h \ + include/zmalloc.h \ $(NULL) pkgconfigdir = $(libdir)/pkgconfig diff --git a/include/zmalloc.h b/include/zmalloc.h new file mode 100644 index 0000000..76a3a7f --- /dev/null +++ b/include/zmalloc.h @@ -0,0 +1,85 @@ +#ifndef ZMALLOC_H +#define ZMALLOC_H + +/* zmalloc - total amount of allocated memory aware version of malloc() + * + * Copyright (c) 2009-2010, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* Double expansion needed for stringification of macro values. */ +#define __xstr(s) __str(s) +#define __str(s) #s + +#if defined(USE_TCMALLOC) +#define ZMALLOC_LIB ("tcmalloc-" __xstr(TC_VERSION_MAJOR) "." __xstr(TC_VERSION_MINOR)) +#include +#if (TC_VERSION_MAJOR == 1 && TC_VERSION_MINOR >= 6) || (TC_VERSION_MAJOR > 1) +#define HAVE_MALLOC_SIZE 1 +#define zmalloc_size(p) tc_malloc_size(p) +#else +#error "Newer version of tcmalloc required" +#endif + +#elif defined(USE_JEMALLOC) +#define ZMALLOC_LIB ("jemalloc-" __xstr(JEMALLOC_VERSION_MAJOR) "." __xstr(JEMALLOC_VERSION_MINOR) "." __xstr(JEMALLOC_VERSION_BUGFIX)) +#include +#if (JEMALLOC_VERSION_MAJOR == 2 && JEMALLOC_VERSION_MINOR >= 1) || (JEMALLOC_VERSION_MAJOR > 2) +#define HAVE_MALLOC_SIZE 1 +#define zmalloc_size(p) je_malloc_usable_size(p) +#else +#error "Newer version of jemalloc required" +#endif + +#elif defined(__APPLE__) +#include +#define HAVE_MALLOC_SIZE 1 +#define zmalloc_size(p) malloc_size(p) +#endif + +#ifndef ZMALLOC_LIB +#define ZMALLOC_LIB "libc" +#endif + +void *zmalloc(size_t size); +void *zcalloc(size_t size); +void *zrealloc(void *ptr, size_t size); +void zfree(void *ptr); +char *zstrdup(const char *s); +size_t zmalloc_used_memory(void); +void zmalloc_enable_thread_safeness(void); +void zmalloc_set_oom_handler(void (*oom_handler)(size_t)); +float zmalloc_get_fragmentation_ratio(size_t rss); +size_t zmalloc_get_rss(void); +size_t zmalloc_get_private_dirty(void); +void zlibc_free(void *ptr); + +#ifndef HAVE_MALLOC_SIZE +size_t zmalloc_size(void *ptr); +#endif + +#endif // ZMALLOC_H diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8ef2c55..3e0e268 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 2.8) include_directories("${PROJECT_SOURCE_DIR}/include") # install(TARGETS swiftnav-static DESTINATION lib${LIB_SUFFIX}) -set(R3_SRCS node.c str.c list.c token.c edge.c) +set(R3_SRCS node.c str.c list.c token.c edge.c zmalloc.c) set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -Wall -pipe -g3 -funroll-loops -O0") # set(LIBS ${LIBS} ${PCRE_LIBRARIES} ${Judy_LIBRARIES} ${Jemalloc_LIBRARIES} r3) diff --git a/src/Makefile.am b/src/Makefile.am index 782cf5e..2e6234c 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,7 +1,6 @@ lib_LTLIBRARIES = libr3.la # lib_LIBRARIES = libr3.a -libr3_la_SOURCES = node.c edge.c str.c token.c - +libr3_la_SOURCES = node.c edge.c str.c token.c zmalloc.c # libr3_la_LDFLAGS = -export-symbols-regex '^r3_|^match_' libr3_la_LIBADD=$(DEPS_LIBS) diff --git a/src/edge.c b/src/edge.c index ae3c4c8..19f5645 100644 --- a/src/edge.c +++ b/src/edge.c @@ -24,7 +24,7 @@ #include "str_array.h" edge * r3_edge_create(char * pattern, int pattern_len, node * child) { - edge * e = (edge*) malloc( sizeof(edge) ); + edge * e = (edge*) zmalloc( sizeof(edge) ); e->pattern = pattern; e->pattern_len = pattern_len; e->child = child; @@ -69,17 +69,17 @@ node * r3_edge_branch(edge *e, int dl) { new_child->data = e->child->data; // copy data pointer e->child->data = NULL; - // truncate the original edge pattern + // truncate the original edge pattern char *op = e->pattern; e->pattern = strndup(e->pattern, dl); e->pattern_len = dl; - free(op); + zfree(op); return new_child; } void r3_edge_free(edge * e) { if (e->pattern) { - free(e->pattern); + zfree(e->pattern); } if ( e->child ) { r3_tree_free(e->child); diff --git a/src/gvc.c b/src/gvc.c index 86ccb67..6f75112 100644 --- a/src/gvc.c +++ b/src/gvc.c @@ -10,10 +10,11 @@ #include #include "r3.h" #include "r3_gvc.h" +#include "zmalloc.h" static char * node_id_str(int id) { - char * name = malloc(sizeof(char) * 20); + char * name = zmalloc(sizeof(char) * 20); sprintf(name, "#%d", id); return name; } @@ -41,7 +42,7 @@ void r3_tree_build_ag_nodes(Agraph_t * g, Agnode_t * ag_parent_node, node * n, i /** - * Render a tree to tree graph image via graphviz (dot) + * Render a tree to tree graph image via graphviz (dot) */ int r3_tree_render_dot(node * tree) { @@ -69,7 +70,7 @@ int r3_tree_render_dot(node * tree) /** - * Render a tree to tree graph image via graphviz (dot) + * Render a tree to tree graph image via graphviz (dot) */ int r3_tree_render_file(node * tree, char * format, char * filename) { diff --git a/src/list.c b/src/list.c index 4e80695..62b7729 100644 --- a/src/list.c +++ b/src/list.c @@ -6,56 +6,57 @@ */ #include #include "r3_list.h" - +#include "zmalloc.h" + /* Naive linked list implementation */ - + list * list_create() { - list *l = (list *) malloc(sizeof(list)); + list *l = (list *) zmalloc(sizeof(list)); l->count = 0; l->head = NULL; l->tail = NULL; pthread_mutex_init(&(l->mutex), NULL); return l; } - + void list_free(l) list *l; { list_item *li, *tmp; - + pthread_mutex_lock(&(l->mutex)); - + if (l != NULL) { li = l->head; while (li != NULL) { tmp = li->next; - free(li); + zfree(li); li = tmp; } } - + pthread_mutex_unlock(&(l->mutex)); pthread_mutex_destroy(&(l->mutex)); - free(l); + zfree(l); } - + list_item * list_add_element(l, ptr) list *l; void *ptr; { list_item *li; - + pthread_mutex_lock(&(l->mutex)); - - li = (list_item *) malloc(sizeof(list_item)); + + li = (list_item *) zmalloc(sizeof(list_item)); li->value = ptr; li->next = NULL; li->prev = l->tail; - + if (l->tail == NULL) { l->head = l->tail = li; } @@ -63,12 +64,12 @@ list_add_element(l, ptr) l->tail = li; } l->count++; - + pthread_mutex_unlock(&(l->mutex)); - + return li; } - + int list_remove_element(l, ptr) list *l; @@ -76,9 +77,9 @@ list_remove_element(l, ptr) { int result = 0; list_item *li = l->head; - + pthread_mutex_lock(&(l->mutex)); - + while (li != NULL) { if (li->value == ptr) { if (li->prev == NULL) { @@ -87,7 +88,7 @@ list_remove_element(l, ptr) else { li->prev->next = li->next; } - + if (li->next == NULL) { l->tail = li->prev; } @@ -95,27 +96,27 @@ list_remove_element(l, ptr) li->next->prev = li->prev; } l->count--; - free(li); + zfree(li); result = 1; break; } li = li->next; } - + pthread_mutex_unlock(&(l->mutex)); - + return result; } - + void list_each_element(l, func) list *l; int (*func)(list_item *); { list_item *li; - + pthread_mutex_lock(&(l->mutex)); - + li = l->head; while (li != NULL) { if (func(li) == 1) { @@ -123,6 +124,6 @@ list_each_element(l, func) } li = li->next; } - + pthread_mutex_unlock(&(l->mutex)); } diff --git a/src/node.c b/src/node.c index 9741e40..7308f65 100644 --- a/src/node.c +++ b/src/node.c @@ -17,13 +17,14 @@ #include "r3_define.h" #include "r3_str.h" #include "str_array.h" +#include "zmalloc.h" // String value as the index http://judy.sourceforge.net/doc/JudySL_3x.htm static int strndiff(char * d1, char * d2, unsigned int n) { char * o = d1; - while ( *d1 == *d2 && n-- > 0 ) { + while ( *d1 == *d2 && n-- > 0 ) { d1++; d2++; } @@ -32,7 +33,7 @@ static int strndiff(char * d1, char * d2, unsigned int n) { static int strdiff(char * d1, char * d2) { char * o = d1; - while( *d1 == *d2 ) { + while( *d1 == *d2 ) { d1++; d2++; } @@ -44,9 +45,9 @@ static int strdiff(char * d1, char * d2) { * Create a node object */ node * r3_tree_create(int cap) { - node * n = (node*) malloc( sizeof(node) ); + node * n = (node*) zmalloc( sizeof(node) ); - n->edges = (edge**) malloc( sizeof(edge*) * cap ); + n->edges = (edge**) zmalloc( sizeof(edge*) * cap ); n->edge_len = 0; n->edge_cap = cap; @@ -70,18 +71,18 @@ void r3_tree_free(node * tree) { } } if (tree->edges) - free(tree->edges); + zfree(tree->edges); if (tree->routes) - free(tree->routes); + zfree(tree->routes); if (tree->combined_pattern) - free(tree->combined_pattern); + zfree(tree->combined_pattern); if (tree->pcre_pattern) - free(tree->pcre_pattern); + zfree(tree->pcre_pattern); if (tree->pcre_extra) - free(tree->pcre_extra); - if (tree->ov) - free(tree->ov); - free(tree); + zfree(tree->pcre_extra); + if (tree->ov) + zfree(tree->ov); + zfree(tree); tree = NULL; } @@ -110,11 +111,11 @@ edge * r3_node_add_child(node * n, char * pat , node *child) { void r3_node_append_edge(node *n, edge *e) { if (n->edges == NULL) { n->edge_cap = 3; - n->edges = malloc(sizeof(edge) * n->edge_cap); + n->edges = zmalloc(sizeof(edge) * n->edge_cap); } if (n->edge_len >= n->edge_cap) { n->edge_cap *= 2; - edge ** p = realloc(n->edges, sizeof(edge) * n->edge_cap); + edge ** p = zrealloc(n->edges, sizeof(edge) * n->edge_cap); if(p) { n->edges = p; } @@ -157,7 +158,7 @@ void r3_tree_compile_patterns(node * n) { char * cpat; char * p; - cpat = calloc(sizeof(char),128); + cpat = zcalloc(128); if (cpat==NULL) return; @@ -189,7 +190,7 @@ void r3_tree_compile_patterns(node * n) { info("pattern: %s\n",cpat); n->ov_cnt = (1 + n->edge_len) * 3; - n->ov = (int*) calloc(sizeof(int), n->ov_cnt); + n->ov = (int*) zcalloc(n->ov_cnt); n->combined_pattern = cpat; @@ -201,9 +202,9 @@ void r3_tree_compile_patterns(node * n) { unsigned int option_bits = 0; if (n->pcre_pattern) - free(n->pcre_pattern); + zfree(n->pcre_pattern); if (n->pcre_extra) - free(n->pcre_extra); + zfree(n->pcre_extra); // n->pcre_pattern; n->pcre_pattern = pcre_compile( @@ -225,7 +226,7 @@ void r3_tree_compile_patterns(node * n) { match_entry * match_entry_createl(char * path, int path_len) { - match_entry * entry = malloc(sizeof(match_entry)); + match_entry * entry = zmalloc(sizeof(match_entry)); if(!entry) return NULL; entry->vars = str_array_create(3); @@ -237,7 +238,7 @@ match_entry * match_entry_createl(char * path, int path_len) { void match_entry_free(match_entry * entry) { str_array_free(entry->vars); - free(entry); + zfree(entry); } @@ -363,7 +364,7 @@ inline edge * r3_node_find_edge_str(const node * n, char * str, int str_len) { node * r3_node_create() { - node * n = (node*) malloc( sizeof(node) ); + node * n = (node*) zmalloc( sizeof(node) ); n->edges = NULL; n->edge_len = 0; n->edge_cap = 0; @@ -384,11 +385,11 @@ route * r3_route_create(char * path) { } void r3_route_free(route * route) { - free(route); + zfree(route); } route * r3_route_createl(char * path, int path_len) { - route * info = malloc(sizeof(route)); + route * info = zmalloc(sizeof(route)); info->path = path; info->path_len = path_len; info->request_method = 0; // can be (GET || POST) @@ -522,7 +523,7 @@ bool r3_node_has_slug_edges(node *n) { for ( int i = 0 ; i < n->edge_len ; i++ ) { e = n->edges[i]; e->has_slug = contains_slug(e->pattern); - if (e->has_slug) + if (e->has_slug) found = TRUE; } return found; @@ -600,16 +601,16 @@ int r3_route_cmp(route *r1, match_entry *r2) { /** - * + * */ void r3_node_append_route(node * n, route * r) { if (n->routes == NULL) { n->route_cap = 3; - n->routes = malloc(sizeof(route) * n->route_cap); + n->routes = zmalloc(sizeof(route) * n->route_cap); } if (n->route_len >= n->route_cap) { n->route_cap *= 2; - n->routes = realloc(n->routes, sizeof(route) * n->route_cap); + n->routes = zrealloc(n->routes, sizeof(route) * n->route_cap); } n->routes[ n->route_len++ ] = r; } diff --git a/src/str.c b/src/str.c index 3d70b7e..564c754 100644 --- a/src/str.c +++ b/src/str.c @@ -11,6 +11,7 @@ #include "r3.h" #include "r3_str.h" #include "str_array.h" +#include "zmalloc.h" /** * provide a quick way to count slugs, simply search for '{' @@ -134,11 +135,11 @@ char * slug_compile(char * str, int len) s1 = find_slug_placeholder(str, &s1_len); if ( s1 == NULL ) { - return strdup(str); + return zstrdup(str); } char * out = NULL; - if ((out = calloc(sizeof(char),200)) == NULL) { + if ((out = zcalloc(200)) == NULL) { return (NULL); } @@ -171,7 +172,7 @@ char * ltrim_slash(char* str) { char * p = str; while (*p == '/') p++; - return strdup(p); + return zstrdup(p); } void str_repeat(char *s, char *c, int len) { @@ -188,13 +189,13 @@ void print_indent(int level) { } #ifndef HAVE_STRDUP -char *strdup(const char *s) { +char *zstrdup(const char *s) { char *out; int count = 0; while( s[count] ) ++count; ++count; - out = malloc(sizeof(char) * count); + out = zmalloc(sizeof(char) * count); out[--count] = 0; while( --count >= 0 ) out[count] = s[count]; @@ -209,7 +210,7 @@ char *strndup(const char *s, int n) { while( count < n && s[count] ) ++count; ++count; - out = malloc(sizeof(char) * count); + out = zmalloc(sizeof(char) * count); out[--count] = 0; while( --count >= 0 ) out[count] = s[count]; diff --git a/src/token.c b/src/token.c index 6f2bc27..e0f23a8 100644 --- a/src/token.c +++ b/src/token.c @@ -10,22 +10,22 @@ #include #include "str_array.h" #include "r3_str.h" - +#include "zmalloc.h" str_array * str_array_create(int cap) { - str_array * list = (str_array*) malloc( sizeof(str_array) ); + str_array * list = (str_array*) zmalloc( sizeof(str_array) ); list->len = 0; list->cap = cap; - list->tokens = (char**) malloc( sizeof(char*) * cap); + list->tokens = (char**) zmalloc( sizeof(char*) * cap); return list; } void str_array_free(str_array *l) { for ( int i = 0; i < l->len ; i++ ) { char * t = l->tokens[ i ]; - free(t); + zfree(t); } - free(l); + zfree(l); } bool str_array_is_full(str_array * l) { @@ -33,7 +33,7 @@ bool str_array_is_full(str_array * l) { } bool str_array_resize(str_array *l, int new_cap) { - l->tokens = realloc(l->tokens, sizeof(char**) * new_cap); + l->tokens = zrealloc(l->tokens, sizeof(char**) * new_cap); l->cap = new_cap; return l->tokens != NULL; } diff --git a/src/zmalloc.c b/src/zmalloc.c new file mode 100644 index 0000000..d0cf726 --- /dev/null +++ b/src/zmalloc.c @@ -0,0 +1,351 @@ +/* zmalloc - total amount of allocated memory aware version of malloc() + * + * Copyright (c) 2009-2010, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +/* This function provide us access to the original libc free(). This is useful + * for instance to free results obtained by backtrace_symbols(). We need + * to define this function before including zmalloc.h that may shadow the + * free implementation if we use jemalloc or another non standard allocator. */ +void zlibc_free(void *ptr) { + free(ptr); +} + +#include +#include +#include "config.h" +#include "zmalloc.h" + +#ifdef HAVE_MALLOC_SIZE +#define PREFIX_SIZE (0) +#else +#if defined(__sun) || defined(__sparc) || defined(__sparc__) +#define PREFIX_SIZE (sizeof(long long)) +#else +#define PREFIX_SIZE (sizeof(size_t)) +#endif +#endif + +/* Explicitly override malloc/free etc when using tcmalloc. */ +#if defined(USE_TCMALLOC) +#define malloc(size) tc_malloc(size) +#define calloc(count,size) tc_calloc(count,size) +#define realloc(ptr,size) tc_realloc(ptr,size) +#define free(ptr) tc_free(ptr) +#elif defined(USE_JEMALLOC) +#define malloc(size) je_malloc(size) +#define calloc(count,size) je_calloc(count,size) +#define realloc(ptr,size) je_realloc(ptr,size) +#define free(ptr) je_free(ptr) +#endif + +#ifdef HAVE_ATOMIC +#define update_zmalloc_stat_add(__n) __sync_add_and_fetch(&used_memory, (__n)) +#define update_zmalloc_stat_sub(__n) __sync_sub_and_fetch(&used_memory, (__n)) +#else +#define update_zmalloc_stat_add(__n) do { \ + pthread_mutex_lock(&used_memory_mutex); \ + used_memory += (__n); \ + pthread_mutex_unlock(&used_memory_mutex); \ +} while(0) + +#define update_zmalloc_stat_sub(__n) do { \ + pthread_mutex_lock(&used_memory_mutex); \ + used_memory -= (__n); \ + pthread_mutex_unlock(&used_memory_mutex); \ +} while(0) + +#endif + +#define update_zmalloc_stat_alloc(__n) do { \ + size_t _n = (__n); \ + if (_n&(sizeof(long)-1)) _n += sizeof(long)-(_n&(sizeof(long)-1)); \ + if (zmalloc_thread_safe) { \ + update_zmalloc_stat_add(_n); \ + } else { \ + used_memory += _n; \ + } \ +} while(0) + +#define update_zmalloc_stat_free(__n) do { \ + size_t _n = (__n); \ + if (_n&(sizeof(long)-1)) _n += sizeof(long)-(_n&(sizeof(long)-1)); \ + if (zmalloc_thread_safe) { \ + update_zmalloc_stat_sub(_n); \ + } else { \ + used_memory -= _n; \ + } \ +} while(0) + +static size_t used_memory = 0; +static int zmalloc_thread_safe = 0; +pthread_mutex_t used_memory_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void zmalloc_default_oom(size_t size) { + fprintf(stderr, "zmalloc: Out of memory trying to allocate %zu bytes\n", + size); + fflush(stderr); + abort(); +} + +static void (*zmalloc_oom_handler)(size_t) = zmalloc_default_oom; + +void *zmalloc(size_t size) { + void *ptr = malloc(size+PREFIX_SIZE); + + if (!ptr) zmalloc_oom_handler(size); +#ifdef HAVE_MALLOC_SIZE + update_zmalloc_stat_alloc(zmalloc_size(ptr)); + return ptr; +#else + *((size_t*)ptr) = size; + update_zmalloc_stat_alloc(size+PREFIX_SIZE); + return (char*)ptr+PREFIX_SIZE; +#endif +} + +void *zcalloc(size_t size) { + void *ptr = calloc(1, size+PREFIX_SIZE); + + if (!ptr) zmalloc_oom_handler(size); +#ifdef HAVE_MALLOC_SIZE + update_zmalloc_stat_alloc(zmalloc_size(ptr)); + return ptr; +#else + *((size_t*)ptr) = size; + update_zmalloc_stat_alloc(size+PREFIX_SIZE); + return (char*)ptr+PREFIX_SIZE; +#endif +} + +void *zrealloc(void *ptr, size_t size) { +#ifndef HAVE_MALLOC_SIZE + void *realptr; +#endif + size_t oldsize; + void *newptr; + + if (ptr == NULL) return zmalloc(size); +#ifdef HAVE_MALLOC_SIZE + oldsize = zmalloc_size(ptr); + newptr = realloc(ptr,size); + if (!newptr) zmalloc_oom_handler(size); + + update_zmalloc_stat_free(oldsize); + update_zmalloc_stat_alloc(zmalloc_size(newptr)); + return newptr; +#else + realptr = (char*)ptr-PREFIX_SIZE; + oldsize = *((size_t*)realptr); + newptr = realloc(realptr,size+PREFIX_SIZE); + if (!newptr) zmalloc_oom_handler(size); + + *((size_t*)newptr) = size; + update_zmalloc_stat_free(oldsize); + update_zmalloc_stat_alloc(size); + return (char*)newptr+PREFIX_SIZE; +#endif +} + +/* Provide zmalloc_size() for systems where this function is not provided by + * malloc itself, given that in that case we store a header with this + * information as the first bytes of every allocation. */ +#ifndef HAVE_MALLOC_SIZE +size_t zmalloc_size(void *ptr) { + void *realptr = (char*)ptr-PREFIX_SIZE; + size_t size = *((size_t*)realptr); + /* Assume at least that all the allocations are padded at sizeof(long) by + * the underlying allocator. */ + if (size&(sizeof(long)-1)) size += sizeof(long)-(size&(sizeof(long)-1)); + return size+PREFIX_SIZE; +} +#endif + +void zfree(void *ptr) { +#ifndef HAVE_MALLOC_SIZE + void *realptr; + size_t oldsize; +#endif + + if (ptr == NULL) return; +#ifdef HAVE_MALLOC_SIZE + update_zmalloc_stat_free(zmalloc_size(ptr)); + free(ptr); +#else + realptr = (char*)ptr-PREFIX_SIZE; + oldsize = *((size_t*)realptr); + update_zmalloc_stat_free(oldsize+PREFIX_SIZE); + free(realptr); +#endif +} + +char *zstrdup(const char *s) { + size_t l = strlen(s)+1; + char *p = zmalloc(l); + + memcpy(p,s,l); + return p; +} + +size_t zmalloc_used_memory(void) { + size_t um; + + if (zmalloc_thread_safe) { +#ifdef HAVE_ATOMIC + um = __sync_add_and_fetch(&used_memory, 0); +#else + pthread_mutex_lock(&used_memory_mutex); + um = used_memory; + pthread_mutex_unlock(&used_memory_mutex); +#endif + } + else { + um = used_memory; + } + + return um; +} + +void zmalloc_enable_thread_safeness(void) { + zmalloc_thread_safe = 1; +} + +void zmalloc_set_oom_handler(void (*oom_handler)(size_t)) { + zmalloc_oom_handler = oom_handler; +} + +/* Get the RSS information in an OS-specific way. + * + * WARNING: the function zmalloc_get_rss() is not designed to be fast + * and may not be called in the busy loops where Redis tries to release + * memory expiring or swapping out objects. + * + * For this kind of "fast RSS reporting" usages use instead the + * function RedisEstimateRSS() that is a much faster (and less precise) + * version of the function. */ + +#if defined(HAVE_PROC_STAT) +#include +#include +#include +#include + +size_t zmalloc_get_rss(void) { + int page = sysconf(_SC_PAGESIZE); + size_t rss; + char buf[4096]; + char filename[256]; + int fd, count; + char *p, *x; + + snprintf(filename,256,"/proc/%d/stat",getpid()); + if ((fd = open(filename,O_RDONLY)) == -1) return 0; + if (read(fd,buf,4096) <= 0) { + close(fd); + return 0; + } + close(fd); + + p = buf; + count = 23; /* RSS is the 24th field in /proc//stat */ + while(p && count--) { + p = strchr(p,' '); + if (p) p++; + } + if (!p) return 0; + x = strchr(p,' '); + if (!x) return 0; + *x = '\0'; + + rss = strtoll(p,NULL,10); + rss *= page; + return rss; +} +#elif defined(HAVE_TASKINFO) +#include +#include +#include +#include +#include +#include +#include + +size_t zmalloc_get_rss(void) { + task_t task = MACH_PORT_NULL; + struct task_basic_info t_info; + mach_msg_type_number_t t_info_count = TASK_BASIC_INFO_COUNT; + + if (task_for_pid(current_task(), getpid(), &task) != KERN_SUCCESS) + return 0; + task_info(task, TASK_BASIC_INFO, (task_info_t)&t_info, &t_info_count); + + return t_info.resident_size; +} +#else +size_t zmalloc_get_rss(void) { + /* If we can't get the RSS in an OS-specific way for this system just + * return the memory usage we estimated in zmalloc().. + * + * Fragmentation will appear to be always 1 (no fragmentation) + * of course... */ + return zmalloc_used_memory(); +} +#endif + +/* Fragmentation = RSS / allocated-bytes */ +float zmalloc_get_fragmentation_ratio(size_t rss) { + return (float)rss/zmalloc_used_memory(); +} + +#if defined(HAVE_PROC_SMAPS) +size_t zmalloc_get_private_dirty(void) { + char line[1024]; + size_t pd = 0; + FILE *fp = fopen("/proc/self/smaps","r"); + + if (!fp) return 0; + while(fgets(line,sizeof(line),fp) != NULL) { + if (strncmp(line,"Private_Dirty:",14) == 0) { + char *p = strchr(line,'k'); + if (p) { + *p = '\0'; + pd += strtol(line+14,NULL,10) * 1024; + } + } + } + fclose(fp); + return pd; +} +#else +size_t zmalloc_get_private_dirty(void) { + return 0; +} +#endif diff --git a/tests/check_tree.c b/tests/check_tree.c index cd5306c..6565f3b 100644 --- a/tests/check_tree.c +++ b/tests/check_tree.c @@ -31,7 +31,7 @@ START_TEST (test_r3_node_find_edge) node * child = r3_tree_create(3); - fail_if( r3_node_add_child(n, strdup("/add") , child) == FALSE ); + fail_if( r3_node_add_child(n, zstrdup("/add") , child) == FALSE ); fail_if( r3_node_find_edge(n, "/add") == NULL ); fail_if( r3_node_find_edge(n, "/bar") != NULL ); @@ -218,16 +218,16 @@ START_TEST (test_str_array) str_array * l = str_array_create(3); fail_if( l == NULL ); - fail_if( FALSE == str_array_append(l, strdup("abc") ) ); + fail_if( FALSE == str_array_append(l, zstrdup("abc") ) ); fail_if( l->len != 1 ); - fail_if( FALSE == str_array_append(l, strdup("foo") ) ); + fail_if( FALSE == str_array_append(l, zstrdup("foo") ) ); fail_if( l->len != 2 ); - fail_if( FALSE == str_array_append(l, strdup("bar") ) ); + fail_if( FALSE == str_array_append(l, zstrdup("bar") ) ); fail_if( l->len != 3 ); - fail_if( FALSE == str_array_append(l, strdup("zoo") ) ); + fail_if( FALSE == str_array_append(l, zstrdup("zoo") ) ); fail_if( l->len != 4 ); fail_if( FALSE == str_array_resize(l, l->cap * 2) ); @@ -707,7 +707,7 @@ r3_tree_insert_path(n, "/garply/grault/corge", NULL); r3_tree_compile(n); // r3_tree_dump(n, 0); - // match_entry *entry = calloc( sizeof(entry) , 1 ); + // match_entry *entry = zcalloc( 1 ); node *m; m = r3_tree_match(n , "/qux/bar/corge", NULL); From 5a84e9aeb2e28ab1ad2c2f8d1ccdcb5a6b59f497 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 14:27:04 +0800 Subject: [PATCH 03/56] update pcre match error message --- src/node.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/node.c b/src/node.c index 9741e40..22b4195 100644 --- a/src/node.c +++ b/src/node.c @@ -280,11 +280,11 @@ node * r3_tree_matchl(const node * n, char * path, int path_len, match_entry * e if (rc < 0) { switch(rc) { - case PCRE_ERROR_NOMATCH: printf("No match\n"); break; + case PCRE_ERROR_NOMATCH: printf("pcre: no match\n"); break; /* Handle other special cases if you like */ - default: printf("Matching error %d\n", rc); break; + default: printf("pcre matching error '%d' on pattern '%s'\n", rc, n->combined_pattern); break; } // does not match all edges, return NULL; return NULL; From de5a308002a66e0b7a9d0fb38415f8bddadb4a4a Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 14:59:07 +0800 Subject: [PATCH 04/56] fix header includes --- src/edge.c | 1 + tests/bench_str.csv | 2 ++ tests/check_tree.c | 1 + 3 files changed, 4 insertions(+) diff --git a/src/edge.c b/src/edge.c index 19f5645..dedd6a0 100644 --- a/src/edge.c +++ b/src/edge.c @@ -22,6 +22,7 @@ #include "r3.h" #include "r3_str.h" #include "str_array.h" +#include "zmalloc.h" edge * r3_edge_create(char * pattern, int pattern_len, node * child) { edge * e = (edge*) zmalloc( sizeof(edge) ); diff --git a/tests/bench_str.csv b/tests/bench_str.csv index b0e5c9f..603fd34 100644 --- a/tests/bench_str.csv +++ b/tests/bench_str.csv @@ -438,3 +438,5 @@ 1400607695,13583203.56 1400607698,13630627.45 1400607700,13972490.11 +1400655532,6648299.49 +1400655542,6685928.45 diff --git a/tests/check_tree.c b/tests/check_tree.c index 6565f3b..718d711 100644 --- a/tests/check_tree.c +++ b/tests/check_tree.c @@ -5,6 +5,7 @@ #include "r3.h" #include "r3_str.h" #include "str_array.h" +#include "zmalloc.h" #include "bench.h" From 0e627e33d59d7f53ec9a219af1e9771a58d523f0 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 21 May 2014 10:50:37 +0300 Subject: [PATCH 05/56] Fixed all warnings from tests. --- src/node.c | 4 ++-- tests/check_slug.c | 2 +- tests/check_tree.c | 18 +++++++++--------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/node.c b/src/node.c index 0b208a0..b2d339c 100644 --- a/src/node.c +++ b/src/node.c @@ -158,7 +158,7 @@ void r3_tree_compile_patterns(node * n) { char * cpat; char * p; - cpat = zcalloc(128); + cpat = zcalloc(sizeof(char) * 128); if (cpat==NULL) return; @@ -190,7 +190,7 @@ void r3_tree_compile_patterns(node * n) { info("pattern: %s\n",cpat); n->ov_cnt = (1 + n->edge_len) * 3; - n->ov = (int*) zcalloc(n->ov_cnt); + n->ov = (int*) zcalloc(sizeof(int) * n->ov_cnt); n->combined_pattern = cpat; diff --git a/tests/check_slug.c b/tests/check_slug.c index 22fe635..8b0adcd 100644 --- a/tests/check_slug.c +++ b/tests/check_slug.c @@ -63,7 +63,7 @@ START_TEST (test_inside_slug) int slug_len = 0; char * pattern = "/user/{name:\\s+}/to/{id}"; char * offset = strchr(pattern, '{') + 2; - ck_assert( inside_slug(pattern, strlen(pattern), offset) ); + ck_assert( (int)inside_slug(pattern, strlen(pattern), offset) ); ck_assert( *(inside_slug(pattern, strlen(pattern), offset)) == '{' ); ck_assert( ! inside_slug(pattern, strlen(pattern), pattern) ); } diff --git a/tests/check_tree.c b/tests/check_tree.c index 718d711..cfa28c5 100644 --- a/tests/check_tree.c +++ b/tests/check_tree.c @@ -72,27 +72,27 @@ START_TEST (test_compile) entry = match_entry_createl( "foo" , strlen("/foo") ); m = r3_tree_matchl( n , "/foo", strlen("/foo"), entry); - ck_assert( m ); + ck_assert( (int)m ); entry = match_entry_createl( "/zoo" , strlen("/zoo") ); m = r3_tree_matchl( n , "/zoo", strlen("/zoo"), entry); - ck_assert( m ); + ck_assert( (int)m ); entry = match_entry_createl( "/bar" , strlen("/bar") ); m = r3_tree_matchl( n , "/bar", strlen("/bar"), entry); - ck_assert( m ); + ck_assert( (int)m ); entry = match_entry_createl( "/xxx" , strlen("/xxx") ); m = r3_tree_matchl( n , "/xxx", strlen("/xxx"), entry); - ck_assert( m ); + ck_assert( (int)m ); entry = match_entry_createl( "/foo/xxx" , strlen("/foo/xxx") ); m = r3_tree_matchl( n , "/foo/xxx", strlen("/foo/xxx"), entry); - ck_assert( m ); + ck_assert( (int)m ); entry = match_entry_createl( "/some_id" , strlen("/some_id") ); m = r3_tree_matchl( n , "/some_id", strlen("/some_id"), entry); - ck_assert( m ); + ck_assert( (int)m ); } END_TEST @@ -107,7 +107,7 @@ START_TEST (test_pcre_patterns_insert) node *matched; matched = r3_tree_matchl(n, "/post/111-222", strlen("/post/111-222"), NULL); - ck_assert(matched); + ck_assert((int)matched); ck_assert(matched->endpoint > 0); // incomplete string shouldn't match @@ -133,7 +133,7 @@ START_TEST (test_pcre_patterns_insert_2) r3_tree_dump(n, 0); node *matched; matched = r3_tree_match(n, "/post/11/22", NULL); - ck_assert(matched); + ck_assert((int)matched); ck_assert(matched->endpoint > 0); } END_TEST @@ -160,7 +160,7 @@ START_TEST (test_pcre_patterns_insert_3) matched = r3_tree_match(n, "/post/11/22", NULL); - ck_assert(matched); + ck_assert((int)matched); matched = r3_tree_match(n, "/post/11", NULL); ck_assert(!matched); From 18a92867fcea1676df27692e04e4f49d2b832795 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 21 May 2014 11:10:24 +0300 Subject: [PATCH 06/56] Removed wrong call to zfree that caused a segfault. --- src/edge.c | 2 +- tests/bench_str.csv | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/edge.c b/src/edge.c index dedd6a0..32ce06d 100644 --- a/src/edge.c +++ b/src/edge.c @@ -74,7 +74,7 @@ node * r3_edge_branch(edge *e, int dl) { char *op = e->pattern; e->pattern = strndup(e->pattern, dl); e->pattern_len = dl; - zfree(op); + return new_child; } diff --git a/tests/bench_str.csv b/tests/bench_str.csv index 603fd34..22df891 100644 --- a/tests/bench_str.csv +++ b/tests/bench_str.csv @@ -440,3 +440,4 @@ 1400607700,13972490.11 1400655532,6648299.49 1400655542,6685928.45 +1400659046,19754150.71 From e34759c1aa9975a98651891f72b40140e16cd209 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 21 May 2014 11:18:12 +0300 Subject: [PATCH 07/56] Using zstrndup instead of strndup) --- src/edge.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/edge.c b/src/edge.c index 32ce06d..a179f77 100644 --- a/src/edge.c +++ b/src/edge.c @@ -72,7 +72,7 @@ node * r3_edge_branch(edge *e, int dl) { // truncate the original edge pattern char *op = e->pattern; - e->pattern = strndup(e->pattern, dl); + e->pattern = zstrndup(e->pattern, dl); e->pattern_len = dl; return new_child; From e0ed62e3084b321a8c22ceaf5f677f8ed14fd719 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 21 May 2014 11:29:36 +0300 Subject: [PATCH 08/56] Running valgrind in a different build in order to detect memory leaks. --- .travis.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9cf85c8..0177dc4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,20 +6,24 @@ compiler: matrix: include: - compiler: gcc - env: CONFIGURE_OPTION='--enable-debug' COVERALLS=yes + env: CONFIGURE_OPTION='--enable-debug' COVERALLS=yes VALGRIND=no + - compiler: gcc + env: CONFIGURE_OPTION='--enable-debug' COVERALLS=yes VALGRIND=yes install: - sudo apt-get update -qq - sudo apt-get install -qq automake pkg-config build-essential libtool automake autoconf m4 gnulib - sudo apt-get install -qq check libpcre3 libpcre3-dev libjemalloc-dev libjemalloc1 - sudo apt-get install -qq graphviz-dev graphviz - - if [ x$COVERALLS == xyes ]; then sudo pip install cpp-coveralls --use-mirror; fi + - if [ x$COVERALLS == xyes ]; then sudo pip install cpp-coveralls; fi + - if [x$VALGRIND == xyes ]; then sudo apt-get install valgrind; fi script: - ./autogen.sh - ./configure $CONFIGURE_OPTION - make - - make check + - if [x$VALGRIND == xyes ]; make check 2>&1 > /dev/null; else; make check; fi + - if [x$VALGRIND == xyes ]; then valgrind ./tests/.libs/* -v --trace-children=yes --show-leak-kinds=full --leak-check=full; fi after_success: - if [ x$COVERALLS == xyes ]; then coveralls ; fi From 3ea9354e0beb54593702b9d358c8b4a3c29818a8 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 21 May 2014 11:36:50 +0300 Subject: [PATCH 09/56] Implemented zstrndup. --- include/zmalloc.h | 1 + src/edge.c | 2 +- src/node.c | 6 +++--- src/str.c | 2 +- src/zmalloc.c | 16 ++++++++++++++++ 5 files changed, 22 insertions(+), 5 deletions(-) diff --git a/include/zmalloc.h b/include/zmalloc.h index 76a3a7f..383f377 100644 --- a/include/zmalloc.h +++ b/include/zmalloc.h @@ -70,6 +70,7 @@ void *zcalloc(size_t size); void *zrealloc(void *ptr, size_t size); void zfree(void *ptr); char *zstrdup(const char *s); +char *zstrndup(const char *s, size_t n); size_t zmalloc_used_memory(void); void zmalloc_enable_thread_safeness(void); void zmalloc_set_oom_handler(void (*oom_handler)(size_t)); diff --git a/src/edge.c b/src/edge.c index a179f77..f50dc1e 100644 --- a/src/edge.c +++ b/src/edge.c @@ -55,7 +55,7 @@ node * r3_edge_branch(edge *e, int dl) { // the suffix edge of the leaf new_child = r3_tree_create(3); s1_len = e->pattern_len - dl; - e1 = r3_edge_create(strndup(s1, s1_len), s1_len, new_child); + e1 = r3_edge_create(zstrndup(s1, s1_len), s1_len, new_child); // Migrate the child edges to the new edge we just created. for ( int i = 0 ; i < tmp_edge_len ; i++ ) { diff --git a/src/node.c b/src/node.c index b2d339c..a9e0a76 100644 --- a/src/node.c +++ b/src/node.c @@ -306,7 +306,7 @@ node * r3_tree_matchl(const node * n, char * path, int path_len, match_entry * e if (entry && e->has_slug) { // append captured token to entry - str_array_append(entry->vars , strndup(substring_start, substring_length)); + str_array_append(entry->vars , zstrndup(substring_start, substring_length)); } if (restlen == 0 ) { return e->child && e->child->endpoint > 0 ? e->child : NULL; @@ -459,14 +459,14 @@ node * r3_tree_insert_pathl_(node *tree, char *path, int path_len, route * route // insert the first one edge, and break at "p" node * child = r3_tree_create(3); - r3_node_add_child(n, strndup(path, (int)(p - path)), child); + r3_node_add_child(n, zstrndup(path, (int)(p - path)), child); child->endpoint = 0; // and insert the rest part to the child return r3_tree_insert_pathl_(child, p, path_len - (int)(p - path), route, data); } else { node * child = r3_tree_create(3); - r3_node_add_child(n, strndup(path, path_len) , child); + r3_node_add_child(n, zstrndup(path, path_len) , child); // info("edge not found, insert one: %s\n", path); child->data = data; child->endpoint++; diff --git a/src/str.c b/src/str.c index 564c754..c83cd92 100644 --- a/src/str.c +++ b/src/str.c @@ -204,7 +204,7 @@ char *zstrdup(const char *s) { #endif #ifndef HAVE_STRNDUP -char *strndup(const char *s, int n) { +char *zstrndup(const char *s, int n) { char *out; int count = 0; while( count < n && s[count] ) diff --git a/src/zmalloc.c b/src/zmalloc.c index d0cf726..72447d8 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -215,6 +215,22 @@ char *zstrdup(const char *s) { return p; } +char * zstrndup (const char *s, size_t n) +{ + char *result; + size_t len = strlen (s); + + if (n < len) + len = n; + + result = (char *) zmalloc (len + 1); + if (!result) + return 0; + + result[len] = '\0'; + return (char *) memcpy (result, s, len); +} + size_t zmalloc_used_memory(void) { size_t um; From 1c8c63e0a56b2ea1c835c79dc73c759880264353 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 21 May 2014 11:44:24 +0300 Subject: [PATCH 10/56] Fixed a bash syntax error. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0177dc4..296f3eb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ script: - ./autogen.sh - ./configure $CONFIGURE_OPTION - make - - if [x$VALGRIND == xyes ]; make check 2>&1 > /dev/null; else; make check; fi + - if [x$VALGRIND == xyes ]; make check 2>&1 > /dev/null; else make check; fi - if [x$VALGRIND == xyes ]; then valgrind ./tests/.libs/* -v --trace-children=yes --show-leak-kinds=full --leak-check=full; fi after_success: From 86cdf3800ed4efab6c7c71af39283763d7499e51 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 21 May 2014 12:00:18 +0300 Subject: [PATCH 11/56] Fixed bash syntax errors. --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 296f3eb..8a15872 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,15 +15,15 @@ install: - sudo apt-get install -qq automake pkg-config build-essential libtool automake autoconf m4 gnulib - sudo apt-get install -qq check libpcre3 libpcre3-dev libjemalloc-dev libjemalloc1 - sudo apt-get install -qq graphviz-dev graphviz - - if [ x$COVERALLS == xyes ]; then sudo pip install cpp-coveralls; fi - - if [x$VALGRIND == xyes ]; then sudo apt-get install valgrind; fi + - if [ "x$COVERALLS" == xyes ]; then sudo pip install cpp-coveralls; fi + - if[ "x$VALGRIND" == xyes ]; then sudo apt-get install valgrind; fi script: - ./autogen.sh - ./configure $CONFIGURE_OPTION - make - - if [x$VALGRIND == xyes ]; make check 2>&1 > /dev/null; else make check; fi - - if [x$VALGRIND == xyes ]; then valgrind ./tests/.libs/* -v --trace-children=yes --show-leak-kinds=full --leak-check=full; fi + - if [ "x$VALGRIND" == xyes ]; then make check > /dev/null 2>&1; else make check; fi + - if [ "x$VALGRIND" == xyes ]; then valgrind ./tests/.libs/* -v --trace-children=yes --show-leak-kinds=full --leak-check=full; fi after_success: - if [ x$COVERALLS == xyes ]; then coveralls ; fi From cd4d4647deb572b4c60d626296c356135db4dfd8 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 21 May 2014 12:02:52 +0300 Subject: [PATCH 12/56] Bash is being touchy. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8a15872..d27cbd1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ install: - sudo apt-get install -qq check libpcre3 libpcre3-dev libjemalloc-dev libjemalloc1 - sudo apt-get install -qq graphviz-dev graphviz - if [ "x$COVERALLS" == xyes ]; then sudo pip install cpp-coveralls; fi - - if[ "x$VALGRIND" == xyes ]; then sudo apt-get install valgrind; fi + - if [ "x$VALGRIND" == xyes ]; then sudo apt-get install valgrind; fi script: - ./autogen.sh From 61b3b24e2e10da646f0243eca889bc38208bc43c Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 21 May 2014 12:46:56 +0300 Subject: [PATCH 13/56] Guarding against freeing NULL pointers. --- src/node.c | 46 ++++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/src/node.c b/src/node.c index a9e0a76..838a50a 100644 --- a/src/node.c +++ b/src/node.c @@ -65,25 +65,27 @@ node * r3_tree_create(int cap) { } void r3_tree_free(node * tree) { - for (int i = 0 ; i < tree->edge_len ; i++ ) { - if (tree->edges[i]) { - r3_edge_free(tree->edges[ i ]); + if (tree) { + for (int i = 0 ; i < tree->edge_len ; i++ ) { + if (tree->edges[i]) { + r3_edge_free(tree->edges[ i ]); + } } + if (tree->edges) + zfree(tree->edges); + if (tree->routes) + zfree(tree->routes); + if (tree->combined_pattern) + zfree(tree->combined_pattern); + if (tree->pcre_pattern) + zfree(tree->pcre_pattern); + if (tree->pcre_extra) + zfree(tree->pcre_extra); + if (tree->ov) + zfree(tree->ov); + zfree(tree); + tree = NULL; } - if (tree->edges) - zfree(tree->edges); - if (tree->routes) - zfree(tree->routes); - if (tree->combined_pattern) - zfree(tree->combined_pattern); - if (tree->pcre_pattern) - zfree(tree->pcre_pattern); - if (tree->pcre_extra) - zfree(tree->pcre_extra); - if (tree->ov) - zfree(tree->ov); - zfree(tree); - tree = NULL; } @@ -237,8 +239,10 @@ match_entry * match_entry_createl(char * path, int path_len) { } void match_entry_free(match_entry * entry) { - str_array_free(entry->vars); - zfree(entry); + if (entry) { + str_array_free(entry->vars); + zfree(entry); + } } @@ -385,7 +389,9 @@ route * r3_route_create(char * path) { } void r3_route_free(route * route) { - zfree(route); + if (route) { + zfree(route); + } } route * r3_route_createl(char * path, int path_len) { From 7303bc8f00c93ea980dbda93b9ebababf29d32e8 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 21 May 2014 12:50:56 +0300 Subject: [PATCH 14/56] More guarding against freeing NULL pointers. --- src/list.c | 25 +++++++++++++------------ src/token.c | 12 ++++++++---- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/list.c b/src/list.c index 62b7729..bf27bbc 100644 --- a/src/list.c +++ b/src/list.c @@ -25,22 +25,23 @@ void list_free(l) list *l; { - list_item *li, *tmp; + if (l) { + list_item *li, *tmp; - pthread_mutex_lock(&(l->mutex)); + pthread_mutex_lock(&(l->mutex)); - if (l != NULL) { - li = l->head; - while (li != NULL) { - tmp = li->next; - zfree(li); - li = tmp; + if (l != NULL) { + li = l->head; + while (li != NULL) { + tmp = li->next; + li = tmp; + } } - } - pthread_mutex_unlock(&(l->mutex)); - pthread_mutex_destroy(&(l->mutex)); - zfree(l); + pthread_mutex_unlock(&(l->mutex)); + pthread_mutex_destroy(&(l->mutex)); + zfree(l); + } } list_item * diff --git a/src/token.c b/src/token.c index e0f23a8..f5f73eb 100644 --- a/src/token.c +++ b/src/token.c @@ -21,11 +21,15 @@ str_array * str_array_create(int cap) { } void str_array_free(str_array *l) { - for ( int i = 0; i < l->len ; i++ ) { - char * t = l->tokens[ i ]; - zfree(t); + if (l) { + for ( int i = 0; i < l->len ; i++ ) { + char * t = l->tokens[ i ]; + if (t) { + zfree(t); + } + } + zfree(l); } - zfree(l); } bool str_array_is_full(str_array * l) { From 7d288404c7277f47fc4c962c31bc22efd9d2ca62 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 18:09:14 +0800 Subject: [PATCH 15/56] libpcre does not use zmalloc --- src/node.c | 8 ++++++-- tests/bench_str.csv | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/node.c b/src/node.c index 838a50a..a0d19d9 100644 --- a/src/node.c +++ b/src/node.c @@ -75,12 +75,14 @@ void r3_tree_free(node * tree) { zfree(tree->edges); if (tree->routes) zfree(tree->routes); - if (tree->combined_pattern) - zfree(tree->combined_pattern); + /* if (tree->pcre_pattern) zfree(tree->pcre_pattern); if (tree->pcre_extra) zfree(tree->pcre_extra); + */ + if (tree->combined_pattern) + zfree(tree->combined_pattern); if (tree->ov) zfree(tree->ov); zfree(tree); @@ -203,10 +205,12 @@ void r3_tree_compile_patterns(node * n) { int erroffset; unsigned int option_bits = 0; + /* if (n->pcre_pattern) zfree(n->pcre_pattern); if (n->pcre_extra) zfree(n->pcre_extra); + */ // n->pcre_pattern; n->pcre_pattern = pcre_compile( diff --git a/tests/bench_str.csv b/tests/bench_str.csv index 22df891..7fb02ba 100644 --- a/tests/bench_str.csv +++ b/tests/bench_str.csv @@ -441,3 +441,5 @@ 1400655532,6648299.49 1400655542,6685928.45 1400659046,19754150.71 +1400666823,6406965.25 +1400666950,6684744.95 From 6762025fe61fe6c0c21009240a752910f689bb96 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 18:12:14 +0800 Subject: [PATCH 16/56] free edge struct --- src/edge.c | 2 ++ src/node.c | 42 +++++++++++++++++++++--------------------- tests/bench_str.csv | 1 + tests/check_tree.c | 4 ++-- 4 files changed, 26 insertions(+), 23 deletions(-) diff --git a/src/edge.c b/src/edge.c index f50dc1e..387ee0c 100644 --- a/src/edge.c +++ b/src/edge.c @@ -85,5 +85,7 @@ void r3_edge_free(edge * e) { if ( e->child ) { r3_tree_free(e->child); } + // free itself + zfree(e); } diff --git a/src/node.c b/src/node.c index a0d19d9..aa459cd 100644 --- a/src/node.c +++ b/src/node.c @@ -65,29 +65,29 @@ node * r3_tree_create(int cap) { } void r3_tree_free(node * tree) { - if (tree) { - for (int i = 0 ; i < tree->edge_len ; i++ ) { - if (tree->edges[i]) { - r3_edge_free(tree->edges[ i ]); - } + for (int i = 0 ; i < tree->edge_len ; i++ ) { + if (tree->edges[i]) { + r3_edge_free(tree->edges[ i ]); } - if (tree->edges) - zfree(tree->edges); - if (tree->routes) - zfree(tree->routes); - /* - if (tree->pcre_pattern) - zfree(tree->pcre_pattern); - if (tree->pcre_extra) - zfree(tree->pcre_extra); - */ - if (tree->combined_pattern) - zfree(tree->combined_pattern); - if (tree->ov) - zfree(tree->ov); - zfree(tree); - tree = NULL; } + if (tree->edges) { + zfree(tree->edges); + } + if (tree->routes) { + zfree(tree->routes); + } + /* + if (tree->pcre_pattern) + zfree(tree->pcre_pattern); + if (tree->pcre_extra) + zfree(tree->pcre_extra); + */ + if (tree->combined_pattern) + zfree(tree->combined_pattern); + if (tree->ov) + zfree(tree->ov); + zfree(tree); + tree = NULL; } diff --git a/tests/bench_str.csv b/tests/bench_str.csv index 7fb02ba..fd0f711 100644 --- a/tests/bench_str.csv +++ b/tests/bench_str.csv @@ -443,3 +443,4 @@ 1400659046,19754150.71 1400666823,6406965.25 1400666950,6684744.95 +1400667091,6771202.47 diff --git a/tests/check_tree.c b/tests/check_tree.c index cfa28c5..88904f1 100644 --- a/tests/check_tree.c +++ b/tests/check_tree.c @@ -20,9 +20,9 @@ END_TEST START_TEST (test_r3_node_construct_and_free) { node * n = r3_tree_create(10); - node * child = r3_tree_create(3); + node * another_tree = r3_tree_create(3); r3_tree_free(n); - r3_tree_free(child); + r3_tree_free(another_tree); } END_TEST From 238aee085f217a6e5dfcad2dc57d1f4a7bd28177 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 18:16:48 +0800 Subject: [PATCH 17/56] use pcre's own functions to free pcre related structures --- src/node.c | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/src/node.c b/src/node.c index aa459cd..33de25d 100644 --- a/src/node.c +++ b/src/node.c @@ -76,12 +76,12 @@ void r3_tree_free(node * tree) { if (tree->routes) { zfree(tree->routes); } - /* - if (tree->pcre_pattern) - zfree(tree->pcre_pattern); - if (tree->pcre_extra) - zfree(tree->pcre_extra); - */ + if (tree->pcre_pattern) { + pcre_free(tree->pcre_pattern); + } + if (n->pcre_extra) { + pcre_free_study(n->pcre_extra); + } if (tree->combined_pattern) zfree(tree->combined_pattern); if (tree->ov) @@ -205,14 +205,12 @@ void r3_tree_compile_patterns(node * n) { int erroffset; unsigned int option_bits = 0; - /* - if (n->pcre_pattern) - zfree(n->pcre_pattern); - if (n->pcre_extra) - zfree(n->pcre_extra); - */ - - // n->pcre_pattern; + if (tree->pcre_pattern) { + pcre_free(tree->pcre_pattern); + } + if (n->pcre_extra) { + pcre_free_study(n->pcre_extra); + } n->pcre_pattern = pcre_compile( n->combined_pattern, /* the pattern */ option_bits, /* default options */ From 175b5942348289494b7915d133f7c0beead2e54b Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 18:20:48 +0800 Subject: [PATCH 18/56] Fix --- src/node.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/node.c b/src/node.c index 33de25d..705f87a 100644 --- a/src/node.c +++ b/src/node.c @@ -205,8 +205,8 @@ void r3_tree_compile_patterns(node * n) { int erroffset; unsigned int option_bits = 0; - if (tree->pcre_pattern) { - pcre_free(tree->pcre_pattern); + if (n->pcre_pattern) { + pcre_free(n->pcre_pattern); } if (n->pcre_extra) { pcre_free_study(n->pcre_extra); From 834d87173a0ed8ccee0f0e624b2206521e506c6a Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 18:21:15 +0800 Subject: [PATCH 19/56] Fix variable name --- src/node.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/node.c b/src/node.c index 705f87a..8ec0aed 100644 --- a/src/node.c +++ b/src/node.c @@ -79,8 +79,8 @@ void r3_tree_free(node * tree) { if (tree->pcre_pattern) { pcre_free(tree->pcre_pattern); } - if (n->pcre_extra) { - pcre_free_study(n->pcre_extra); + if (tree->pcre_extra) { + pcre_free_study(tree->pcre_extra); } if (tree->combined_pattern) zfree(tree->combined_pattern); From 82ea09e146fc686ba11860902d7760a70201618d Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 18:22:11 +0800 Subject: [PATCH 20/56] when we free it, we know it's not NULL --- src/token.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/token.c b/src/token.c index f5f73eb..5c00e2e 100644 --- a/src/token.c +++ b/src/token.c @@ -21,15 +21,13 @@ str_array * str_array_create(int cap) { } void str_array_free(str_array *l) { - if (l) { - for ( int i = 0; i < l->len ; i++ ) { - char * t = l->tokens[ i ]; - if (t) { - zfree(t); - } + for ( int i = 0; i < l->len ; i++ ) { + char * t = l->tokens[ i ]; + if (t) { + zfree(t); } - zfree(l); } + zfree(l); } bool str_array_is_full(str_array * l) { From 946bac8f3c376979399cd4e33320ee46e85a5117 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 18:26:08 +0800 Subject: [PATCH 21/56] seems like pcre_free_study is not defined on older ubuntu --- src/node.c | 4 ++++ tests/bench_str.csv | 1 + 2 files changed, 5 insertions(+) diff --git a/src/node.c b/src/node.c index 8ec0aed..d7219fa 100644 --- a/src/node.c +++ b/src/node.c @@ -79,9 +79,11 @@ void r3_tree_free(node * tree) { if (tree->pcre_pattern) { pcre_free(tree->pcre_pattern); } + /* if (tree->pcre_extra) { pcre_free_study(tree->pcre_extra); } + */ if (tree->combined_pattern) zfree(tree->combined_pattern); if (tree->ov) @@ -208,9 +210,11 @@ void r3_tree_compile_patterns(node * n) { if (n->pcre_pattern) { pcre_free(n->pcre_pattern); } + /* if (n->pcre_extra) { pcre_free_study(n->pcre_extra); } + */ n->pcre_pattern = pcre_compile( n->combined_pattern, /* the pattern */ option_bits, /* default options */ diff --git a/tests/bench_str.csv b/tests/bench_str.csv index fd0f711..12a32f5 100644 --- a/tests/bench_str.csv +++ b/tests/bench_str.csv @@ -444,3 +444,4 @@ 1400666823,6406965.25 1400666950,6684744.95 1400667091,6771202.47 +1400667800,6695271.95 From b83946e487257b69a0c6354d1775e482834faa9b Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 18:28:21 +0800 Subject: [PATCH 22/56] check macro PCRE_STUDY_JIT_COMPILE to compile pcre with JIT --- src/node.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/node.c b/src/node.c index d7219fa..302c20e 100644 --- a/src/node.c +++ b/src/node.c @@ -79,11 +79,12 @@ void r3_tree_free(node * tree) { if (tree->pcre_pattern) { pcre_free(tree->pcre_pattern); } - /* +#ifdef PCRE_STUDY_JIT_COMPILE if (tree->pcre_extra) { pcre_free_study(tree->pcre_extra); } - */ +#endif + if (tree->combined_pattern) zfree(tree->combined_pattern); if (tree->ov) @@ -210,11 +211,6 @@ void r3_tree_compile_patterns(node * n) { if (n->pcre_pattern) { pcre_free(n->pcre_pattern); } - /* - if (n->pcre_extra) { - pcre_free_study(n->pcre_extra); - } - */ n->pcre_pattern = pcre_compile( n->combined_pattern, /* the pattern */ option_bits, /* default options */ @@ -225,11 +221,16 @@ void r3_tree_compile_patterns(node * n) { printf("PCRE compilation failed at offset %d: %s, pattern: %s\n", erroffset, error, n->combined_pattern); return; } +#ifdef PCRE_STUDY_JIT_COMPILE + if (n->pcre_extra) { + pcre_free_study(n->pcre_extra); + } n->pcre_extra = pcre_study(n->pcre_pattern, 0, &error); if (n->pcre_extra == NULL) { printf("PCRE study failed at offset %s\n", error); return; } +#endif } From 05b698b841c31c3e2cd7379d64ade3618295438e Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 18:31:51 +0800 Subject: [PATCH 23/56] r3_tree_matchl does not return int --- tests/bench_str.csv | 1 + tests/check_tree.c | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/bench_str.csv b/tests/bench_str.csv index 12a32f5..6a7117a 100644 --- a/tests/bench_str.csv +++ b/tests/bench_str.csv @@ -445,3 +445,4 @@ 1400666950,6684744.95 1400667091,6771202.47 1400667800,6695271.95 +1400668268,13174604.57 diff --git a/tests/check_tree.c b/tests/check_tree.c index 88904f1..72af3e1 100644 --- a/tests/check_tree.c +++ b/tests/check_tree.c @@ -72,27 +72,27 @@ START_TEST (test_compile) entry = match_entry_createl( "foo" , strlen("/foo") ); m = r3_tree_matchl( n , "/foo", strlen("/foo"), entry); - ck_assert( (int)m ); + ck_assert( m ); entry = match_entry_createl( "/zoo" , strlen("/zoo") ); m = r3_tree_matchl( n , "/zoo", strlen("/zoo"), entry); - ck_assert( (int)m ); + ck_assert( m ); entry = match_entry_createl( "/bar" , strlen("/bar") ); m = r3_tree_matchl( n , "/bar", strlen("/bar"), entry); - ck_assert( (int)m ); + ck_assert( m ); entry = match_entry_createl( "/xxx" , strlen("/xxx") ); m = r3_tree_matchl( n , "/xxx", strlen("/xxx"), entry); - ck_assert( (int)m ); + ck_assert( m ); entry = match_entry_createl( "/foo/xxx" , strlen("/foo/xxx") ); m = r3_tree_matchl( n , "/foo/xxx", strlen("/foo/xxx"), entry); - ck_assert( (int)m ); + ck_assert( m ); entry = match_entry_createl( "/some_id" , strlen("/some_id") ); m = r3_tree_matchl( n , "/some_id", strlen("/some_id"), entry); - ck_assert( (int)m ); + ck_assert( m ); } END_TEST @@ -107,7 +107,7 @@ START_TEST (test_pcre_patterns_insert) node *matched; matched = r3_tree_matchl(n, "/post/111-222", strlen("/post/111-222"), NULL); - ck_assert((int)matched); + ck_assert(matched); ck_assert(matched->endpoint > 0); // incomplete string shouldn't match @@ -273,7 +273,7 @@ START_TEST(test_pcre_pattern_simple) // r3_tree_dump(n, 0); node *matched; matched = r3_tree_matchl(n, "/user/123", strlen("/user/123"), entry); - fail_if(matched == NULL); + ck_assert(matched); ck_assert(entry->vars->len > 0); ck_assert_str_eq(entry->vars->tokens[0],"123"); r3_tree_free(n); @@ -306,7 +306,7 @@ START_TEST(test_pcre_pattern_more) node *matched; matched = r3_tree_matchl(n, "/user/123", strlen("/user/123"), entry); - fail_if(matched == NULL); + ck_assert(matched); ck_assert(entry->vars->len > 0); ck_assert_str_eq(entry->vars->tokens[0],"123"); @@ -315,13 +315,13 @@ START_TEST(test_pcre_pattern_more) ck_assert_int_eq( *((int*) matched->data), var1); matched = r3_tree_matchl(n, "/user2/123", strlen("/user2/123"), entry); - fail_if(matched == NULL); + ck_assert(matched); ck_assert(entry->vars->len > 0); ck_assert_str_eq(entry->vars->tokens[0],"123"); ck_assert_int_eq( *((int*)matched->data), var2); matched = r3_tree_matchl(n, "/user3/123", strlen("/user3/123"), entry); - fail_if(matched == NULL); + ck_assert(matched); ck_assert(entry->vars->len > 0); ck_assert_str_eq(entry->vars->tokens[0],"123"); ck_assert_int_eq( *((int*)matched->data), var3); From 27531aac5633a5aa20fbe481b7378ec69d47cb93 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 18:36:52 +0800 Subject: [PATCH 24/56] remove benchmark result with debug flags --- tests/bench_str.csv | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/bench_str.csv b/tests/bench_str.csv index 6a7117a..61887e0 100644 --- a/tests/bench_str.csv +++ b/tests/bench_str.csv @@ -438,11 +438,6 @@ 1400607695,13583203.56 1400607698,13630627.45 1400607700,13972490.11 -1400655532,6648299.49 -1400655542,6685928.45 1400659046,19754150.71 -1400666823,6406965.25 -1400666950,6684744.95 -1400667091,6771202.47 -1400667800,6695271.95 1400668268,13174604.57 +1400668574,13632260.72 From 79fb3a3b8ad59021f55d80d6f9483fd72ca13d19 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 18:50:45 +0800 Subject: [PATCH 25/56] check_slug: free memory --- tests/check_slug.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/check_slug.c b/tests/check_slug.c index 8b0adcd..88b11e6 100644 --- a/tests/check_slug.c +++ b/tests/check_slug.c @@ -15,16 +15,21 @@ START_TEST (test_slug_compile) { char * path = "/user/{id}"; - ck_assert_str_eq( slug_compile(path, strlen(path) ) , "/user/([^/]+)" ); + char * c = NULL; + ck_assert_str_eq( c = slug_compile(path, strlen(path) ) , "/user/([^/]+)" ); + zfree(c); char * path2 = "/what/{id}-foo"; - ck_assert_str_eq( slug_compile(path2, strlen(path2) ) , "/what/([^/]+)-foo" ); + ck_assert_str_eq( c = slug_compile(path2, strlen(path2) ) , "/what/([^/]+)-foo" ); + zfree(c); char * path3 = "-{id}"; - ck_assert_str_eq(slug_compile(path3, strlen(path3)), "-([^/]+)" ); + ck_assert_str_eq( c = slug_compile(path3, strlen(path3)), "-([^/]+)" ); + zfree(c); char * path4 = "-{idx:\\d{3}}"; - ck_assert_str_eq(slug_compile(path4, strlen(path4)), "-(\\d{3})" ); + ck_assert_str_eq( c = slug_compile(path4, strlen(path4)), "-(\\d{3})" ); + zfree(c); } END_TEST From e7c1b3c29d483afa534be9436f9394dbd818f9c2 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 19:03:05 +0800 Subject: [PATCH 26/56] remove benchmark result with debug option --- tests/bench_str.csv | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/bench_str.csv b/tests/bench_str.csv index 6a7117a..29dcc4c 100644 --- a/tests/bench_str.csv +++ b/tests/bench_str.csv @@ -438,11 +438,5 @@ 1400607695,13583203.56 1400607698,13630627.45 1400607700,13972490.11 -1400655532,6648299.49 -1400655542,6685928.45 1400659046,19754150.71 -1400666823,6406965.25 -1400666950,6684744.95 -1400667091,6771202.47 -1400667800,6695271.95 1400668268,13174604.57 From 4e702e92fb5f48c9f5bcc616ab700b173f4deee1 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 21 May 2014 14:06:23 +0300 Subject: [PATCH 27/56] Installed the compiled shared libraries globally. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index d27cbd1..067ee3a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,7 @@ script: - ./autogen.sh - ./configure $CONFIGURE_OPTION - make + - sudo make install - if [ "x$VALGRIND" == xyes ]; then make check > /dev/null 2>&1; else make check; fi - if [ "x$VALGRIND" == xyes ]; then valgrind ./tests/.libs/* -v --trace-children=yes --show-leak-kinds=full --leak-check=full; fi From 08a127d30c46ace41a40267fab87fbd03279b7b9 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 21 May 2014 14:19:13 +0300 Subject: [PATCH 28/56] Making sure that the .so files are on the ldconfig path. --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 067ee3a..c719ba5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,9 @@ install: - if [ "x$COVERALLS" == xyes ]; then sudo pip install cpp-coveralls; fi - if [ "x$VALGRIND" == xyes ]; then sudo apt-get install valgrind; fi +before_script: + - echo "/usr/local/lib/" >> /etc/ld.so.conf + - ldconfig script: - ./autogen.sh - ./configure $CONFIGURE_OPTION From a23c6c487505e5e0cf5fdd4cef5d6fdba8c6fd2c Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 21 May 2014 14:22:24 +0300 Subject: [PATCH 29/56] Forgot to sudo. --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c719ba5..7031441 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,8 +19,8 @@ install: - if [ "x$VALGRIND" == xyes ]; then sudo apt-get install valgrind; fi before_script: - - echo "/usr/local/lib/" >> /etc/ld.so.conf - - ldconfig + - sudo echo "/usr/local/lib/" >> /etc/ld.so.conf + - sudo ldconfig script: - ./autogen.sh - ./configure $CONFIGURE_OPTION From 5dbef0cf24b299cec1c1e802c1e093d1293d4d5c Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 21 May 2014 14:30:06 +0300 Subject: [PATCH 30/56] Seriously travis? sudo won't give me permissions?? --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7031441..c7a1ad4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ matrix: - compiler: gcc env: CONFIGURE_OPTION='--enable-debug' COVERALLS=yes VALGRIND=no - compiler: gcc - env: CONFIGURE_OPTION='--enable-debug' COVERALLS=yes VALGRIND=yes + env: CONFIGURE_OPTION='--enable-debug' COVERALLS=yes VALGRIND=yes LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/ install: - sudo apt-get update -qq @@ -19,7 +19,6 @@ install: - if [ "x$VALGRIND" == xyes ]; then sudo apt-get install valgrind; fi before_script: - - sudo echo "/usr/local/lib/" >> /etc/ld.so.conf - sudo ldconfig script: - ./autogen.sh From 33eea2592690a59bf4d103133fa158b8f18b8bb7 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 20:30:35 +0800 Subject: [PATCH 31/56] check length --- src/str.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/str.c b/src/str.c index c83cd92..673eee4 100644 --- a/src/str.c +++ b/src/str.c @@ -41,15 +41,15 @@ char * inside_slug(char * needle, int needle_len, char *offset) { char * s1 = offset; char * s2 = offset; - while( s1 >= needle ) { + while( s1 >= needle && (s1 - needle < needle_len) ) { if ( *s1 == '{' ) { break; } s1--; } - char *end = needle+ needle_len; - while( s2 < end ) { + char * end = needle + needle_len; + while( (s2 + 1) < end ) { if ( *s2 == '}' ) { break; } From e7aaaac63f2b9499f0a4e4600f6b74f129194afa Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 21:58:30 +0800 Subject: [PATCH 32/56] delete generated config.h --- config.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/config.h b/config.h index e766a43..b57282b 100644 --- a/config.h +++ b/config.h @@ -2,7 +2,7 @@ /* config.h.in. Generated from configure.ac by autoheader. */ /* "whether graphviz is enable" */ -#define ENABLE_GRAPHVIZ test "x$enable_graphviz" = "xyes" +#define ENABLE_GRAPHVIZ test "x$enable_graphviz" == "xyes" /* Define to 1 if you have the header file. */ #define HAVE_DLFCN_H 1 @@ -13,6 +13,9 @@ /* Define to 1 if you have the header file. */ #define HAVE_INTTYPES_H 1 +/* Define to 1 if you have the header file. */ +#define HAVE_JEMALLOC_JEMALLOC_H 1 + /* Define to 1 if your system has a GNU libc compatible `malloc' function, and to 0 otherwise. */ #define HAVE_MALLOC 1 @@ -91,6 +94,9 @@ /* Define to 1 if you have the ANSI C header files. */ #define STDC_HEADERS 1 +/* "use jemalloc" */ +#define USE_JEMALLOC test "x$found_jemalloc" = "xyes" + /* Version number of package */ #define VERSION "1.0.0" From 45690da3dcf520e1c174a16c45075ed81995454c Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 22:05:10 +0800 Subject: [PATCH 33/56] found flag for slug char --- src/str.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/str.c b/src/str.c index 673eee4..581cf63 100644 --- a/src/str.c +++ b/src/str.c @@ -41,8 +41,12 @@ char * inside_slug(char * needle, int needle_len, char *offset) { char * s1 = offset; char * s2 = offset; + short found_s1 = 0; + short found_s2 = 0; + while( s1 >= needle && (s1 - needle < needle_len) ) { if ( *s1 == '{' ) { + found_s1 = 1; break; } s1--; @@ -51,12 +55,12 @@ char * inside_slug(char * needle, int needle_len, char *offset) { char * end = needle + needle_len; while( (s2 + 1) < end ) { if ( *s2 == '}' ) { + found_s2 = 1; break; } s2++; } - - if ( *s1 == '{' && *s2 == '}' ) { + if (found_s1 && found_s2) { return s1; } return NULL; From f6d2502896886f5d7c15e7cd8f7814aeb6244fcb Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 22:08:25 +0800 Subject: [PATCH 34/56] include jemalloc.h --- src/zmalloc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/zmalloc.c b/src/zmalloc.c index 72447d8..fad1c2b 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -61,6 +61,7 @@ void zlibc_free(void *ptr) { #define realloc(ptr,size) tc_realloc(ptr,size) #define free(ptr) tc_free(ptr) #elif defined(USE_JEMALLOC) +#include #define malloc(size) je_malloc(size) #define calloc(count,size) je_calloc(count,size) #define realloc(ptr,size) je_realloc(ptr,size) From 7e13b0be9c2f99b301de7cad1d9215f389aee2ba Mon Sep 17 00:00:00 2001 From: mattn Date: Thu, 22 May 2014 01:14:37 +0900 Subject: [PATCH 35/56] AM_PATH_CHECK is deprecated, so check it defined --- configure.ac | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index b6fd07b..224c326 100644 --- a/configure.ac +++ b/configure.ac @@ -8,9 +8,12 @@ AC_PROG_CC AC_PROG_CC_STDC # AM_PATH_CHECK() -AM_PATH_CHECK(,[have_check="yes"], +PKG_CHECK_MODULES(CHECK,[check >= 0.9.4],:,[ + ifdef([AM_PATH_CHECK], + [AM_PATH_CHECK(,[have_check="yes"])], AC_MSG_WARN([Check not found; cannot run unit tests!]) - [have_check="no"]) + [have_check="no"])] +]) AM_CONDITIONAL(HAVE_CHECK, test x"$have_check" = "xyes") AC_CHECK_HEADERS([stdlib.h string.h sys/time.h]) From eaa11d5380cd28ed6832118ef063f6aa7944824e Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 22:05:10 +0800 Subject: [PATCH 36/56] found flag for slug char --- src/str.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/str.c b/src/str.c index 673eee4..581cf63 100644 --- a/src/str.c +++ b/src/str.c @@ -41,8 +41,12 @@ char * inside_slug(char * needle, int needle_len, char *offset) { char * s1 = offset; char * s2 = offset; + short found_s1 = 0; + short found_s2 = 0; + while( s1 >= needle && (s1 - needle < needle_len) ) { if ( *s1 == '{' ) { + found_s1 = 1; break; } s1--; @@ -51,12 +55,12 @@ char * inside_slug(char * needle, int needle_len, char *offset) { char * end = needle + needle_len; while( (s2 + 1) < end ) { if ( *s2 == '}' ) { + found_s2 = 1; break; } s2++; } - - if ( *s1 == '{' && *s2 == '}' ) { + if (found_s1 && found_s2) { return s1; } return NULL; From e8653eeb289f591f218b0e025f71fecd1ee6e61e Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 22:08:25 +0800 Subject: [PATCH 37/56] include jemalloc.h --- src/zmalloc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/zmalloc.c b/src/zmalloc.c index 72447d8..fad1c2b 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -61,6 +61,7 @@ void zlibc_free(void *ptr) { #define realloc(ptr,size) tc_realloc(ptr,size) #define free(ptr) tc_free(ptr) #elif defined(USE_JEMALLOC) +#include #define malloc(size) je_malloc(size) #define calloc(count,size) je_calloc(count,size) #define realloc(ptr,size) je_realloc(ptr,size) From a3866114e343edc50fb47bd5253ca16ced703340 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 21:59:46 +0800 Subject: [PATCH 38/56] Add --with-malloc option to check jemalloc --- configure.ac | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index 224c326..bfafdcd 100644 --- a/configure.ac +++ b/configure.ac @@ -46,9 +46,19 @@ AC_ARG_ENABLE(debug, [enable debug]), , enable_debug=unset) -AM_CONDITIONAL(ENABLE_DEBUG, test "x$enable_debug" = "xyes") -AM_CONDITIONAL(ENABLE_GRAPHVIZ, test "x$enable_graphviz" = "xyes") -AC_DEFINE(ENABLE_GRAPHVIZ, test "x$enable_graphviz" = "xyes", "whether graphviz is enable") +AC_ARG_WITH([malloc], + AS_HELP_STRING([--without-malloc], [Use the default malloc])) + +AS_IF([test "x$with_malloc" != "xno"], + [AC_CHECK_HEADERS([jemalloc/jemalloc.h], [ + found_jemalloc=yes; break + ])]) +AM_CONDITIONAL(USE_JEMALLOC, test "x$found_jemalloc" = "xyes") +AC_DEFINE(USE_JEMALLOC, test "x$found_jemalloc" = "xyes" , "use jemalloc") + +AM_CONDITIONAL(ENABLE_DEBUG, test "x$enable_debug" == "xyes") +AM_CONDITIONAL(ENABLE_GRAPHVIZ, test "x$enable_graphviz" == "xyes") +AC_DEFINE(ENABLE_GRAPHVIZ, test "x$enable_graphviz" == "xyes", "whether graphviz is enable") PKG_CHECK_MODULES(DEPS, [libpcre check]) From e48538c52690836eb5b762fd4848ca91a7aa5c81 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 22:00:30 +0800 Subject: [PATCH 39/56] check USE_JEMALLOC flag in Makefile.am --- src/Makefile.am | 4 ++++ tests/Makefile.am | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/src/Makefile.am b/src/Makefile.am index 2e6234c..f28be76 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -6,6 +6,10 @@ libr3_la_SOURCES = node.c edge.c str.c token.c zmalloc.c libr3_la_LIBADD=$(DEPS_LIBS) AM_CFLAGS=$(DEPS_CFLAGS) -I$(top_builddir) -I$(top_builddir)/include -Wall +if USE_JEMALLOC +AM_CFLAGS += -ljemalloc +endif + if ENABLE_DEBUG AM_CFLAGS += -ggdb -fprofile-arcs -ftest-coverage endif diff --git a/tests/Makefile.am b/tests/Makefile.am index d0b1388..21dd302 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -8,6 +8,7 @@ TESTS = check_tree AM_CFLAGS = -ggdb $(DEPS_CFLAGS) -I$(top_builddir) -I$(top_builddir)/include @CHECK_CFLAGS@ AM_LDFLAGS = $(DEPS_LIBS) -L$(top_builddir)/src -lr3 @CHECK_LIBS@ + noinst_HEADERS = \ bench.h \ $(NULL) @@ -16,6 +17,10 @@ dist_noinst_DATA = \ bench_str.csv \ $(NULL) +if USE_JEMALLOC +AM_CFLAGS += -ljemalloc +endif + if ENABLE_GRAPHVIZ TESTS += check_gvc check_gvc_SOURCES = check_gvc.c bench.c From 03c351dfe0df0e747352f437b919804b024b40ca Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 22:00:58 +0800 Subject: [PATCH 40/56] Update autoheader template --- config.h.in | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config.h.in b/config.h.in index c74bb06..cdfbdff 100644 --- a/config.h.in +++ b/config.h.in @@ -12,6 +12,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_INTTYPES_H +/* Define to 1 if you have the header file. */ +#undef HAVE_JEMALLOC_JEMALLOC_H + /* Define to 1 if your system has a GNU libc compatible `malloc' function, and to 0 otherwise. */ #undef HAVE_MALLOC @@ -90,6 +93,9 @@ /* Define to 1 if you have the ANSI C header files. */ #undef STDC_HEADERS +/* "use jemalloc" */ +#undef USE_JEMALLOC + /* Version number of package */ #undef VERSION From 2c4236f5b85e494dff58ee835da69afee564b064 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 22:02:27 +0800 Subject: [PATCH 41/56] include zmalloc.h --- tests/check_slug.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/check_slug.c b/tests/check_slug.c index 88b11e6..2662981 100644 --- a/tests/check_slug.c +++ b/tests/check_slug.c @@ -11,6 +11,7 @@ #include "r3.h" #include "r3_str.h" #include "str_array.h" +#include "zmalloc.h" START_TEST (test_slug_compile) { From 4d0c22d8ec5d530de8ba4997da98dfc5da8344cf Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 23:14:19 +0800 Subject: [PATCH 42/56] fix linking issue --- include/zmalloc.h | 2 ++ tests/bench_str.csv | 1 + 2 files changed, 3 insertions(+) diff --git a/include/zmalloc.h b/include/zmalloc.h index 383f377..c8ef195 100644 --- a/include/zmalloc.h +++ b/include/zmalloc.h @@ -48,12 +48,14 @@ #elif defined(USE_JEMALLOC) #define ZMALLOC_LIB ("jemalloc-" __xstr(JEMALLOC_VERSION_MAJOR) "." __xstr(JEMALLOC_VERSION_MINOR) "." __xstr(JEMALLOC_VERSION_BUGFIX)) #include +/* #if (JEMALLOC_VERSION_MAJOR == 2 && JEMALLOC_VERSION_MINOR >= 1) || (JEMALLOC_VERSION_MAJOR > 2) #define HAVE_MALLOC_SIZE 1 #define zmalloc_size(p) je_malloc_usable_size(p) #else #error "Newer version of jemalloc required" #endif +*/ #elif defined(__APPLE__) #include diff --git a/tests/bench_str.csv b/tests/bench_str.csv index 61887e0..e8dd1f9 100644 --- a/tests/bench_str.csv +++ b/tests/bench_str.csv @@ -441,3 +441,4 @@ 1400659046,19754150.71 1400668268,13174604.57 1400668574,13632260.72 +1400681414,10832905.89 From 07ef938e3d1da5a79b7be7d948caf999487739f2 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 23:15:38 +0800 Subject: [PATCH 43/56] check jemalloc version --- include/zmalloc.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/include/zmalloc.h b/include/zmalloc.h index c8ef195..eaeb765 100644 --- a/include/zmalloc.h +++ b/include/zmalloc.h @@ -48,14 +48,13 @@ #elif defined(USE_JEMALLOC) #define ZMALLOC_LIB ("jemalloc-" __xstr(JEMALLOC_VERSION_MAJOR) "." __xstr(JEMALLOC_VERSION_MINOR) "." __xstr(JEMALLOC_VERSION_BUGFIX)) #include -/* -#if (JEMALLOC_VERSION_MAJOR == 2 && JEMALLOC_VERSION_MINOR >= 1) || (JEMALLOC_VERSION_MAJOR > 2) +// #if (JEMALLOC_VERSION_MAJOR == 2 && JEMALLOC_VERSION_MINOR >= 1) || (JEMALLOC_VERSION_MAJOR > 2) +#if JEMALLOC_VERSION_MAJOR > 2 #define HAVE_MALLOC_SIZE 1 #define zmalloc_size(p) je_malloc_usable_size(p) #else #error "Newer version of jemalloc required" #endif -*/ #elif defined(__APPLE__) #include From 2d4bd481161c6fd5f08bb739086232051906f137 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 23:24:30 +0800 Subject: [PATCH 44/56] Fix autoconf checking --- configure.ac | 2 +- tests/bench_str.csv | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index bfafdcd..779a4d9 100644 --- a/configure.ac +++ b/configure.ac @@ -49,7 +49,7 @@ AC_ARG_ENABLE(debug, AC_ARG_WITH([malloc], AS_HELP_STRING([--without-malloc], [Use the default malloc])) -AS_IF([test "x$with_malloc" != "xno"], +AS_IF([test "x$with_malloc" == "xjemalloc"], [AC_CHECK_HEADERS([jemalloc/jemalloc.h], [ found_jemalloc=yes; break ])]) diff --git a/tests/bench_str.csv b/tests/bench_str.csv index e8dd1f9..a10d1c5 100644 --- a/tests/bench_str.csv +++ b/tests/bench_str.csv @@ -442,3 +442,4 @@ 1400668268,13174604.57 1400668574,13632260.72 1400681414,10832905.89 +1400685490,13185955.87 From 594b10cbcb2398a17ac07cca24cd1a6907278965 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 23:26:01 +0800 Subject: [PATCH 45/56] travis-ci is using jemalloc 2.1+ --- include/zmalloc.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/zmalloc.h b/include/zmalloc.h index eaeb765..383f377 100644 --- a/include/zmalloc.h +++ b/include/zmalloc.h @@ -48,8 +48,7 @@ #elif defined(USE_JEMALLOC) #define ZMALLOC_LIB ("jemalloc-" __xstr(JEMALLOC_VERSION_MAJOR) "." __xstr(JEMALLOC_VERSION_MINOR) "." __xstr(JEMALLOC_VERSION_BUGFIX)) #include -// #if (JEMALLOC_VERSION_MAJOR == 2 && JEMALLOC_VERSION_MINOR >= 1) || (JEMALLOC_VERSION_MAJOR > 2) -#if JEMALLOC_VERSION_MAJOR > 2 +#if (JEMALLOC_VERSION_MAJOR == 2 && JEMALLOC_VERSION_MINOR >= 1) || (JEMALLOC_VERSION_MAJOR > 2) #define HAVE_MALLOC_SIZE 1 #define zmalloc_size(p) je_malloc_usable_size(p) #else From a8cec68f725611ce2328f59228af78954e1968c5 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 23:29:27 +0800 Subject: [PATCH 46/56] use -std=c99 --- src/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile.am b/src/Makefile.am index f28be76..6616a97 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -4,7 +4,7 @@ libr3_la_SOURCES = node.c edge.c str.c token.c zmalloc.c # libr3_la_LDFLAGS = -export-symbols-regex '^r3_|^match_' libr3_la_LIBADD=$(DEPS_LIBS) -AM_CFLAGS=$(DEPS_CFLAGS) -I$(top_builddir) -I$(top_builddir)/include -Wall +AM_CFLAGS=$(DEPS_CFLAGS) -I$(top_builddir) -I$(top_builddir)/include -Wall -std=c99 if USE_JEMALLOC AM_CFLAGS += -ljemalloc From dc24b7cb9725285610bbb7b6e0230d506ba2561c Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 23:46:33 +0800 Subject: [PATCH 47/56] remove jemalloc --- tests/Makefile.am | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/Makefile.am b/tests/Makefile.am index 21dd302..e52bd8d 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -17,10 +17,6 @@ dist_noinst_DATA = \ bench_str.csv \ $(NULL) -if USE_JEMALLOC -AM_CFLAGS += -ljemalloc -endif - if ENABLE_GRAPHVIZ TESTS += check_gvc check_gvc_SOURCES = check_gvc.c bench.c From c1abe926bf2f5de488e190964db29b2cbb626c58 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 23:47:26 +0800 Subject: [PATCH 48/56] Add --with-malloc to travis config --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c7a1ad4..e4b4774 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,9 +6,9 @@ compiler: matrix: include: - compiler: gcc - env: CONFIGURE_OPTION='--enable-debug' COVERALLS=yes VALGRIND=no + env: CONFIGURE_OPTION='--enable-debug --with-malloc=jemalloc' COVERALLS=yes VALGRIND=no - compiler: gcc - env: CONFIGURE_OPTION='--enable-debug' COVERALLS=yes VALGRIND=yes LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/ + env: CONFIGURE_OPTION='--enable-debug --with-malloc=jemalloc' COVERALLS=yes VALGRIND=yes LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/ install: - sudo apt-get update -qq From 06324f48abf1396548209d2f1762c1ebb440846b Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 23:50:04 +0800 Subject: [PATCH 49/56] delete generated config.h --- config.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.h b/config.h index b57282b..eae0dba 100644 --- a/config.h +++ b/config.h @@ -14,7 +14,7 @@ #define HAVE_INTTYPES_H 1 /* Define to 1 if you have the header file. */ -#define HAVE_JEMALLOC_JEMALLOC_H 1 +/* #undef HAVE_JEMALLOC_JEMALLOC_H */ /* Define to 1 if your system has a GNU libc compatible `malloc' function, and to 0 otherwise. */ From aed75694e69702065cb42ce70900b664453cff84 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 21 May 2014 23:50:22 +0800 Subject: [PATCH 50/56] ignore config.h --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 8ed8af3..04e1a1d 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ m4/ missing test-driver test-suite.log +config.h tests/*.log tests/*.trs From 1fdfaef16e9ec39b6531cb1b6e104017e0d83106 Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 22 May 2014 11:34:37 +0800 Subject: [PATCH 51/56] what if we always use -ljemalloc --- src/Makefile.am | 1 + tests/Makefile.am | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/src/Makefile.am b/src/Makefile.am index 6616a97..9ad21b8 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -9,6 +9,7 @@ AM_CFLAGS=$(DEPS_CFLAGS) -I$(top_builddir) -I$(top_builddir)/include -Wall -std= if USE_JEMALLOC AM_CFLAGS += -ljemalloc endif +AM_CFLAGS += -ljemalloc if ENABLE_DEBUG AM_CFLAGS += -ggdb -fprofile-arcs -ftest-coverage diff --git a/tests/Makefile.am b/tests/Makefile.am index e52bd8d..534a841 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -17,6 +17,11 @@ dist_noinst_DATA = \ bench_str.csv \ $(NULL) +if USE_JEMALLOC +AM_CFLAGS += -ljemalloc +endif +AM_CFLAGS += -ljemalloc + if ENABLE_GRAPHVIZ TESTS += check_gvc check_gvc_SOURCES = check_gvc.c bench.c From c503a6d5d346749a738e3b26b339e8b960925c99 Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 22 May 2014 11:41:51 +0800 Subject: [PATCH 52/56] use jemalloc only in jemalloc major version > 2 --- include/zmalloc.h | 2 +- src/zmalloc.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/zmalloc.h b/include/zmalloc.h index 383f377..cacfb92 100644 --- a/include/zmalloc.h +++ b/include/zmalloc.h @@ -45,7 +45,7 @@ #error "Newer version of tcmalloc required" #endif -#elif defined(USE_JEMALLOC) +#elif defined(USE_JEMALLOC) && (JEMALLOC_VERSION_MAJOR > 2) #define ZMALLOC_LIB ("jemalloc-" __xstr(JEMALLOC_VERSION_MAJOR) "." __xstr(JEMALLOC_VERSION_MINOR) "." __xstr(JEMALLOC_VERSION_BUGFIX)) #include #if (JEMALLOC_VERSION_MAJOR == 2 && JEMALLOC_VERSION_MINOR >= 1) || (JEMALLOC_VERSION_MAJOR > 2) diff --git a/src/zmalloc.c b/src/zmalloc.c index fad1c2b..44f6d6a 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -60,7 +60,7 @@ void zlibc_free(void *ptr) { #define calloc(count,size) tc_calloc(count,size) #define realloc(ptr,size) tc_realloc(ptr,size) #define free(ptr) tc_free(ptr) -#elif defined(USE_JEMALLOC) +#elif defined(USE_JEMALLOC) && (JEMALLOC_VERSION_MAJOR > 2) #include #define malloc(size) je_malloc(size) #define calloc(count,size) je_calloc(count,size) From 9d26a53a08f3be6631fd880d6d592426b282efd6 Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 22 May 2014 11:42:19 +0800 Subject: [PATCH 53/56] update Makefile.am --- src/Makefile.am | 1 - tests/Makefile.am | 1 - 2 files changed, 2 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index 9ad21b8..6616a97 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -9,7 +9,6 @@ AM_CFLAGS=$(DEPS_CFLAGS) -I$(top_builddir) -I$(top_builddir)/include -Wall -std= if USE_JEMALLOC AM_CFLAGS += -ljemalloc endif -AM_CFLAGS += -ljemalloc if ENABLE_DEBUG AM_CFLAGS += -ggdb -fprofile-arcs -ftest-coverage diff --git a/tests/Makefile.am b/tests/Makefile.am index 534a841..21dd302 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -20,7 +20,6 @@ dist_noinst_DATA = \ if USE_JEMALLOC AM_CFLAGS += -ljemalloc endif -AM_CFLAGS += -ljemalloc if ENABLE_GRAPHVIZ TESTS += check_gvc From 91097e3119ed8ad8c63f54915e24abcb2fc1b036 Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 22 May 2014 11:54:01 +0800 Subject: [PATCH 54/56] enable jemalloc only when major version > 2 --- configure.ac | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/configure.ac b/configure.ac index 779a4d9..10a4ce2 100644 --- a/configure.ac +++ b/configure.ac @@ -53,12 +53,37 @@ AS_IF([test "x$with_malloc" == "xjemalloc"], [AC_CHECK_HEADERS([jemalloc/jemalloc.h], [ found_jemalloc=yes; break ])]) -AM_CONDITIONAL(USE_JEMALLOC, test "x$found_jemalloc" = "xyes") -AC_DEFINE(USE_JEMALLOC, test "x$found_jemalloc" = "xyes" , "use jemalloc") -AM_CONDITIONAL(ENABLE_DEBUG, test "x$enable_debug" == "xyes") -AM_CONDITIONAL(ENABLE_GRAPHVIZ, test "x$enable_graphviz" == "xyes") -AC_DEFINE(ENABLE_GRAPHVIZ, test "x$enable_graphviz" == "xyes", "whether graphviz is enable") +if test "x$found_jemalloc" == "xyes" ; then + +AC_MSG_CHECKING([Checking jemalloc version]) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], + [[ + #ifdef JEMALLOC_VERSION_MAJOR > 2 + return 0; + #endif + return 1; + ]])], + [ + AC_MSG_RESULT([yes]) + AC_DEFINE_UNQUOTED([USE_JEMALLOC], 1, [Define to 1 if you have the PATH_MAX macro.]) + have_jemalloc=yes + ], + [ + AC_MSG_RESULT([no]) + AC_DEFINE_UNQUOTED([USE_JEMALLOC], 0, [Define to 1 if you have the PATH_MAX macro.]) + have_jemalloc=no + ] + ) +fi +AM_CONDITIONAL(USE_JEMALLOC, test "x$have_jemalloc" = "xyes") + +# AM_CONDITIONAL(USE_JEMALLOC, test "x$found_jemalloc" = "xyes") +# AC_DEFINE(USE_JEMALLOC, test "x$found_jemalloc" = "xyes" , "use jemalloc") + +AM_CONDITIONAL(ENABLE_DEBUG, test "x$enable_debug" = "xyes") +AM_CONDITIONAL(ENABLE_GRAPHVIZ, test "x$enable_graphviz" = "xyes") +AC_DEFINE(ENABLE_GRAPHVIZ, test "x$enable_graphviz" = "xyes", "whether graphviz is enable") PKG_CHECK_MODULES(DEPS, [libpcre check]) From f5222c6bc29f34f7fc15c20df643f1acf32e6224 Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 22 May 2014 11:54:42 +0800 Subject: [PATCH 55/56] update config.h.in --- config.h.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.h.in b/config.h.in index cdfbdff..7ae1a2a 100644 --- a/config.h.in +++ b/config.h.in @@ -93,7 +93,7 @@ /* Define to 1 if you have the ANSI C header files. */ #undef STDC_HEADERS -/* "use jemalloc" */ +/* Define to 1 if you have the PATH_MAX macro. */ #undef USE_JEMALLOC /* Version number of package */ From 18b2cf644d9e81f9508b07162fc44abc20e07e20 Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 22 May 2014 11:59:45 +0800 Subject: [PATCH 56/56] update README --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0cb4045..3306ef2 100644 --- a/README.md +++ b/README.md @@ -206,10 +206,13 @@ Install make check # run tests sudo make install -### Enable Graphviz +#### Enable Graphviz ./configure --enable-graphviz +#### With jemalloc + + ./configure --with-malloc=jemalloc License