Repository: trafficserver Updated Branches: refs/heads/master a2328d635 -> e56d1f9eb
TS-3918: add a runtime option to disable freelists - use an unsigned allocation count in ProxyAllocator - add a command-line option to disable the freelist - always use the thread proxy allocators - document the --disable_freelist option to traffic_server - remove the --disable-freelist build option - remove unnecessary fastmemtotal global - update freelist stats when using malloc - fix test_RamCache to allocate IOBufferData objects properly Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/e56d1f9e Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/e56d1f9e Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/e56d1f9e Branch: refs/heads/master Commit: e56d1f9eb6f55b0257d1fb758b773dea468159ee Parents: a2328d6 Author: James Peach <[email protected]> Authored: Mon Sep 21 10:49:30 2015 -0700 Committer: James Peach <[email protected]> Committed: Thu Oct 8 20:59:33 2015 -0700 ---------------------------------------------------------------------- cmd/traffic_layout/traffic_layout.cc | 1 - configure.ac | 15 --- doc/reference/commands/traffic_server.en.rst | 9 ++ iocore/cache/CacheTest.cc | 2 +- iocore/eventsystem/I_ProxyAllocator.h | 18 +-- lib/ts/ink_config.h.in | 1 - lib/ts/ink_queue.cc | 147 ++++++++++++++-------- lib/ts/ink_queue.h | 12 +- proxy/Main.cc | 15 ++- 9 files changed, 121 insertions(+), 99 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e56d1f9e/cmd/traffic_layout/traffic_layout.cc ---------------------------------------------------------------------- diff --git a/cmd/traffic_layout/traffic_layout.cc b/cmd/traffic_layout/traffic_layout.cc index cd426f4..b561132 100644 --- a/cmd/traffic_layout/traffic_layout.cc +++ b/cmd/traffic_layout/traffic_layout.cc @@ -96,7 +96,6 @@ produce_features(bool json) print_feature("TS_HAS_SPDY", TS_HAS_SPDY, json); print_feature("TS_HAS_IP_TOS", TS_HAS_IP_TOS, json); print_feature("TS_USE_HWLOC", TS_USE_HWLOC, json); - print_feature("TS_USE_FREELIST", TS_USE_FREELIST, json); print_feature("TS_USE_TLS_NPN", TS_USE_TLS_NPN, json); print_feature("TS_USE_TLS_ALPN", TS_USE_TLS_ALPN, json); print_feature("TS_USE_TLS_SNI", TS_USE_TLS_SNI, json); http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e56d1f9e/configure.ac ---------------------------------------------------------------------- diff --git a/configure.ac b/configure.ac index 0512d5e..1dfffd3 100644 --- a/configure.ac +++ b/configure.ac @@ -344,21 +344,6 @@ AC_ARG_ENABLE([tproxy], AC_MSG_RESULT([$enable_tproxy]) # -# Disable our freelist implementation, reverting it to whatever -# allocator (malloc, tcmalloc or jemalloc) that is in use. This is -# useful for debugging. -# -AC_MSG_CHECKING([whether to disable freelist]) -AC_ARG_ENABLE([freelist], - [AS_HELP_STRING([--disable-freelist],[turn off freelist and use allocators])], - [], - [enable_freelist="yes"] -) -AC_MSG_RESULT([$enable_freelist]) -TS_ARG_ENABLE_VAR([use], [freelist]) -AC_SUBST(use_freelist) - -# # Options for SPDY # AC_MSG_CHECKING([whether to enable spdy]) http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e56d1f9e/doc/reference/commands/traffic_server.en.rst ---------------------------------------------------------------------- diff --git a/doc/reference/commands/traffic_server.en.rst b/doc/reference/commands/traffic_server.en.rst index abbee2a..c8179f7 100644 --- a/doc/reference/commands/traffic_server.en.rst +++ b/doc/reference/commands/traffic_server.en.rst @@ -40,6 +40,15 @@ Description .. option:: -P PORT, --cluster_port PORT +.. option:: -f, --disable_freelist + +In order to improve performance, :program:`traffic_server` caches +commonly used data structures in a set of free object lists. This +option disables these caches, causing :program:`traffic_server` to +use :manpage:`malloc(3)` for every allocation. Though this option +should not commonly be needed, it may be beneficial in memory-constrained +environments or where the working set is highly variable. + .. option:: -o LEVEL, --dprintf_level LEVEL .. option:: -R LEVEL, --regression LEVEL http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e56d1f9e/iocore/cache/CacheTest.cc ---------------------------------------------------------------------- diff --git a/iocore/cache/CacheTest.cc b/iocore/cache/CacheTest.cc index fb55123..096f5c9 100644 --- a/iocore/cache/CacheTest.cc +++ b/iocore/cache/CacheTest.cc @@ -481,7 +481,7 @@ test_RamCache(RegressionTest *t, RamCache *cache) for (int l = 0; l < 10; l++) { for (int i = 0; i < 200; i++) { - IOBufferData *d = new (ats_malloc(sizeof(IOBufferData))) IOBufferData; + IOBufferData *d = THREAD_ALLOC(ioDataAllocator, this_thread()); INK_MD5 md5; d->alloc(BUFFER_SIZE_INDEX_16K); http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e56d1f9e/iocore/eventsystem/I_ProxyAllocator.h ---------------------------------------------------------------------- diff --git a/iocore/eventsystem/I_ProxyAllocator.h b/iocore/eventsystem/I_ProxyAllocator.h index 58a636d..6fbf139 100644 --- a/iocore/eventsystem/I_ProxyAllocator.h +++ b/iocore/eventsystem/I_ProxyAllocator.h @@ -39,7 +39,7 @@ extern int thread_freelist_high_watermark; extern int thread_freelist_low_watermark; struct ProxyAllocator { - int allocated; + unsigned allocated; void *freelist; ProxyAllocator() : allocated(0), freelist(0) {} @@ -49,7 +49,6 @@ template <class C> inline C * thread_alloc(ClassAllocator<C> &a, ProxyAllocator &l) { -#if TS_USE_FREELIST if (l.freelist) { C *v = (C *)l.freelist; l.freelist = *(C **)l.freelist; @@ -57,9 +56,6 @@ thread_alloc(ClassAllocator<C> &a, ProxyAllocator &l) *(void **)v = *(void **)&a.proto.typeObject; return v; } -#else - (void)l; -#endif return a.alloc(); } @@ -67,7 +63,6 @@ template <class C> inline C * thread_alloc_init(ClassAllocator<C> &a, ProxyAllocator &l) { -#if TS_USE_FREELIST if (l.freelist) { C *v = (C *)l.freelist; l.freelist = *(C **)l.freelist; @@ -75,9 +70,6 @@ thread_alloc_init(ClassAllocator<C> &a, ProxyAllocator &l) memcpy((void *)v, (void *)&a.proto.typeObject, sizeof(C)); return v; } -#else - (void)l; -#endif return a.alloc(); } @@ -122,7 +114,6 @@ void thread_freeup(Allocator &a, ProxyAllocator &l); #define THREAD_ALLOC(_a, _t) thread_alloc(::_a, _t->_a) #define THREAD_ALLOC_INIT(_a, _t) thread_alloc_init(::_a, _t->_a) -#if TS_USE_FREELIST #define THREAD_FREE(_p, _a, _t) \ do { \ *(char **)_p = (char *)_t->_a.freelist; \ @@ -131,12 +122,5 @@ void thread_freeup(Allocator &a, ProxyAllocator &l); if (_t->_a.allocated > thread_freelist_high_watermark) \ thread_freeup(::_a, _t->_a); \ } while (0) -#else /* !TS_USE_FREELIST */ -#define THREAD_FREE(_p, _a, _t) \ - do { \ - (void) _t; \ - thread_free(::_a, _p); \ - } while (0) -#endif #endif /* _ProxyAllocator_h_ */ http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e56d1f9e/lib/ts/ink_config.h.in ---------------------------------------------------------------------- diff --git a/lib/ts/ink_config.h.in b/lib/ts/ink_config.h.in index ca4c69e..a2c1ac9 100644 --- a/lib/ts/ink_config.h.in +++ b/lib/ts/ink_config.h.in @@ -71,7 +71,6 @@ #define TS_HAS_SPDY @has_spdy@ #define TS_HAS_IP_TOS @has_ip_tos@ #define TS_USE_HWLOC @use_hwloc@ -#define TS_USE_FREELIST @use_freelist@ #define TS_USE_TLS_NPN @use_tls_npn@ #define TS_USE_TLS_ALPN @use_tls_alpn@ #define TS_USE_TLS_SNI @use_tls_sni@ http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e56d1f9e/lib/ts/ink_queue.cc ---------------------------------------------------------------------- diff --git a/lib/ts/ink_queue.cc b/lib/ts/ink_queue.cc index 758ac5a..79dbd4c 100644 --- a/lib/ts/ink_queue.cc +++ b/lib/ts/ink_queue.cc @@ -65,11 +65,53 @@ inkcoreapi volatile int64_t fastalloc_mem_total = 0; #define DEADBEEF #endif -ink_freelist_list *freelists = NULL; +struct ink_freelist_ops { + void *(*fl_new)(InkFreeList *); + void (*fl_free)(InkFreeList *, void *); + void (*fl_bulkfree)(InkFreeList *, void *, void *, size_t); +}; + +typedef struct _ink_freelist_list { + InkFreeList *fl; + struct _ink_freelist_list *next; +} ink_freelist_list; + +static void *freelist_new(InkFreeList *f); +static void freelist_free(InkFreeList *f, void *item); +static void freelist_bulkfree(InkFreeList *f, void *head, void *tail, size_t num_item); + +static void *malloc_new(InkFreeList *f); +static void malloc_free(InkFreeList *f, void *item); +static void malloc_bulkfree(InkFreeList *f, void *head, void *tail, size_t num_item); + +static const ink_freelist_ops malloc_ops = {malloc_new, malloc_free, malloc_bulkfree}; +static const ink_freelist_ops freelist_ops = {freelist_new, freelist_free, freelist_bulkfree}; +static const ink_freelist_ops *default_ops = &freelist_ops; + +static ink_freelist_list *freelists = NULL; +static const ink_freelist_ops *freelist_freelist_ops = default_ops; + +const InkFreeListOps * +ink_freelist_malloc_ops() +{ + return &malloc_ops; +} + +const InkFreeListOps * +ink_freelist_freelist_ops() +{ + return &freelist_ops; +} -inkcoreapi volatile int64_t freelist_allocated_mem = 0; +void +ink_freelist_init_ops(const InkFreeListOps *ops) +{ + // This *MUST* only be called at startup before any freelists allocate anything. We will certainly crash if object + // allocated from the freelist are freed by malloc. + ink_release_assert(freelist_freelist_ops == default_ops); -#define fl_memadd(_x_) ink_atomic_increment(&freelist_allocated_mem, (int64_t)(_x_)); + freelist_freelist_ops = ops; +} void ink_freelist_init(InkFreeList **fl, const char *name, uint32_t type_size, uint32_t chunk_size, uint32_t alignment) @@ -80,11 +122,14 @@ ink_freelist_init(InkFreeList **fl, const char *name, uint32_t type_size, uint32 /* its safe to add to this global list because ink_freelist_init() is only called from single-threaded initialization code. */ f = (InkFreeList *)ats_memalign(alignment, sizeof(InkFreeList)); + ink_zero(*f); + fll = (ink_freelist_list *)ats_malloc(sizeof(ink_freelist_list)); fll->fl = f; fll->next = freelists; freelists = fll; + f->name = name; /* quick test for power of 2 */ ink_assert(!(alignment & (alignment - 1))); @@ -98,11 +143,6 @@ ink_freelist_init(InkFreeList **fl, const char *name, uint32_t type_size, uint32 } SET_FREELIST_POINTER_VERSION(f->head, FROM_PTR(0), 0); - f->used = 0; - f->allocated = 0; - f->allocated_base = 0; - f->used_base = 0; - f->advice = 0; *fl = f; } @@ -129,11 +169,22 @@ ink_freelist_create(const char *name, uint32_t type_size, uint32_t chunk_size, u int fake_global_for_ink_queue = 0; #endif -int fastmemtotal = 0; void * ink_freelist_new(InkFreeList *f) { -#if TS_USE_FREELIST + void * ptr; + + if (likely(ptr = freelist_freelist_ops->fl_new(f))) { + ink_atomic_increment((int *)&f->used, 1); + ink_atomic_increment(&fastalloc_mem_in_use, (int64_t)f->type_size); + } + + return ptr; +} + +static void * +freelist_new(InkFreeList *f) +{ head_p item; head_p next; int result = 0; @@ -143,11 +194,8 @@ ink_freelist_new(InkFreeList *f) if (TO_PTR(FREELIST_POINTER(item)) == NULL) { uint32_t type_size = f->type_size; uint32_t i; - void *newp = NULL; -#ifdef DEBUG - char *oldsbrk = (char *)sbrk(0), *newsbrk = NULL; -#endif + if (ats_hugepage_enabled()) newp = ats_alloc_hugepage(f->chunk_size * type_size); @@ -158,13 +206,6 @@ ink_freelist_new(InkFreeList *f) newp = ats_malloc(f->chunk_size * type_size); } ats_madvise((caddr_t)newp, f->chunk_size * type_size, f->advice); - fl_memadd(f->chunk_size * type_size); -#ifdef DEBUG - newsbrk = (char *)sbrk(0); - ink_atomic_increment(&fastmemtotal, newsbrk - oldsbrk); -/* printf("fastmem %d, %d, %d\n", f->chunk_size * type_size, - newsbrk - oldsbrk, fastmemtotal); */ -#endif SET_FREELIST_POINTER_VERSION(item, newp, 0); ink_atomic_increment((int *)&f->allocated, f->chunk_size); @@ -180,8 +221,6 @@ ink_freelist_new(InkFreeList *f) #endif ink_freelist_free(f, a); } - ink_atomic_increment((int *)&f->used, f->chunk_size); - ink_atomic_increment(&fastalloc_mem_in_use, (int64_t)f->chunk_size * f->type_size); } else { SET_FREELIST_POINTER_VERSION(next, *ADDRESS_OF_NEXT(TO_PTR(FREELIST_POINTER(item)), 0), FREELIST_VERSION(item) + 1); @@ -205,11 +244,12 @@ ink_freelist_new(InkFreeList *f) } while (result == 0); ink_assert(!((uintptr_t)TO_PTR(FREELIST_POINTER(item)) & (((uintptr_t)f->alignment) - 1))); - ink_atomic_increment((int *)&f->used, 1); - ink_atomic_increment(&fastalloc_mem_in_use, (int64_t)f->type_size); - return TO_PTR(FREELIST_POINTER(item)); -#else // ! TS_USE_FREELIST +} + +static void * +malloc_new(InkFreeList *f) +{ void *newp = NULL; if (f->alignment) @@ -218,13 +258,22 @@ ink_freelist_new(InkFreeList *f) newp = ats_malloc(f->type_size); ats_madvise((caddr_t)newp, f->type_size, f->advice); return newp; -#endif } void ink_freelist_free(InkFreeList *f, void *item) { -#if TS_USE_FREELIST + if (likely(item != NULL)) { + ink_assert(f->used != 0); + freelist_freelist_ops->fl_free(f, item); + ink_atomic_decrement((int *)&f->used, 1); + ink_atomic_decrement(&fastalloc_mem_in_use, f->type_size); + } +} + +static void +freelist_free(InkFreeList *f, void *item) +{ volatile void **adr_of_next = (volatile void **)ADDRESS_OF_NEXT(item, 0); head_p h; head_p item_pair; @@ -262,20 +311,30 @@ ink_freelist_free(InkFreeList *f, void *item) #endif } - ink_atomic_increment((int *)&f->used, -1); - ink_atomic_increment(&fastalloc_mem_in_use, -(int64_t)f->type_size); -#else +} + +static void +malloc_free(InkFreeList *f, void *item) +{ if (f->alignment) ats_memalign_free(item); else ats_free(item); -#endif } void ink_freelist_free_bulk(InkFreeList *f, void *head, void *tail, size_t num_item) { -#if TS_USE_FREELIST + ink_assert(f->used >= num_item); + + freelist_freelist_ops->fl_bulkfree(f, head, tail, num_item); + ink_atomic_decrement((int *)&f->used, num_item); + ink_atomic_decrement(&fastalloc_mem_in_use, f->type_size * num_item); +} + +static void +freelist_bulkfree(InkFreeList *f, void *head, void *tail, size_t num_item) +{ volatile void **adr_of_next = (volatile void **)ADDRESS_OF_NEXT(tail, 0); head_p h; head_p item_pair; @@ -317,10 +376,11 @@ ink_freelist_free_bulk(InkFreeList *f, void *head, void *tail, size_t num_item) result = ink_atomic_cas((int64_t *)&f->head, h.data, item_pair.data); #endif /* TS_HAS_128BIT_CAS */ } +} - ink_atomic_increment((int *)&f->used, -1 * num_item); - ink_atomic_increment(&fastalloc_mem_in_use, -(int64_t)f->type_size * num_item); -#else /* !TS_USE_FREELIST */ +static void +malloc_bulkfree(InkFreeList *f, void *head, void *tail, size_t num_item) +{ void *item = head; // Avoid compiler warnings @@ -335,13 +395,11 @@ ink_freelist_free_bulk(InkFreeList *f, void *head, void *tail, size_t num_item) ats_free(item); } } -#endif /* TS_USE_FREELIST */ } void ink_freelists_snap_baseline() { -#if TS_USE_FREELIST ink_freelist_list *fll; fll = freelists; while (fll) { @@ -349,15 +407,11 @@ ink_freelists_snap_baseline() fll->fl->used_base = fll->fl->used; fll = fll->next; } -#else // ! TS_USE_FREELIST -// TODO? -#endif } void ink_freelists_dump_baselinerel(FILE *f) { -#if TS_USE_FREELIST ink_freelist_list *fll; if (f == NULL) f = stderr; @@ -377,15 +431,11 @@ ink_freelists_dump_baselinerel(FILE *f) } fll = fll->next; } -#else // ! TS_USE_FREELIST - (void)f; -#endif } void ink_freelists_dump(FILE *f) { -#if TS_USE_FREELIST ink_freelist_list *fll; if (f == NULL) f = stderr; @@ -405,9 +455,6 @@ ink_freelists_dump(FILE *f) fll = fll->next; } fprintf(f, " %18" PRIu64 " | %18" PRIu64 " | | TOTAL\n", total_allocated, total_used); -#else // ! TS_USE_FREELIST - (void)f; -#endif } http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e56d1f9e/lib/ts/ink_queue.h ---------------------------------------------------------------------- diff --git a/lib/ts/ink_queue.h b/lib/ts/ink_queue.h index a5f5ea4..8fe5d27 100644 --- a/lib/ts/ink_queue.h +++ b/lib/ts/ink_queue.h @@ -156,12 +156,12 @@ inkcoreapi extern volatile int64_t fastalloc_mem_in_use; inkcoreapi extern volatile int64_t fastalloc_mem_total; inkcoreapi extern volatile int64_t freelist_allocated_mem; -typedef struct _InkFreeList InkFreeList, *PInkFreeList; -typedef struct _ink_freelist_list { - InkFreeList *fl; - struct _ink_freelist_list *next; -} ink_freelist_list; -extern ink_freelist_list *freelists; +typedef struct ink_freelist_ops InkFreeListOps; +typedef struct _InkFreeList InkFreeList; + +const InkFreeListOps *ink_freelist_malloc_ops(); +const InkFreeListOps *ink_freelist_freelist_ops(); +void ink_freelist_init_ops(const InkFreeListOps *); /* * alignment must be a power of 2 http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e56d1f9e/proxy/Main.cc ---------------------------------------------------------------------- diff --git a/proxy/Main.cc b/proxy/Main.cc index b1e19e0..85f267e 100644 --- a/proxy/Main.cc +++ b/proxy/Main.cc @@ -164,6 +164,7 @@ HttpBodyFactory *body_factory = NULL; static int accept_mss = 0; static int cmd_line_dprintf_level = 0; // default debug output level from ink_dprintf function static int poll_timeout = -1; // No value set. +static bool cmd_disable_freelist = 0; static volatile bool sigusr1_received = false; @@ -183,6 +184,7 @@ static const ArgumentDescription argument_descriptions[] = { {"httpport", 'p', "Port descriptor for HTTP Accept", "S*", &http_accept_port_descriptor, "PROXY_HTTP_ACCEPT_PORT", NULL}, {"cluster_port", 'P', "Cluster Port Number", "I", &cluster_port_number, "PROXY_CLUSTER_PORT", NULL}, {"dprintf_level", 'o', "Debug output level", "I", &cmd_line_dprintf_level, "PROXY_DPRINTF_LEVEL", NULL}, + {"disable_freelist", 'f', "Disable the freelist memory allocator", "T", &cmd_disable_freelist, "PROXY_DPRINTF_LEVEL", NULL}, #if TS_HAS_TESTS {"regression", 'R', "Regression Level (quick:1..long:3)", "I", ®ression_level, "PROXY_REGRESSION", NULL}, @@ -214,12 +216,10 @@ class SignalContinuation : public Continuation public: char *end; char *snap; - int fastmemsnap; SignalContinuation() : Continuation(new_ProxyMutex()) { end = snap = 0; - fastmemsnap = 0; SET_HANDLER(&SignalContinuation::periodic); } @@ -241,13 +241,8 @@ public: // This is not error condition at the first place // so why stderr? // - fprintf(stderr, "sbrk 0x%" PRIu64 "x from first %" PRIu64 " from last %" PRIu64 "\n", (uint64_t)((ptrdiff_t)now), + fprintf(stderr, "sbrk 0x%" PRIu64 " from first %" PRIu64 " from last %" PRIu64 "\n", (uint64_t)((ptrdiff_t)now), (uint64_t)((ptrdiff_t)(now - end)), (uint64_t)((ptrdiff_t)(now - snap))); -#ifdef DEBUG - int fmdelta = fastmemtotal - fastmemsnap; - fprintf(stderr, "fastmem %" PRId64 " from last %" PRId64 "\n", (int64_t)fastmemtotal, (int64_t)fmdelta); - fastmemsnap += fmdelta; -#endif snap = now; } @@ -1405,6 +1400,10 @@ main(int /* argc ATS_UNUSED */, const char **argv) command_index = find_cmd_index(command_string); command_valid = command_flag && command_index >= 0; + if (cmd_disable_freelist) { + ink_freelist_init_ops(ink_freelist_malloc_ops()); + } + // Specific validity checks. if (*conf_dir && command_index != find_cmd_index(CMD_VERIFY_CONFIG)) { fprintf(stderr, "-D option can only be used with the %s command\n", CMD_VERIFY_CONFIG);
