IMPALA-5166: clean up BufferPool counters Misc changes to improve usability of the profiles.
* Separate out detailed BufferPool metrics into a "Buffer pool" sub-profile. * Only create the limit counter if there is a limit * Show BufferPool using in query MemTracker (it was accidentally disabled before because there was no query-level profile). * Reduce clutter in MemTracker dump by only showing buffer pool reservation, not usage (the usage was misleading anyway because it didn't include child usage). * Remove TotalUnpinnedBytes, which had limited value - WriteIoBytes and PeakUnpinnedBytes can answer most of the same questions - i.e. did it unpin any pages, and how many did it need to write to disk. * Add buffer pool metrics to /memz (if buffer pool is enabled) and reorder /memz so more useful information is up the top. Change-Id: I34b7f4d94c3d396ac89026c7559d6b2c6e02697c Reviewed-on: http://gerrit.cloudera.org:8080/6690 Reviewed-by: Tim Armstrong <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/f5ef7e6a Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/f5ef7e6a Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/f5ef7e6a Branch: refs/heads/master Commit: f5ef7e6ae76db02d89a8d2b0b32183716fe537c9 Parents: 0051015 Author: Tim Armstrong <[email protected]> Authored: Tue Apr 4 17:21:06 2017 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Mon May 22 10:44:04 2017 +0000 ---------------------------------------------------------------------- be/src/runtime/bufferpool/buffer-allocator.cc | 4 +- .../runtime/bufferpool/buffer-pool-counters.h | 8 +-- be/src/runtime/bufferpool/buffer-pool-test.cc | 20 ++++-- be/src/runtime/bufferpool/buffer-pool.cc | 30 ++++----- .../runtime/bufferpool/reservation-tracker.cc | 21 +++--- be/src/runtime/mem-tracker.cc | 14 ++-- be/src/util/default-path-handlers.cc | 11 +++- be/src/util/memory-metrics.cc | 4 +- be/src/util/runtime-profile.cc | 9 +++ be/src/util/runtime-profile.h | 6 ++ common/thrift/metrics.json | 2 +- www/memz.tmpl | 67 ++++++++++++++------ 12 files changed, 127 insertions(+), 69 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f5ef7e6a/be/src/runtime/bufferpool/buffer-allocator.cc ---------------------------------------------------------------------- diff --git a/be/src/runtime/bufferpool/buffer-allocator.cc b/be/src/runtime/bufferpool/buffer-allocator.cc index 77169ce..0bc4519 100644 --- a/be/src/runtime/bufferpool/buffer-allocator.cc +++ b/be/src/runtime/bufferpool/buffer-allocator.cc @@ -180,8 +180,8 @@ BufferPool::BufferAllocator::~BufferAllocator() { Status BufferPool::BufferAllocator::Allocate( ClientHandle* client, int64_t len, BufferHandle* buffer) { SCOPED_TIMER(client->impl_->counters().alloc_time); - COUNTER_ADD(client->impl_->counters().bytes_alloced, len); - COUNTER_ADD(client->impl_->counters().num_allocations, 1); + COUNTER_ADD(client->impl_->counters().cumulative_bytes_alloced, len); + COUNTER_ADD(client->impl_->counters().cumulative_allocations, 1); RETURN_IF_ERROR(AllocateInternal(len, buffer)); DCHECK(buffer->is_open()); http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f5ef7e6a/be/src/runtime/bufferpool/buffer-pool-counters.h ---------------------------------------------------------------------- diff --git a/be/src/runtime/bufferpool/buffer-pool-counters.h b/be/src/runtime/bufferpool/buffer-pool-counters.h index 58257de..f67fa99 100644 --- a/be/src/runtime/bufferpool/buffer-pool-counters.h +++ b/be/src/runtime/bufferpool/buffer-pool-counters.h @@ -29,10 +29,10 @@ struct BufferPoolClientCounters { RuntimeProfile::Counter* alloc_time; /// Number of buffers allocated via BufferAllocator::AllocateBuffer(). - RuntimeProfile::Counter* num_allocations; + RuntimeProfile::Counter* cumulative_allocations; /// Bytes of buffers allocated via BufferAllocator::AllocateBuffer(). - RuntimeProfile::Counter* bytes_alloced; + RuntimeProfile::Counter* cumulative_bytes_alloced; /// Amount of time spent waiting for reads from disk to complete. RuntimeProfile::Counter* read_wait_time; @@ -54,10 +54,6 @@ struct BufferPoolClientCounters { /// The peak total size of unpinned pages. RuntimeProfile::HighWaterMarkCounter* peak_unpinned_bytes; - - /// The total bytes of data unpinned. Every time a page's pin count goes from 1 to 0, - /// this counter is incremented by the page size. - RuntimeProfile::Counter* total_unpinned_bytes; }; } http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f5ef7e6a/be/src/runtime/bufferpool/buffer-pool-test.cc ---------------------------------------------------------------------- diff --git a/be/src/runtime/bufferpool/buffer-pool-test.cc b/be/src/runtime/bufferpool/buffer-pool-test.cc index 27a2ea9..860e45b 100644 --- a/be/src/runtime/bufferpool/buffer-pool-test.cc +++ b/be/src/runtime/bufferpool/buffer-pool-test.cc @@ -1142,17 +1142,27 @@ void BufferPoolTest::TestEvictionPolicy(int64_t page_size) { nullptr, total_mem, profile, &client)); ASSERT_TRUE(client.IncreaseReservation(total_mem)); - RuntimeProfile::Counter* bytes_alloced = - profile->GetCounter("BufferPoolAllocationBytes"); - RuntimeProfile::Counter* write_ios = profile->GetCounter("BufferPoolWriteIoOps"); - RuntimeProfile::Counter* read_ios = profile->GetCounter("BufferPoolReadIoOps"); + RuntimeProfile* buffer_pool_profile = nullptr; + vector<RuntimeProfile*> profile_children; + profile->GetChildren(&profile_children); + for (RuntimeProfile* child : profile_children) { + if (child->name() == "Buffer pool") { + buffer_pool_profile = child; + break; + } + } + ASSERT_TRUE(buffer_pool_profile != nullptr); + RuntimeProfile::Counter* cumulative_bytes_alloced = + buffer_pool_profile->GetCounter("CumulativeAllocationBytes"); + RuntimeProfile::Counter* write_ios = buffer_pool_profile->GetCounter("WriteIoOps"); + RuntimeProfile::Counter* read_ios = buffer_pool_profile->GetCounter("ReadIoOps"); vector<PageHandle> pages; CreatePages(&pool, &client, page_size, total_mem, &pages); WriteData(pages, 0); // Unpin pages. Writes should be started and memory should not be deallocated. - EXPECT_EQ(total_mem, bytes_alloced->value()); + EXPECT_EQ(total_mem, cumulative_bytes_alloced->value()); EXPECT_EQ(total_mem, SystemBytesAllocated(&pool)); UnpinAll(&pool, &client, &pages); ASSERT_GT(write_ios->value(), 0); http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f5ef7e6a/be/src/runtime/bufferpool/buffer-pool.cc ---------------------------------------------------------------------- diff --git a/be/src/runtime/bufferpool/buffer-pool.cc b/be/src/runtime/bufferpool/buffer-pool.cc index c1bde5f..6d65457 100644 --- a/be/src/runtime/bufferpool/buffer-pool.cc +++ b/be/src/runtime/bufferpool/buffer-pool.cc @@ -200,7 +200,6 @@ void BufferPool::Unpin(ClientHandle* client, PageHandle* handle) { // Data is in memory - move it to dirty unpinned. client->impl_->MoveToDirtyUnpinned(page); } - COUNTER_ADD(client->impl_->counters().total_unpinned_bytes, handle->len()); COUNTER_ADD(client->impl_->counters().peak_unpinned_bytes, handle->len()); } @@ -302,22 +301,23 @@ BufferPool::Client::Client(BufferPool* pool, TmpFileMgr::FileGroup* file_group, debug_write_delay_ms_(0), num_pages_(0), buffers_allocated_bytes_(0) { + // Set up a child profile with buffer pool info. + RuntimeProfile* child_profile = profile->CreateChild("Buffer pool", true, true); reservation_.InitChildTracker( - profile, parent_reservation, mem_tracker, reservation_limit); - counters_.alloc_time = ADD_TIMER(profile, "BufferPoolAllocTime"); - counters_.num_allocations = ADD_COUNTER(profile, "BufferPoolAllocations", TUnit::UNIT); - counters_.bytes_alloced = - ADD_COUNTER(profile, "BufferPoolAllocationBytes", TUnit::BYTES); - counters_.read_wait_time = ADD_TIMER(profile, "BufferPoolReadIoWaitTime"); - counters_.read_io_ops = ADD_COUNTER(profile, "BufferPoolReadIoOps", TUnit::UNIT); - counters_.bytes_read = ADD_COUNTER(profile, "BufferPoolReadIoBytes", TUnit::BYTES); - counters_.write_wait_time = ADD_TIMER(profile, "BufferPoolWriteIoWaitTime"); - counters_.write_io_ops = ADD_COUNTER(profile, "BufferPoolWriteIoOps", TUnit::UNIT); - counters_.bytes_written = ADD_COUNTER(profile, "BufferPoolWriteIoBytes", TUnit::BYTES); + child_profile, parent_reservation, mem_tracker, reservation_limit); + counters_.alloc_time = ADD_TIMER(profile, "AllocTime"); + counters_.cumulative_allocations = + ADD_COUNTER(child_profile, "CumulativeAllocations", TUnit::UNIT); + counters_.cumulative_bytes_alloced = + ADD_COUNTER(child_profile, "CumulativeAllocationBytes", TUnit::BYTES); + counters_.read_wait_time = ADD_TIMER(child_profile, "ReadIoWaitTime"); + counters_.read_io_ops = ADD_COUNTER(child_profile, "ReadIoOps", TUnit::UNIT); + counters_.bytes_read = ADD_COUNTER(child_profile, "ReadIoBytes", TUnit::BYTES); + counters_.write_wait_time = ADD_TIMER(child_profile, "WriteIoWaitTime"); + counters_.write_io_ops = ADD_COUNTER(child_profile, "WriteIoOps", TUnit::UNIT); + counters_.bytes_written = ADD_COUNTER(child_profile, "WriteIoBytes", TUnit::BYTES); counters_.peak_unpinned_bytes = - profile->AddHighWaterMarkCounter("BufferPoolPeakUnpinnedBytes", TUnit::BYTES); - counters_.total_unpinned_bytes = - ADD_COUNTER(profile, "BufferPoolTotalUnpinnedBytes", TUnit::BYTES); + child_profile->AddHighWaterMarkCounter("PeakUnpinnedBytes", TUnit::BYTES); } BufferPool::Page* BufferPool::Client::CreatePinnedPage(BufferHandle&& buffer) { http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f5ef7e6a/be/src/runtime/bufferpool/reservation-tracker.cc ---------------------------------------------------------------------- diff --git a/be/src/runtime/bufferpool/reservation-tracker.cc b/be/src/runtime/bufferpool/reservation-tracker.cc index 13cd67c..972d825 100644 --- a/be/src/runtime/bufferpool/reservation-tracker.cc +++ b/be/src/runtime/bufferpool/reservation-tracker.cc @@ -101,16 +101,17 @@ void ReservationTracker::InitCounters( } // Check that another tracker's counters aren't already registered in the profile. - DCHECK(profile->GetCounter("BufferPoolInitialReservation") == nullptr); - counters_.reservation_limit = - ADD_COUNTER(profile, "BufferPoolReservationLimit", TUnit::BYTES); + DCHECK(profile->GetCounter("PeakReservation") == nullptr); counters_.peak_reservation = - profile->AddHighWaterMarkCounter("BufferPoolPeakReservation", TUnit::BYTES); + profile->AddHighWaterMarkCounter("PeakReservation", TUnit::BYTES); counters_.peak_used_reservation = - profile->AddHighWaterMarkCounter("BufferPoolPeakUsedReservation", TUnit::BYTES); - - COUNTER_SET(counters_.reservation_limit, reservation_limit); - + profile->AddHighWaterMarkCounter("PeakUsedReservation", TUnit::BYTES); + // Only show the limit if set. + counters_.reservation_limit = nullptr; + if (reservation_limit != numeric_limits<int64_t>::max()) { + counters_.reservation_limit = ADD_COUNTER(profile, "ReservationLimit", TUnit::BYTES); + COUNTER_SET(counters_.reservation_limit, reservation_limit); + } if (mem_tracker_ != nullptr) mem_tracker_->EnableReservationReporting(counters_); } @@ -365,7 +366,9 @@ void ReservationTracker::CheckConsistency() const { DCHECK_LE(reservation_, counters_.peak_reservation->value()); DCHECK_EQ(used_reservation_, counters_.peak_used_reservation->current_value()); DCHECK_LE(used_reservation_, counters_.peak_used_reservation->value()); - DCHECK_EQ(reservation_limit_, counters_.reservation_limit->value()); + if (counters_.reservation_limit != nullptr) { + DCHECK_EQ(reservation_limit_, counters_.reservation_limit->value()); + } } void ReservationTracker::UpdateUsedReservation(int64_t delta) { http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f5ef7e6a/be/src/runtime/mem-tracker.cc ---------------------------------------------------------------------- diff --git a/be/src/runtime/mem-tracker.cc b/be/src/runtime/mem-tracker.cc index bd2f039..e328c01 100644 --- a/be/src/runtime/mem-tracker.cc +++ b/be/src/runtime/mem-tracker.cc @@ -225,7 +225,7 @@ void MemTracker::RegisterMetrics(MetricGroup* metrics, const string& prefix) { // DataStreamSender (dst_id=4): Total=680.00 B Peak=680.00 B // // If 'reservation_metrics_' are set, we ge a more granular breakdown: -// TrackerName: Limit=5.00 MB BufferPoolUsed/Reservation=0/5.00 MB OtherMemory=1.04 MB +// TrackerName: Limit=5.00 MB Reservation=5.00 MB OtherMemory=1.04 MB // Total=6.04 MB Peak=6.45 MB // string MemTracker::LogUsage(const string& prefix, int64_t* logged_consumption) const { @@ -243,14 +243,10 @@ string MemTracker::LogUsage(const string& prefix, int64_t* logged_consumption) c ReservationTrackerCounters* reservation_counters = reservation_counters_.Load(); if (reservation_counters != nullptr) { int64_t reservation = reservation_counters->peak_reservation->current_value(); - int64_t used_reservation = - reservation_counters->peak_used_reservation->current_value(); - int64_t reservation_limit = reservation_counters->reservation_limit->value(); - ss << " BufferPoolUsed/Reservation=" - << PrettyPrinter::Print(used_reservation, TUnit::BYTES) << "/" - << PrettyPrinter::Print(reservation, TUnit::BYTES); - if (reservation_limit != numeric_limits<int64_t>::max()) { - ss << " BufferPoolLimit=" << PrettyPrinter::Print(reservation_limit, TUnit::BYTES); + ss << " Reservation=" << PrettyPrinter::Print(reservation, TUnit::BYTES); + if (reservation_counters->reservation_limit != nullptr) { + int64_t limit = reservation_counters->reservation_limit->value(); + ss << " ReservationLimit=" << PrettyPrinter::Print(limit, TUnit::BYTES); } ss << " OtherMemory=" << PrettyPrinter::Print(curr_consumption - reservation, TUnit::BYTES); http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f5ef7e6a/be/src/util/default-path-handlers.cc ---------------------------------------------------------------------- diff --git a/be/src/util/default-path-handlers.cc b/be/src/util/default-path-handlers.cc index 732d3b3..ef17a1f 100644 --- a/be/src/util/default-path-handlers.cc +++ b/be/src/util/default-path-handlers.cc @@ -143,9 +143,9 @@ void MemUsageHandler(MemTracker* mem_tracker, MetricGroup* metric_group, Value detailed(mem_tracker->LogUsage().c_str(), document->GetAllocator()); document->AddMember("detailed", detailed, document->GetAllocator()); - if (metric_group != NULL) { + if (metric_group != nullptr) { MetricGroup* jvm_group = metric_group->FindChildGroup("jvm"); - if (jvm_group != NULL) { + if (jvm_group != nullptr) { Value jvm(kObjectType); jvm_group->ToJson(false, document, &jvm); Value heap(kArrayType); @@ -164,6 +164,13 @@ void MemUsageHandler(MemTracker* mem_tracker, MetricGroup* metric_group, document->AddMember("jvm_heap", heap, document->GetAllocator()); document->AddMember("jvm_non_heap", non_heap, document->GetAllocator()); } + MetricGroup* buffer_pool_group = metric_group->FindChildGroup("buffer-pool"); + if (buffer_pool_group != nullptr) { + Value json_metrics(kObjectType); + buffer_pool_group->ToJson(false, document, &json_metrics); + document->AddMember( + "buffer_pool", json_metrics["metrics"], document->GetAllocator()); + } } } http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f5ef7e6a/be/src/util/memory-metrics.cc ---------------------------------------------------------------------- diff --git a/be/src/util/memory-metrics.cc b/be/src/util/memory-metrics.cc index 5f40a69..9336cf3 100644 --- a/be/src/util/memory-metrics.cc +++ b/be/src/util/memory-metrics.cc @@ -52,8 +52,8 @@ Status impala::RegisterMemoryMetrics(MetricGroup* metrics, bool register_jvm_met ReservationTracker* global_reservations, BufferPool* buffer_pool) { if (global_reservations != nullptr) { DCHECK(buffer_pool != nullptr); - RETURN_IF_ERROR( - BufferPoolMetric::InitMetrics(metrics, global_reservations, buffer_pool)); + RETURN_IF_ERROR(BufferPoolMetric::InitMetrics( + metrics->GetOrCreateChildGroup("buffer-pool"), global_reservations, buffer_pool)); } #ifndef ADDRESS_SANITIZER // We rely on TCMalloc for our global memory metrics, so skip setting them up http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f5ef7e6a/be/src/util/runtime-profile.cc ---------------------------------------------------------------------- diff --git a/be/src/util/runtime-profile.cc b/be/src/util/runtime-profile.cc index 19044fa..9efe556 100644 --- a/be/src/util/runtime-profile.cc +++ b/be/src/util/runtime-profile.cc @@ -433,6 +433,15 @@ void RuntimeProfile::PrependChild(RuntimeProfile* child, bool indent) { AddChildLocked(child, indent, children_.begin()); } +RuntimeProfile* RuntimeProfile::CreateChild(const string& name, bool indent, + bool prepend) { + lock_guard<SpinLock> l(children_lock_); + DCHECK(child_map_.find(name) == child_map_.end()); + RuntimeProfile* child = pool_->Add(new RuntimeProfile(pool_, name)); + AddChildLocked(child, indent, prepend ? children_.begin() : children_.end()); + return child; +} + void RuntimeProfile::GetChildren(vector<RuntimeProfile*>* children) { children->clear(); lock_guard<SpinLock> l(children_lock_); http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f5ef7e6a/be/src/util/runtime-profile.h ---------------------------------------------------------------------- diff --git a/be/src/util/runtime-profile.h b/be/src/util/runtime-profile.h index a514f8b..244ab17 100644 --- a/be/src/util/runtime-profile.h +++ b/be/src/util/runtime-profile.h @@ -130,6 +130,12 @@ class RuntimeProfile { // NOLINT: This struct is not packed, but there are not s /// existing profiles. void PrependChild(RuntimeProfile* child, bool indent = true); + /// Creates a new child profile with the given 'name'. A child profile with that name + /// must not already exist. If 'prepend' is true, prepended before other child profiles, + /// otherwise appended after other child profiles. + RuntimeProfile* CreateChild( + const std::string& name, bool indent = true, bool prepend = false); + /// Sorts all children according to a custom comparator. Does not /// invalidate pointers to profiles. template <class Compare> http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f5ef7e6a/common/thrift/metrics.json ---------------------------------------------------------------------- diff --git a/common/thrift/metrics.json b/common/thrift/metrics.json index cef9b02..a3fbd7b 100644 --- a/common/thrift/metrics.json +++ b/common/thrift/metrics.json @@ -1132,7 +1132,7 @@ "key": "buffer-pool.system-allocated" }, { - "description": "Total memory currently reserved for buffers.", + "description": "Total bytes of buffers reserved by Impala subsystems", "contexts": [ "IMPALAD" ], http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f5ef7e6a/www/memz.tmpl ---------------------------------------------------------------------- diff --git a/www/memz.tmpl b/www/memz.tmpl index 48bd651..2c221af 100644 --- a/www/memz.tmpl +++ b/www/memz.tmpl @@ -22,21 +22,21 @@ under the License. Memory consumption / limit: <strong>{{consumption}}</strong> / <strong>{{mem_limit}}</strong> -<h3>tcmalloc</h3> -<pre>{{overview}}</pre> - <h3>Breakdown</h3> <pre>{{detailed}}</pre> -{{?jvm_heap}} -<h3>JVM heap usage</h3> +<h3>tcmalloc</h3> +<pre>{{overview}}</pre> + +{{?buffer_pool}} +<h3>Buffer pool memory metrics</h3> <table class='table table-bordered table-hover'> <tr> <th>Name</th> <th>Value</th> <th>Description</th> </tr> - {{#jvm_heap}} + {{#buffer_pool}} <tr> <td><tt>{{name}}</tt></td> {{! Is this a stats metric? }} @@ -54,19 +54,19 @@ Memory consumption / limit: <strong>{{consumption}}</strong> / <strong>{{mem_lim {{description}} </td> </tr> - {{/jvm_heap}} + {{/buffer_pool}} </table> -{{/jvm_heap}} +{{/buffer_pool}} -{{?jvm_non_heap}} -<h3>JVM non-heap usage</h3> +{{?jvm_total}} +<h3>JVM aggregate memory metrics</h3> <table class='table table-bordered table-hover'> <tr> <th>Name</th> <th>Value</th> <th>Description</th> </tr> - {{#jvm_non_heap}} + {{#jvm_total}} <tr> <td><tt>{{name}}</tt></td> {{! Is this a stats metric? }} @@ -84,19 +84,20 @@ Memory consumption / limit: <strong>{{consumption}}</strong> / <strong>{{mem_lim {{description}} </td> </tr> - {{/jvm_non_heap}} + {{/jvm_total}} </table> -{{/jvm_non_heap}} +{{/jvm_total}} -{{?jvm_total}} -<h3>JVM total memory usage</h3> + +{{?jvm_heap}} +<h3>JVM heap memory metrics</h3> <table class='table table-bordered table-hover'> <tr> <th>Name</th> <th>Value</th> <th>Description</th> </tr> - {{#jvm_total}} + {{#jvm_heap}} <tr> <td><tt>{{name}}</tt></td> {{! Is this a stats metric? }} @@ -114,8 +115,38 @@ Memory consumption / limit: <strong>{{consumption}}</strong> / <strong>{{mem_lim {{description}} </td> </tr> - {{/jvm_total}} + {{/jvm_heap}} </table> -{{/jvm_total}} +{{/jvm_heap}} + +{{?jvm_non_heap}} +<h3>JVM non-heap memory metrics</h3> +<table class='table table-bordered table-hover' + <tr> + <th>Name</th> + <th>Value</th> + <th>Description</th> + </tr> + {{#jvm_non_heap}} + <tr> + <td><tt>{{name}}</tt></td> + {{! Is this a stats metric? }} + {{?mean}} + <td> + Last (of {{count}}): <strong>{{last}}</strong>. + Min: {{min}}, max: {{max}}, avg: {{mean}}</td> + {{/mean}} + {{^mean}} + <td> + {{human_readable}} + </td> + {{/mean}} + <td> + {{description}} + </td> + </tr> + {{/jvm_non_heap}} +</table> +{{/jvm_non_heap}} {{> www/common-footer.tmpl }}
