[
https://issues.apache.org/jira/browse/KUDU-1762?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15750277#comment-15750277
]
DawnZhang commented on KUDU-1762:
---------------------------------
/varz page:
Command-line Flags
--block_cache_capacity_mb=512
--block_cache_type=DRAM
--cfile_lazy_open=true
--cfile_default_block_size=262144
--cfile_default_compression_codec=none
--cfile_do_on_finish=flush
--codegen_dump_functions=false
--codegen_dump_mc=false
--codegen_cache_capacity=100
--codegen_time_compilation=false
--materializing_iterator_decoder_eval=true
--materializing_iterator_do_pushdown=true
--consensus_rpc_timeout_ms=1000
--enable_tablet_copy=true
--fault_crash_after_leader_request_fraction=0
--fault_crash_on_leader_request_fraction=0
--raft_get_node_instance_timeout_ms=30000
--consensus_inject_latency_ms_in_notifications=0
--consensus_max_batch_size_bytes=1048576
--follower_unavailable_considered_failed_sec=300
--fault_crash_before_append_commit=0
--fs_wal_dir_reserved_bytes=0
--group_commit_queue_size_bytes=4194304
--log_inject_io_error_on_append_fraction=0
--log_inject_io_error_on_preallocate_fraction=0
--log_inject_latency=false
--log_inject_latency_ms_mean=100
--log_inject_latency_ms_stddev=100
--log_max_segments_to_retain=10
--log_min_segments_to_retain=2
--global_log_cache_size_limit_mb=1024
--log_cache_size_limit_mb=128
--log_async_preallocate_segments=true
--log_force_fsync_all=false
--log_preallocate_segments=true
--log_segment_size_mb=64
--enable_leader_failure_detection=true
--evict_failed_followers=true
--follower_fail_all_prepare=false
--follower_reject_update_consensus_requests=false
--leader_failure_exp_backoff_max_delta_ms=20000
--leader_failure_max_missed_heartbeat_periods=3
--leader_failure_monitor_check_mean_ms=-1
--leader_failure_monitor_check_stddev_ms=-1
--raft_heartbeat_interval_ms=500
--block_coalesce_close=false
--block_manager_lock_dirs=true
--block_manager=log
--enable_data_block_fsync=true
--fs_data_dirs=/data/kudu/tablet/data
--fs_wal_dir=/data/kudu/tablet/wal
--fs_data_dirs_reserved_bytes=0
--log_block_manager_full_disk_cache_seconds=30
--log_block_manager_test_hole_punching=true
--log_container_max_size=10737418240
--log_container_preallocate_bytes=33554432
--rpc_acceptor_listen_backlog=128
--rpc_default_keepalive_time_ms=65000
--rpc_negotiation_inject_delay_ms=0
--rpc_trace_negotiation=false
--rpc_callback_max_cycles=100000000
--rpc_negotiation_timeout_ms=3000
--remember_clients_ttl_ms=3600000
--remember_responses_ttl_ms=600000
--result_tracker_gc_interval_ms=1000
--rpc_dump_all_traces=false
--enable_exactly_once=true
--rpc_max_message_size=52428800
--web_log_bytes=1048576
--max_clock_sync_error_usec=10000000
--use_hybrid_clock=false
--use_mock_wall_clock=false
--rpc_bind_addresses=0.0.0.0:7050
--rpc_num_acceptors_per_address=1
--rpc_num_service_threads=20
--rpc_server_allow_ephemeral_ports=false
--rpc_service_queue_length=50
--num_reactor_threads=4
--metrics_log_interval_ms=0
--server_dump_info_format=json
--server_dump_info_path=
--webserver_max_post_length_bytes=1048576
--webserver_authentication_domain=
--webserver_certificate_file=
--webserver_doc_root=/data/cloudera/parcels/KUDU-1.0.1-1.kudu1.0.1.p0.14/lib/kudu/www
--webserver_enable_doc_root=true
--webserver_interface=
--webserver_num_worker_threads=50
--webserver_password_file=
--webserver_port=8050
--consult_bloom_filters=true
--budgeted_compaction_target_rowset_size=33554432
--compaction_approximation_ratio=1.0499999523162842
--deltafile_default_block_size=32768
--default_composite_key_index_block_size_bytes=4096
--tablet_delta_store_major_compact_min_ratio=0.10000000149011612
--tablet_delta_store_minor_compact_max=1000
--mrs_use_codegen=true
--compaction_policy_dump_svgs_pattern=
--fault_crash_before_flush_tablet_meta_after_compaction=0
--fault_crash_before_flush_tablet_meta_after_flush_mrs=0
--tablet_bloom_block_size=4096
--tablet_bloom_target_fp_rate=9.9999997473787516e-05
--tablet_compaction_budget_mb=128
--tablet_history_max_age_sec=900
--tablet_throttler_burst_factor=1
--tablet_throttler_bytes_per_sec=0
--tablet_throttler_rpc_per_sec=0
--fault_crash_during_log_replay=0
--skip_remove_old_recovery_dir=false
--enable_tablet_orphaned_block_deletion=true
--flush_threshold_mb=1024
--tablet_transaction_memory_limit_mb=64
--tablet_inject_latency_on_apply_write_txn_ms=0
--heartbeat_interval_ms=1000
--heartbeat_max_failures_before_backoff=3
--heartbeat_rpc_timeout_ms=15000
--scanner_gc_check_interval_us=5000000
--scanner_ttl_ms=60000
--tablet_copy_begin_session_timeout_ms=3000
--tablet_copy_dowload_file_inject_latency_ms=0
--tablet_copy_save_downloaded_metadata=false
--fault_crash_on_handle_tc_fetch_data=0
--tablet_copy_idle_timeout_ms=180000
--tablet_copy_timeout_poll_period_ms=10000
--tablet_copy_transfer_chunk_size_bytes=4194304
--tserver_master_addrs=data02.ymtcluster.sa
--scanner_batch_size_rows=100
--scanner_default_batch_size_bytes=1048576
--scanner_inject_latency_on_each_batch_ms=0
--scanner_max_batch_size_bytes=8388608
--fault_crash_after_blocks_deleted=0
--fault_crash_after_cmeta_deleted=0
--fault_crash_after_tc_files_fetched=0
--fault_crash_after_wal_deleted=0
--num_tablets_to_open_simultaneously=0
--tablet_start_warn_threshold_ms=500
--trace_to_console=
--never_fsync=false
--suicide_on_eio=true
--writable_file_use_fsync=false
--disk_reserved_bytes_free_for_testing=-1
--disk_reserved_override_prefix_1_bytes_free_for_testing=-1
--disk_reserved_override_prefix_1_path_for_testing=
--disk_reserved_override_prefix_2_bytes_free_for_testing=-1
--disk_reserved_override_prefix_2_path_for_testing=
--disable_core_dumps=false
--dump_metrics_json=false
--enable_process_lifetime_heap_profiling=true
--heap_profile_path=/data/kudu_debug/pf
--unlock_experimental_flags=false
--unlock_unsafe_flags=false
--hung_task_check_interval_ms=200
--log_filename=kudu-tserver
--enable_maintenance_manager=true
--maintenance_manager_history_size=8
--maintenance_manager_num_threads=1
--maintenance_manager_polling_interval_ms=250
--memory_limit_hard_bytes=4294967296
--memory_limit_soft_percentage=60
--memory_limit_warn_threshold_percentage=98
--tcmalloc_max_free_bytes_percentage=10
--arena_warn_threshold_bytes=268435456
--allocator_aligned_mode=false
--metrics_retirement_age_ms=120000
--fail_dns_resolution=false
--local_ip_for_outbound_sockets=
--socket_inject_short_recvs=false
--nvm_cache_allocation_retry_count=10
--nvm_cache_path=/vmem
--nvm_cache_simulate_allocation_failure=false
--lock_contention_trace_threshold_cycles=2000000
--flagfile=/var/run/cloudera-scm-agent/process/362-kudu-KUDU_TSERVER/gflagfile
--fromenv=
--tryfromenv=
--undefok=
--tab_completion_columns=80
--tab_completion_word=
--help=false
--helpfull=false
--helpmatch=
--helpon=
--helppackage=false
--helpshort=false
--helpxml=false
--version=false
--alsologtoemail=
--alsologtostderr=false
--colorlogtostderr=false
--drop_log_memory=true
--log_backtrace_at=
--log_dir=/data/cloudera/log/kudu/tserver
--log_link=
--log_prefix=true
--logbuflevel=0
--logbufsecs=30
--logemaillevel=999
--logmailer=/bin/mail
--logtostderr=false
--max_log_size=10
--minloglevel=0
--stderrthreshold=2
--stop_logging_if_full_disk=false
--symbolize_stacktrace=true
--v=0
--vmodule=
kudu 1.0.1 (rev e60b610253f4303b24d41575f7bafbc5d69edddb)
server uuid 0c79993cd5504785a68f07c52463a4dc
> suspected tablet memory leak
> ----------------------------
>
> Key: KUDU-1762
> URL: https://issues.apache.org/jira/browse/KUDU-1762
> Project: Kudu
> Issue Type: Bug
> Components: tablet
> Affects Versions: 1.0.1
> Environment: CentOS 6.5
> Kudu 1.0.1 (rev e60b610253f4303b24d41575f7bafbc5d69edddb)
> Reporter: Fu Lili
> Priority: Critical
> Attachments: 0B2CE7BB-EF26-4EA1-B824-3584D7D79256.png,
> kudu_heap_prof_20161206.tar.gz, tserver_smaps1
>
>
> here is the memory total info:
> {quote}
> ------------------------------------------------
> MALLOC: 1691715680 ( 1613.3 MiB) Bytes in use by application
> MALLOC: + 178733056 ( 170.5 MiB) Bytes in page heap freelist
> MALLOC: + 37483104 ( 35.7 MiB) Bytes in central cache freelist
> MALLOC: + 4071488 ( 3.9 MiB) Bytes in transfer cache freelist
> MALLOC: + 13739264 ( 13.1 MiB) Bytes in thread cache freelists
> MALLOC: + 12202144 ( 11.6 MiB) Bytes in malloc metadata
> MALLOC: ------------
> MALLOC: = 1937944736 ( 1848.2 MiB) Actual memory used (physical + swap)
> MALLOC: + 311296 ( 0.3 MiB) Bytes released to OS (aka unmapped)
> MALLOC: ------------
> MALLOC: = 1938256032 ( 1848.5 MiB) Virtual address space used
> MALLOC:
> MALLOC: 174694 Spans in use
> MALLOC: 201 Thread heaps in use
> MALLOC: 8192 Tcmalloc page size
> ------------------------------------------------
> Call ReleaseFreeMemory() to release freelist memory to the OS (via madvise()).
> Bytes released to the OS take up virtual address space but no physical memory.
> {quote}
> but in memroy detail, sum of all the sub Current Consumption is far less than
> the to the root Current Consumption。
> ||Id||Parent||Limit||Current Consumption||Peak consumption||
> |root|none|4.00G|1.58G|1.74G|
> |log_cache|root|1.00G|480.8K|5.32M|
> |log_cache:0c79993cd5504785a68f07c52463a4dc:70c8d889b0314b04a240fcb02c24a012|log_cache|128.00M|160B|160B|
> |log_cache:0c79993cd5504785a68f07c52463a4dc:16d3c8193579445f8f766da6c7abc237|log_cache|128.00M|160B|160B|
> |log_cache:0c79993cd5504785a68f07c52463a4dc:2c69c5cb9eb04eb48323a9268afc36a7|log_cache|128.00M|160B|160B|
> |log_cache:0c79993cd5504785a68f07c52463a4dc:2b11d9220dab4a5f952c5b1c10a68ccd|log_cache|128.00M|69.2K|139.5K|
> |log_cache:0c79993cd5504785a68f07c52463a4dc:cec045be60af4f759497234d8815238b|log_cache|128.00M|68.6K|138.7K|
> |log_cache:0c79993cd5504785a68f07c52463a4dc:cea7a54cebd242e4997da641f5b32e3a|log_cache|128.00M|68.5K|139.3K|
> |log_cache:0c79993cd5504785a68f07c52463a4dc:9625dfde17774690a888b55024ac797a|log_cache|128.00M|68.5K|140.0K|
> |log_cache:0c79993cd5504785a68f07c52463a4dc:6046b33901ca43d0975f59cf7e491186|log_cache|128.00M|0B|133.0K|
> |log_cache:0c79993cd5504785a68f07c52463a4dc:1a18ab0915f0407b922fa7ecbe7a2f46|log_cache|128.00M|0B|132.6K|
> |log_cache:0c79993cd5504785a68f07c52463a4dc:ac54d1c1813a4e39943971cb56f248ef|log_cache|128.00M|0B|130.5K|
> |log_cache:0c79993cd5504785a68f07c52463a4dc:4438580df6cc4d469393b9d6adee68d8|log_cache|128.00M|0B|131.2K|
> |log_cache:0c79993cd5504785a68f07c52463a4dc:2f1cef7d2a494575b941baa22b8a3dc9|log_cache|128.00M|0B|131.6K|
> |log_cache:0c79993cd5504785a68f07c52463a4dc:d2ad22d202c04b2d98f1c5800df1c3b5|log_cache|128.00M|0B|132.5K|
> |log_cache:0c79993cd5504785a68f07c52463a4dc:b19b21d6b4c84f9895aad9e81559d019|log_cache|128.00M|0B|131.0K|
> |log_cache:0c79993cd5504785a68f07c52463a4dc:27e9531cd5814b1c9637493f05860b19|log_cache|128.00M|0B|131.1K|
> |log_cache:0c79993cd5504785a68f07c52463a4dc:425a19940239447faa0eaab4e380d644|log_cache|128.00M|68.5K|146.9K|
> |log_cache:0c79993cd5504785a68f07c52463a4dc:178bd7bc39a941a887f393b0a7848066|log_cache|128.00M|68.5K|139.9K|
> |log_cache:0c79993cd5504785a68f07c52463a4dc:91524acd28a440318918f11292ac8fdc|log_cache|128.00M|0B|132.0K|
> |log_cache:0c79993cd5504785a68f07c52463a4dc:be6f093aabf9460b97fc35dd026820b6|log_cache|128.00M|0B|130.4K|
> |log_cache:0c79993cd5504785a68f07c52463a4dc:dd8dd794f0f44426a3c46ce8f4b54652|log_cache|128.00M|0B|131.2K|
> |log_cache:0c79993cd5504785a68f07c52463a4dc:ed128ca7b19c4e3eaa48e9e3eb341492|log_cache|128.00M|68.5K|141.5K|
> |block_cache-sharded_lru_cache|root|none|257.05M|257.05M|
> |code_cache-sharded_lru_cache|root|none|112B|113B|
> |server|root|none|2.06M|121.97M|
> |tablet-70c8d889b0314b04a240fcb02c24a012|server|none|265B|265B|
> |txn_tracker|tablet-70c8d889b0314b04a240fcb02c24a012|64.00M|0B|0B|
> |MemRowSet-0|tablet-70c8d889b0314b04a240fcb02c24a012|none|265B|265B|
> |DeltaMemStores|tablet-70c8d889b0314b04a240fcb02c24a012|none|0B|0B|
> |tablet-16d3c8193579445f8f766da6c7abc237|server|none|265B|265B|
> |txn_tracker|tablet-16d3c8193579445f8f766da6c7abc237|64.00M|0B|0B|
> |MemRowSet-0|tablet-16d3c8193579445f8f766da6c7abc237|none|265B|265B|
> |DeltaMemStores|tablet-16d3c8193579445f8f766da6c7abc237|none|0B|0B|
> |tablet-2c69c5cb9eb04eb48323a9268afc36a7|server|none|265B|265B|
> |txn_tracker|tablet-2c69c5cb9eb04eb48323a9268afc36a7|64.00M|0B|0B|
> |MemRowSet-0|tablet-2c69c5cb9eb04eb48323a9268afc36a7|none|265B|265B|
> |DeltaMemStores|tablet-2c69c5cb9eb04eb48323a9268afc36a7|none|0B|0B|
> |tablet-2b11d9220dab4a5f952c5b1c10a68ccd|server|none|25.7K|193.7K|
> |MemRowSet-5|tablet-2b11d9220dab4a5f952c5b1c10a68ccd|none|25.4K|25.4K|
> |txn_tracker|tablet-2b11d9220dab4a5f952c5b1c10a68ccd|64.00M|0B|70.2K|
> |DeltaMemStores|tablet-2b11d9220dab4a5f952c5b1c10a68ccd|none|265B|1.0K|
> |tablet-cec045be60af4f759497234d8815238b|server|none|58.6K|192.9K|
> |MemRowSet-5|tablet-cec045be60af4f759497234d8815238b|none|58.3K|58.3K|
> |txn_tracker|tablet-cec045be60af4f759497234d8815238b|64.00M|0B|70.1K|
> |DeltaMemStores|tablet-cec045be60af4f759497234d8815238b|none|265B|1.0K|
> |tablet-cea7a54cebd242e4997da641f5b32e3a|server|none|124.4K|193.5K|
> |MemRowSet-5|tablet-cea7a54cebd242e4997da641f5b32e3a|none|124.1K|124.1K|
> |txn_tracker|tablet-cea7a54cebd242e4997da641f5b32e3a|64.00M|0B|70.0K|
> |DeltaMemStores|tablet-cea7a54cebd242e4997da641f5b32e3a|none|265B|795B|
> |tablet-9625dfde17774690a888b55024ac797a|server|none|530B|326.6K|
> |MemRowSet-22|tablet-9625dfde17774690a888b55024ac797a|none|265B|265B|
> |txn_tracker|tablet-9625dfde17774690a888b55024ac797a|64.00M|0B|71.3K|
> |DeltaMemStores|tablet-9625dfde17774690a888b55024ac797a|none|265B|1.3K|
> |tablet-6046b33901ca43d0975f59cf7e491186|server|none|530B|587.7K|
> |MemRowSet-22|tablet-6046b33901ca43d0975f59cf7e491186|none|265B|265B|
> |txn_tracker|tablet-6046b33901ca43d0975f59cf7e491186|64.00M|0B|139.3K|
> |DeltaMemStores|tablet-6046b33901ca43d0975f59cf7e491186|none|265B|1.0K|
> |tablet-1a18ab0915f0407b922fa7ecbe7a2f46|server|none|530B|383.4K|
> |MemRowSet-22|tablet-1a18ab0915f0407b922fa7ecbe7a2f46|none|265B|265B|
> |txn_tracker|tablet-1a18ab0915f0407b922fa7ecbe7a2f46|64.00M|0B|70.5K|
> |DeltaMemStores|tablet-1a18ab0915f0407b922fa7ecbe7a2f46|none|265B|1.0K|
> |tablet-ac54d1c1813a4e39943971cb56f248ef|server|none|530B|324.5K|
> |MemRowSet-11|tablet-ac54d1c1813a4e39943971cb56f248ef|none|265B|265B|
> |txn_tracker|tablet-ac54d1c1813a4e39943971cb56f248ef|64.00M|0B|69.7K|
> |DeltaMemStores|tablet-ac54d1c1813a4e39943971cb56f248ef|none|265B|1.0K|
> |tablet-4438580df6cc4d469393b9d6adee68d8|server|none|530B|325.4K|
> |MemRowSet-11|tablet-4438580df6cc4d469393b9d6adee68d8|none|265B|265B|
> |txn_tracker|tablet-4438580df6cc4d469393b9d6adee68d8|64.00M|0B|69.9K|
> |DeltaMemStores|tablet-4438580df6cc4d469393b9d6adee68d8|none|265B|1.0K|
> |tablet-2f1cef7d2a494575b941baa22b8a3dc9|server|none|530B|325.3K|
> |MemRowSet-11|tablet-2f1cef7d2a494575b941baa22b8a3dc9|none|265B|265B|
> |txn_tracker|tablet-2f1cef7d2a494575b941baa22b8a3dc9|64.00M|0B|70.3K|
> |DeltaMemStores|tablet-2f1cef7d2a494575b941baa22b8a3dc9|none|265B|1.0K|
> |tablet-d2ad22d202c04b2d98f1c5800df1c3b5|server|none|530B|326.0K|
> |MemRowSet-22|tablet-d2ad22d202c04b2d98f1c5800df1c3b5|none|265B|265B|
> |txn_tracker|tablet-d2ad22d202c04b2d98f1c5800df1c3b5|64.00M|0B|136.7K|
> |DeltaMemStores|tablet-d2ad22d202c04b2d98f1c5800df1c3b5|none|265B|1.0K|
> |tablet-b19b21d6b4c84f9895aad9e81559d019|server|none|530B|326.3K|
> |MemRowSet-22|tablet-b19b21d6b4c84f9895aad9e81559d019|none|265B|265B|
> |txn_tracker|tablet-b19b21d6b4c84f9895aad9e81559d019|64.00M|0B|70.0K|
> |DeltaMemStores|tablet-b19b21d6b4c84f9895aad9e81559d019|none|265B|1.0K|
> |tablet-27e9531cd5814b1c9637493f05860b19|server|none|530B|327.8K|
> |MemRowSet-22|tablet-27e9531cd5814b1c9637493f05860b19|none|265B|265B|
> |txn_tracker|tablet-27e9531cd5814b1c9637493f05860b19|64.00M|0B|71.8K|
> |DeltaMemStores|tablet-27e9531cd5814b1c9637493f05860b19|none|265B|1.3K|
> |tablet-425a19940239447faa0eaab4e380d644|server|none|795B|332.9K|
> |MemRowSet-11|tablet-425a19940239447faa0eaab4e380d644|none|265B|265B|
> |txn_tracker|tablet-425a19940239447faa0eaab4e380d644|64.00M|0B|76.8K|
> |DeltaMemStores|tablet-425a19940239447faa0eaab4e380d644|none|530B|1.0K|
> |tablet-178bd7bc39a941a887f393b0a7848066|server|none|530B|325.8K|
> |MemRowSet-10|tablet-178bd7bc39a941a887f393b0a7848066|none|265B|265B|
> |txn_tracker|tablet-178bd7bc39a941a887f393b0a7848066|64.00M|0B|70.4K|
> |DeltaMemStores|tablet-178bd7bc39a941a887f393b0a7848066|none|265B|1.0K|
> |tablet-91524acd28a440318918f11292ac8fdc|server|none|530B|326.4K|
> |MemRowSet-11|tablet-91524acd28a440318918f11292ac8fdc|none|265B|265B|
> |txn_tracker|tablet-91524acd28a440318918f11292ac8fdc|64.00M|0B|72.0K|
> |DeltaMemStores|tablet-91524acd28a440318918f11292ac8fdc|none|265B|1.0K|
> |tablet-be6f093aabf9460b97fc35dd026820b6|server|none|530B|588.6K|
> |MemRowSet-22|tablet-be6f093aabf9460b97fc35dd026820b6|none|265B|265B|
> |txn_tracker|tablet-be6f093aabf9460b97fc35dd026820b6|64.00M|0B|72.3K|
> |DeltaMemStores|tablet-be6f093aabf9460b97fc35dd026820b6|none|265B|1.0K|
> |tablet-dd8dd794f0f44426a3c46ce8f4b54652|server|none|530B|325.7K|
> |MemRowSet-22|tablet-dd8dd794f0f44426a3c46ce8f4b54652|none|265B|265B|
> |txn_tracker|tablet-dd8dd794f0f44426a3c46ce8f4b54652|64.00M|0B|72.4K|
> |DeltaMemStores|tablet-dd8dd794f0f44426a3c46ce8f4b54652|none|265B|795B|
> |tablet-ed128ca7b19c4e3eaa48e9e3eb341492|server|none|530B|325.6K|
> |MemRowSet-22|tablet-ed128ca7b19c4e3eaa48e9e3eb341492|none|265B|265B|
> |txn_tracker|tablet-ed128ca7b19c4e3eaa48e9e3eb341492|64.00M|0B|71.7K|
> |DeltaMemStores|tablet-ed128ca7b19c4e3eaa48e9e3eb341492|none|265B|1.0K|
> |log_block_manager|server|none|1.84M|6.19M|
> |result-tracker|server|none|0B|0B
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
