ZK server version: zk_version 3.8.4-9316c2a7a97e1666d8f4593f34dd6fc36ecc436c, built on 2024-02-12 22:16 UTC
Please find the config/stats from all the three nodes: Leader: CONFIG: # TEMPLATE_VERSION:3.5.8 # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # the port at which the clients will connect clientPort=2181 # the directory where the snapshot is stored. dataDir=/data/d98d54dd-dd9f-49cf-85e4-ddd922a635d8 # disable the per-ip limit on the number of connections since this is a non-production config maxClientCnxns=0 # Standalone mode or a Distributed mode standaloneEnabled=false tickTime=2000 initLimit=300 syncLimit=10 #SASL configuration quorum.auth.enableSasl=true quorum.auth.learnerRequireSasl=true quorum.auth.serverRequireSasl=true 4lw.commands.whitelist=stat, ruok, mntr metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider metricsProvider.httpPort=4888 metricsProvider.exportJvmInfo=true server.0=169.254.1.4:2888:3888 server.1=169.254.2.4:2888:3888 server.2=169.254.3.4:2888:3888 nms.instance.id=0 # Auto purge logs and snapshots. Keep latest 20 snapshots and logs. Run purge per 3 hours. autopurge.snapRetainCount=20 autopurge.purgeInterval=3 # It is the maximum value that client may request and the server will accept. # NOTE: high maxSessionTimeout will cause ephemeral node could not be deleted correctly. maxSessionTimeout=180000 # To avoid seeks ZooKeeper allocates space in the transaction log file in # blocks of preAllocSize kilobytes. The default block size is 64M. One reason # for changing the size of the blocks is to reduce the block size if snapshots # are taken more often. (Also, see snapCount). preAllocSize=131072 # ZooKeeper logs transactions to a transaction log. After snapCount transactions # are written to a log file a snapshot is started and a new transaction log file # is started. The default snapCount is 100000. snapCount=3000000 leaderServes=yes MNTR: zk_version 3.8.4-9316c2a7a97e1666d8f4593f34dd6fc36ecc436c, built on 2024-02-12 22:16 UTC zk_server_state leader zk_peer_state leading - broadcast zk_pending_session_queue_size{quantile="0.5"} 1.0 zk_pending_session_queue_size_count 1.6421529E8 zk_pending_session_queue_size_sum 2.73808658E8 zk_watch_bytes 8.01944662E8 zk_read_per_namespace{key="zookeeper",quantile="0.5"} NaN zk_read_per_namespace_count{key="zookeeper"} 21.0 zk_read_per_namespace_sum{key="zookeeper"} 4683.0 zk_read_per_namespace{key="interprocess_lockvedge_serial_list_handler",quantile="0.5"} NaN zk_read_per_namespace_count{key="interprocess_lockvedge_serial_list_handler"} 20.0 zk_read_per_namespace_sum{key="interprocess_lockvedge_serial_list_handler"} 3212.0 zk_read_per_namespace{key="resource",quantile="0.5"} NaN zk_read_per_namespace_count{key="resource"} 10.0 zk_read_per_namespace_sum{key="resource"} 1246.0 zk_read_per_namespace{key="bucket_9",quantile="0.5"} 242.0 zk_read_per_namespace_count{key="bucket_9"} 3637420.0 zk_read_per_namespace_sum{key="bucket_9"} 7.45108299E8 zk_read_per_namespace{key="cluster-oracle",quantile="0.5"} NaN zk_read_per_namespace_count{key="cluster-oracle"} 244.0 zk_read_per_namespace_sum{key="cluster-oracle"} 114093.0 zk_read_per_namespace{key="bucket_8",quantile="0.5"} 242.0 zk_read_per_namespace_count{key="bucket_8"} 4016328.0 zk_read_per_namespace_sum{key="bucket_8"} 8.22947049E8 zk_read_per_namespace{key="bucket_7",quantile="0.5"} 242.0 zk_read_per_namespace_count{key="bucket_7"} 3270890.0 zk_read_per_namespace_sum{key="bucket_7"} 6.69811733E8 zk_read_per_namespace{key="bucket_6",quantile="0.5"} 242.0 zk_read_per_namespace_count{key="bucket_6"} 3125579.0 zk_read_per_namespace_sum{key="bucket_6"} 6.39954595E8 zk_read_per_namespace{key="bucket_COMPUTE_0",quantile="0.5"} 177.0 zk_read_per_namespace_count{key="bucket_COMPUTE_0"} 10963.0 zk_read_per_namespace_sum{key="bucket_COMPUTE_0"} 1737617.0 zk_read_per_namespace{key="bucket_5",quantile="0.5"} 242.0 zk_read_per_namespace_count{key="bucket_5"} 3017838.0 zk_read_per_namespace_sum{key="bucket_5"} 6.17825522E8 zk_read_per_namespace{key="bucket_4",quantile="0.5"} 242.0 zk_read_per_namespace_count{key="bucket_4"} 3585639.0 zk_read_per_namespace_sum{key="bucket_4"} 7.34465507E8 zk_read_per_namespace{key="bucket_3",quantile="0.5"} 297.0 zk_read_per_namespace_count{key="bucket_3"} 3287333.0 zk_read_per_namespace_sum{key="bucket_3"} 7.60065072E8 zk_read_per_namespace{key="vmanage",quantile="0.5"} 171.0 zk_read_per_namespace_count{key="vmanage"} 7515.0 zk_read_per_namespace_sum{key="vmanage"} 1248192.0 zk_read_per_namespace{key="bucket_2",quantile="0.5"} 242.0 zk_read_per_namespace_count{key="bucket_2"} 2919316.0 zk_read_per_namespace_sum{key="bucket_2"} 5.97587062E8 zk_read_per_namespace{key="bucket_1",quantile="0.5"} 242.0 zk_read_per_namespace_count{key="bucket_1"} 2928652.0 zk_read_per_namespace_sum{key="bucket_1"} 5.99507423E8 zk_read_per_namespace{key="bucket_0",quantile="0.5"} 242.0 zk_read_per_namespace_count{key="bucket_0"} 3697060.0 zk_read_per_namespace_sum{key="bucket_0"} 7.57368454E8 zk_read_per_namespace{key="clickhouse",quantile="0.5"} 184.0 zk_read_per_namespace_count{key="clickhouse"} 1.158463718E9 zk_read_per_namespace_sum{key="clickhouse"} 1.391316441086E12 zk_read_per_namespace{key="interprocess_lock",quantile="0.5"} 112.0 zk_read_per_namespace_count{key="interprocess_lock"} 2493.0 zk_read_per_namespace_sum{key="interprocess_lock"} 346737.0 zk_read_final_proc_time_ms{quantile="0.5"} 0.0 zk_read_final_proc_time_ms{quantile="0.9"} 0.0 zk_read_final_proc_time_ms{quantile="0.99"} 1.0 zk_read_final_proc_time_ms_count 1.840708036E9 zk_read_final_proc_time_ms_sum 4.6922278E7 zk_synced_observers 0.0 zk_election_time{quantile="0.5"} NaN zk_election_time_count 1.0 zk_election_time_sum 359.0 zk_om_proposal_process_time_ms{quantile="0.5"} NaN zk_om_proposal_process_time_ms{quantile="0.9"} NaN zk_om_proposal_process_time_ms{quantile="0.99"} NaN zk_om_proposal_process_time_ms_count 0.0 zk_om_proposal_process_time_ms_sum 0.0 zk_quorum_size 3.0 zk_snapshottime{quantile="0.5"} NaN zk_snapshottime_count 74.0 zk_snapshottime_sum 592067.0 zk_connection_drop_count 18988.0 zk_ephemerals_count 20802.0 zk_learner_handler_qp_time_ms{key="1",quantile="0.5"} 0.0 zk_learner_handler_qp_time_ms{key="1",quantile="0.9"} 0.0 zk_learner_handler_qp_time_ms{key="1",quantile="0.99"} 2.0 zk_learner_handler_qp_time_ms_count{key="1"} 328872.0 zk_learner_handler_qp_time_ms_sum{key="1"} 545910.0 zk_learner_handler_qp_time_ms{key="2",quantile="0.5"} 0.0 zk_learner_handler_qp_time_ms{key="2",quantile="0.9"} 0.0 zk_learner_handler_qp_time_ms{key="2",quantile="0.99"} 1.0 zk_learner_handler_qp_time_ms_count{key="2"} 328934.0 zk_learner_handler_qp_time_ms_sum{key="2"} 28276.0 zk_znode_count 1844397.0 zk_min_proposal_size 36.0 zk_diff_count 5.0 zk_connection_revalidate_count 9.0 zk_outstanding_changes_queued 6.16965194E8 zk_proposal_process_time{quantile="0.5"} 0.0 zk_proposal_process_time_count 2.00481483E9 zk_proposal_process_time_sum 2.0630437E7 zk_process_cpu_seconds_total 968025.77 zk_process_start_time_seconds 1.73824491204E9 zk_process_open_fds 165.0 zk_process_max_fds 16000.0 zk_process_virtual_memory_bytes 3.703478272E10 zk_process_resident_memory_bytes 1.0832961536E10 zk_node_changed_watch_count{quantile="0.5"} 1.0 zk_node_changed_watch_count_count 33442.0 zk_node_changed_watch_count_sum 33442.0 zk_uptime 6.00389155E8 zk_prep_processor_queue_time_ms{quantile="0.5"} 0.0 zk_prep_processor_queue_time_ms{quantile="0.9"} 0.0 zk_prep_processor_queue_time_ms{quantile="0.99"} 2.0 zk_prep_processor_queue_time_ms_count 2.004813533E9 zk_prep_processor_queue_time_ms_sum 2.48315427E8 zk_write_commitproc_time_ms{quantile="0.5"} 1.0 zk_write_commitproc_time_ms{quantile="0.9"} 7.0 zk_write_commitproc_time_ms{quantile="0.99"} 26.0 zk_write_commitproc_time_ms_count 1.64209453E8 zk_write_commitproc_time_ms_sum 6.57199982E8 zk_synced_non_voting_followers 0.0 zk_jvm_memory_bytes_used{area="heap"} 3.135899936E9 zk_jvm_memory_bytes_used{area="nonheap"} 4.7517576E7 zk_jvm_memory_bytes_committed{area="heap"} 9.974054912E9 zk_jvm_memory_bytes_committed{area="nonheap"} 5.6434688E7 zk_jvm_memory_bytes_max{area="heap"} 9.974054912E9 zk_jvm_memory_bytes_max{area="nonheap"} -1.0 zk_jvm_memory_bytes_init{area="heap"} 9.974054912E9 zk_jvm_memory_bytes_init{area="nonheap"} 7667712.0 zk_jvm_memory_pool_bytes_used{pool="CodeHeap 'non-nmethods'"} 1226496.0 zk_jvm_memory_pool_bytes_used{pool="Metaspace"} 2.7758904E7 zk_jvm_memory_pool_bytes_used{pool="CodeHeap 'profiled nmethods'"} 8041728.0 zk_jvm_memory_pool_bytes_used{pool="Compressed Class Space"} 2726608.0 zk_jvm_memory_pool_bytes_used{pool="G1 Eden Space"} 9.98244352E8 zk_jvm_memory_pool_bytes_used{pool="G1 Old Gen"} 1.92374608E9 zk_jvm_memory_pool_bytes_used{pool="G1 Survivor Space"} 2.13909504E8 zk_jvm_memory_pool_bytes_used{pool="CodeHeap 'non-profiled nmethods'"} 7763840.0 zk_jvm_memory_pool_bytes_committed{pool="CodeHeap 'non-nmethods'"} 4784128.0 zk_jvm_memory_pool_bytes_committed{pool="Metaspace"} 2.8962816E7 zk_jvm_memory_pool_bytes_committed{pool="CodeHeap 'profiled nmethods'"} 1.1730944E7 zk_jvm_memory_pool_bytes_committed{pool="Compressed Class Space"} 3092480.0 zk_jvm_memory_pool_bytes_committed{pool="G1 Eden Space"} 4.726980608E9 zk_jvm_memory_pool_bytes_committed{pool="G1 Old Gen"} 5.0331648E9 zk_jvm_memory_pool_bytes_committed{pool="G1 Survivor Space"} 2.13909504E8 zk_jvm_memory_pool_bytes_committed{pool="CodeHeap 'non-profiled nmethods'"} 7864320.0 zk_jvm_memory_pool_bytes_max{pool="CodeHeap 'non-nmethods'"} 8773632.0 zk_jvm_memory_pool_bytes_max{pool="Metaspace"} -1.0 zk_jvm_memory_pool_bytes_max{pool="CodeHeap 'profiled nmethods'"} 1.21442304E8 zk_jvm_memory_pool_bytes_max{pool="Compressed Class Space"} 1.073741824E9 zk_jvm_memory_pool_bytes_max{pool="G1 Eden Space"} -1.0 zk_jvm_memory_pool_bytes_max{pool="G1 Old Gen"} 9.974054912E9 zk_jvm_memory_pool_bytes_max{pool="G1 Survivor Space"} -1.0 zk_jvm_memory_pool_bytes_max{pool="CodeHeap 'non-profiled nmethods'"} 1.21442304E8 zk_jvm_memory_pool_bytes_init{pool="CodeHeap 'non-nmethods'"} 2555904.0 zk_jvm_memory_pool_bytes_init{pool="Metaspace"} 0.0 zk_jvm_memory_pool_bytes_init{pool="CodeHeap 'profiled nmethods'"} 2555904.0 zk_jvm_memory_pool_bytes_init{pool="Compressed Class Space"} 0.0 zk_jvm_memory_pool_bytes_init{pool="G1 Eden Space"} 5.20093696E8 zk_jvm_memory_pool_bytes_init{pool="G1 Old Gen"} 9.453961216E9 zk_jvm_memory_pool_bytes_init{pool="G1 Survivor Space"} 0.0 zk_jvm_memory_pool_bytes_init{pool="CodeHeap 'non-profiled nmethods'"} 2555904.0 zk_request_throttle_queue_time_ms{quantile="0.5"} 0.0 zk_request_throttle_queue_time_ms{quantile="0.9"} 0.0 zk_request_throttle_queue_time_ms{quantile="0.99"} 1.0 zk_request_throttle_queue_time_ms_count 1.91572117E9 zk_request_throttle_queue_time_ms_sum 1.29291005E8 zk_prep_process_time{quantile="0.5"} 0.0 zk_prep_process_time_count 2.004817205E9 zk_prep_process_time_sum 9143122.0 zk_dbinittime{quantile="0.5"} NaN zk_dbinittime_count 1.0 zk_dbinittime_sum 14265.0 zk_commit_process_time{quantile="0.5"} 0.0 zk_commit_process_time_count 1.645014549E9 zk_commit_process_time_sum 3.8168051E7 zk_global_sessions 27.0 zk_last_proposal_size 633.0 zk_response_packet_get_children_cache_hits 7.5519812E8 zk_min_client_response_size 16.0 zk_throttled_ops 0.0 zk_netty_queued_buffer_capacity{quantile="0.5"} NaN zk_netty_queued_buffer_capacity_count 0.0 zk_netty_queued_buffer_capacity_sum 0.0 zk_updatelatency{quantile="0.5"} 2.0 zk_updatelatency{quantile="0.9"} 8.0 zk_updatelatency{quantile="0.99"} 26.0 zk_updatelatency_count 7.5171719E7 zk_updatelatency_sum 3.94137615E8 zk_jvm_buffer_pool_used_bytes{pool="mapped"} 0.0 zk_jvm_buffer_pool_used_bytes{pool="direct"} 1.4668779E7 zk_jvm_buffer_pool_capacity_bytes{pool="mapped"} 0.0 zk_jvm_buffer_pool_capacity_bytes{pool="direct"} 1.4668779E7 zk_jvm_buffer_pool_used_buffers{pool="mapped"} 0.0 zk_jvm_buffer_pool_used_buffers{pool="direct"} 258.0 zk_non_mtls_remote_conn_count 0.0 zk_prep_processor_request_queued 2.0062173E9 zk_max_file_descriptor_count 16000.0 zk_pending_syncs 0.0 zk_tls_handshake_exceeded 0.0 zk_add_dead_watcher_stall_time 0.0 zk_proposal_ack_creation_latency{quantile="0.5"} 1.0 zk_proposal_ack_creation_latency{quantile="0.9"} 3.0 zk_proposal_ack_creation_latency{quantile="0.99"} 44.0 zk_proposal_ack_creation_latency_count 1.64158045E8 zk_proposal_ack_creation_latency_sum 5.54626081E8 zk_min_latency 0.0 zk_quorum_ack_latency{quantile="0.5"} 1.0 zk_quorum_ack_latency{quantile="0.9"} 3.0 zk_quorum_ack_latency{quantile="0.99"} 9.0 zk_quorum_ack_latency_count 1.64136346E8 zk_quorum_ack_latency_sum 3.64618175E8 zk_commit_count 1.64516411E8 zk_commit_propagation_latency{quantile="0.5"} NaN zk_commit_propagation_latency{quantile="0.9"} NaN zk_commit_propagation_latency{quantile="0.99"} NaN zk_commit_propagation_latency_count 0.0 zk_commit_propagation_latency_sum 0.0 zk_prep_processor_queue_size{quantile="0.5"} 0.0 zk_prep_processor_queue_size_count 2.004811861E9 zk_prep_processor_queue_size_sum 2.563809426E9 zk_request_commit_queued 1.64516412E8 zk_startup_txns_load_time{quantile="0.5"} NaN zk_startup_txns_load_time_count 1.0 zk_startup_txns_load_time_sum 7080.0 zk_concurrent_request_processing_in_commit_processor{quantile="0.5"} 1.0 zk_concurrent_request_processing_in_commit_processor_count 4.058447E7 zk_concurrent_request_processing_in_commit_processor_sum 8.9925008E7 zk_jvm_pause_time_ms{quantile="0.5"} NaN zk_jvm_pause_time_ms{quantile="0.9"} NaN zk_jvm_pause_time_ms{quantile="0.99"} NaN zk_jvm_pause_time_ms_count 0.0 zk_jvm_pause_time_ms_sum 0.0 zk_sync_processor_queue_time_ms{quantile="0.5"} 0.0 zk_sync_processor_queue_time_ms{quantile="0.9"} 1.0 zk_sync_processor_queue_time_ms{quantile="0.99"} 15.0 zk_sync_processor_queue_time_ms_count 1.64160741E8 zk_sync_processor_queue_time_ms_sum 2.39328165E8 zk_revalidate_count 4.0 zk_node_created_watch_count{quantile="0.5"} NaN zk_node_created_watch_count_count 0.0 zk_node_created_watch_count_sum 0.0 zk_proposal_count 1.64516412E8 zk_leader_unavailable_time{quantile="0.5"} NaN zk_leader_unavailable_time_count 1.0 zk_leader_unavailable_time_sum 11119.0 zk_sync_processor_queue_flush_time_ms{quantile="0.5"} 0.0 zk_sync_processor_queue_flush_time_ms{quantile="0.9"} 1.0 zk_sync_processor_queue_flush_time_ms{quantile="0.99"} 3.0 zk_sync_processor_queue_flush_time_ms_count 7.4609113E7 zk_sync_processor_queue_flush_time_ms_sum 4.1127514E7 zk_digest_mismatches_count 0.0 zk_packets_sent 1.924174884E9 zk_learner_handler_qp_size{key="1",quantile="0.5"} 0.0 zk_learner_handler_qp_size_count{key="1"} 3.29231382E8 zk_learner_handler_qp_size_sum{key="1"} 2.877862036E9 zk_learner_handler_qp_size{key="2",quantile="0.5"} 0.0 zk_learner_handler_qp_size_count{key="2"} 3.29279451E8 zk_learner_handler_qp_size_sum{key="2"} 1.123999896E9 zk_socket_closing_time{quantile="0.5"} NaN zk_socket_closing_time_count 3.0 zk_socket_closing_time_sum 3.0 zk_packets_received 1.917140155E9 zk_response_packet_cache_misses 1.34410375E8 zk_cnxn_closed_without_zk_server_running 0.0 zk_dead_watchers_queued 0.0 zk_startup_txns_loaded{quantile="0.5"} NaN zk_startup_txns_loaded_count 1.0 zk_startup_txns_loaded_sum 732597.0 zk_stale_requests 14.0 zk_write_final_proc_time_ms{quantile="0.5"} 0.0 zk_write_final_proc_time_ms{quantile="0.9"} 0.0 zk_write_final_proc_time_ms{quantile="0.99"} 1.0 zk_write_final_proc_time_ms_count 1.64213498E8 zk_write_final_proc_time_ms_sum 7801806.0 zk_sync_process_time{quantile="0.5"} 0.0 zk_sync_process_time_count 1.64160844E8 zk_sync_process_time_sum 3322136.0 zk_local_write_committed_time_ms{quantile="0.5"} 0.0 zk_local_write_committed_time_ms{quantile="0.9"} 3.0 zk_local_write_committed_time_ms{quantile="0.99"} 22.0 zk_local_write_committed_time_ms_count 1.64208298E8 zk_local_write_committed_time_ms_sum 2.96474885E8 zk_connection_drop_probability 0.0 zk_jvm_info{version="11.0.7+10",vendor="AdoptOpenJDK",runtime="OpenJDK Runtime Environment"} 1.0 zk_skip_learner_request_to_next_processor_count 0.0 zk_connection_token_deficit{quantile="0.5"} 0.0 zk_connection_token_deficit_count 18955.0 zk_connection_token_deficit_sum 0.0 zk_max_latency 6331.0 zk_learner_proposal_received_count 0.0 zk_jvm_threads_current 261.0 zk_jvm_threads_daemon 208.0 zk_jvm_threads_peak 262.0 zk_jvm_threads_started_total 346.0 zk_jvm_threads_deadlocked 0.0 zk_jvm_threads_deadlocked_monitor 0.0 zk_jvm_threads_state{state="WAITING"} 203.0 zk_jvm_threads_state{state="RUNNABLE"} 47.0 zk_jvm_threads_state{state="NEW"} 0.0 zk_jvm_threads_state{state="TIMED_WAITING"} 11.0 zk_jvm_threads_state{state="BLOCKED"} 0.0 zk_jvm_threads_state{state="TERMINATED"} 0.0 zk_leader_uptime 6.00378435E8 zk_dead_watchers_cleared 0.0 zk_commit_commit_proc_req_queued{quantile="0.5"} 0.0 zk_commit_commit_proc_req_queued_count 1.645012484E9 zk_commit_commit_proc_req_queued_sum 1.085482139E9 zk_large_requests_rejected 0.0 zk_avg_latency 0.855 zk_follower_sync_time{quantile="0.5"} NaN zk_follower_sync_time_count 0.0 zk_follower_sync_time_sum 0.0 zk_dead_watchers_cleaner_latency{quantile="0.5"} NaN zk_dead_watchers_cleaner_latency{quantile="0.9"} NaN zk_dead_watchers_cleaner_latency{quantile="0.99"} NaN zk_dead_watchers_cleaner_latency_count 0.0 zk_dead_watchers_cleaner_latency_sum 0.0 zk_startup_snap_load_time{quantile="0.5"} NaN zk_startup_snap_load_time_count 1.0 zk_startup_snap_load_time_sum 7114.0 zk_connection_request_count 18962.0 zk_write_per_namespace{key="zookeeper",quantile="0.5"} NaN zk_write_per_namespace_count{key="zookeeper"} 1.0 zk_write_per_namespace_sum{key="zookeeper"} 155.0 zk_write_per_namespace{key="interprocess_lockvedge_serial_list_handler",quantile="0.5"} NaN zk_write_per_namespace_count{key="interprocess_lockvedge_serial_list_handler"} 502.0 zk_write_per_namespace_sum{key="interprocess_lockvedge_serial_list_handler"} 41256.0 zk_write_per_namespace{key="resource",quantile="0.5"} NaN zk_write_per_namespace_count{key="resource"} 96.0 zk_write_per_namespace_sum{key="resource"} 4594.0 zk_write_per_namespace{key="cluster-oracle",quantile="0.5"} NaN zk_write_per_namespace_count{key="cluster-oracle"} 76.0 zk_write_per_namespace_sum{key="cluster-oracle"} 17773.0 zk_write_per_namespace{key="bucket_9",quantile="0.5"} NaN zk_write_per_namespace_count{key="bucket_9"} 20.0 zk_write_per_namespace_sum{key="bucket_9"} 1696.0 zk_write_per_namespace{key="bucket_8",quantile="0.5"} NaN zk_write_per_namespace_count{key="bucket_8"} 20.0 zk_write_per_namespace_sum{key="bucket_8"} 1696.0 zk_write_per_namespace{key="bucket_7",quantile="0.5"} NaN zk_write_per_namespace_count{key="bucket_7"} 22.0 zk_write_per_namespace_sum{key="bucket_7"} 1862.0 zk_write_per_namespace{key="bucket_6",quantile="0.5"} NaN zk_write_per_namespace_count{key="bucket_6"} 22.0 zk_write_per_namespace_sum{key="bucket_6"} 1862.0 zk_write_per_namespace{key="bucket_COMPUTE_0",quantile="0.5"} NaN zk_write_per_namespace_count{key="bucket_COMPUTE_0"} 10.0 zk_write_per_namespace_sum{key="bucket_COMPUTE_0"} 854.0 zk_write_per_namespace{key="bucket_5",quantile="0.5"} NaN zk_write_per_namespace_count{key="bucket_5"} 24.0 zk_write_per_namespace_sum{key="bucket_5"} 2028.0 zk_write_per_namespace{key="bucket_4",quantile="0.5"} NaN zk_write_per_namespace_count{key="bucket_4"} 22.0 zk_write_per_namespace_sum{key="bucket_4"} 1862.0 zk_write_per_namespace{key="bucket_3",quantile="0.5"} NaN zk_write_per_namespace_count{key="bucket_3"} 25.0 zk_write_per_namespace_sum{key="bucket_3"} 2129.0 zk_write_per_namespace{key="vmanage",quantile="0.5"} 106.0 zk_write_per_namespace_count{key="vmanage"} 55163.0 zk_write_per_namespace_sum{key="vmanage"} 5245809.0 zk_write_per_namespace{key="bucket_2",quantile="0.5"} NaN zk_write_per_namespace_count{key="bucket_2"} 20.0 zk_write_per_namespace_sum{key="bucket_2"} 1696.0 zk_write_per_namespace{key="bucket_1",quantile="0.5"} NaN zk_write_per_namespace_count{key="bucket_1"} 24.0 zk_write_per_namespace_sum{key="bucket_1"} 2028.0 zk_write_per_namespace{key="bucket_0",quantile="0.5"} NaN zk_write_per_namespace_count{key="bucket_0"} 20.0 zk_write_per_namespace_sum{key="bucket_0"} 1696.0 zk_write_per_namespace{key="cloudDock",quantile="0.5"} NaN zk_write_per_namespace_count{key="cloudDock"} 62.0 zk_write_per_namespace_sum{key="cloudDock"} 3792.0 zk_write_per_namespace{key="clickhouse",quantile="0.5"} 133.0 zk_write_per_namespace_count{key="clickhouse"} 3.26682672E8 zk_write_per_namespace_sum{key="clickhouse"} 5.2725736734E10 zk_write_per_namespace{key="interprocess_lock",quantile="0.5"} 51.0 zk_write_per_namespace_count{key="interprocess_lock"} 33113.0 zk_write_per_namespace_sum{key="interprocess_lock"} 2885258.0 zk_time_waiting_empty_pool_in_commit_processor_read_ms{quantile="0.5"} 0.0 zk_time_waiting_empty_pool_in_commit_processor_read_ms_count 1.64215091E8 zk_time_waiting_empty_pool_in_commit_processor_read_ms_sum 1.3427886E7 zk_reads_after_write_in_session_queue{quantile="0.5"} 0.0 zk_reads_after_write_in_session_queue_count 1.64210601E8 zk_reads_after_write_in_session_queue_sum 6.6708317E7 zk_watch_count 25882.0 zk_stale_sessions_expired 54.0 zk_jvm_memory_pool_allocated_bytes_total{pool="CodeHeap 'profiled nmethods'"} 1.7771648E7 zk_jvm_memory_pool_allocated_bytes_total{pool="G1 Old Gen"} 1.69240372056E11 zk_jvm_memory_pool_allocated_bytes_total{pool="G1 Eden Space"} 4.3332860379136E13 zk_jvm_memory_pool_allocated_bytes_total{pool="CodeHeap 'non-profiled nmethods'"} 1.1479808E7 zk_jvm_memory_pool_allocated_bytes_total{pool="G1 Survivor Space"} 1.38332340224E11 zk_jvm_memory_pool_allocated_bytes_total{pool="Compressed Class Space"} 2802736.0 zk_jvm_memory_pool_allocated_bytes_total{pool="Metaspace"} 2.8036456E7 zk_jvm_memory_pool_allocated_bytes_total{pool="CodeHeap 'non-nmethods'"} 2.0250624E7 zk_connection_rejected 0.0 zk_bytes_received_count 2.10117921067E11 zk_ack_latency{key="1",quantile="0.5"} 1.0 zk_ack_latency{key="1",quantile="0.9"} 3.0 zk_ack_latency{key="1",quantile="0.99"} 5.0 zk_ack_latency_count{key="1"} 65949.0 zk_ack_latency_sum{key="1"} 168367.0 zk_ack_latency{key="2",quantile="0.5"} 1.0 zk_ack_latency{key="2",quantile="0.9"} 2.0 zk_ack_latency{key="2",quantile="0.99"} 3.0 zk_ack_latency_count{key="2"} 119519.0 zk_ack_latency_sum{key="2"} 182874.0 zk_ack_latency{key="0",quantile="0.5"} 1.0 zk_ack_latency{key="0",quantile="0.9"} 2.0 zk_ack_latency{key="0",quantile="0.99"} 3.0 zk_ack_latency_count{key="0"} 142874.0 zk_ack_latency_sum{key="0"} 189988.0 zk_snap_count 0.0 zk_write_commit_proc_req_queued{quantile="0.5"} 0.0 zk_write_commit_proc_req_queued_count 1.645013858E9 zk_write_commit_proc_req_queued_sum 3.14748476E9 zk_requests_not_forwarded_to_commit_processor 0.0 zk_outstanding_changes_removed 6.16831683E8 zk_synced_followers 2.0 zk_read_commitproc_time_ms{quantile="0.5"} 0.0 zk_read_commitproc_time_ms{quantile="0.9"} 1.0 zk_read_commitproc_time_ms{quantile="0.99"} 7.0 zk_read_commitproc_time_ms_count 1.84070831E9 zk_read_commitproc_time_ms_sum 8.7145171E8 zk_unrecoverable_error_count 0.0 zk_session_queues_drained{quantile="0.5"} 1.0 zk_session_queues_drained_count 1.64214057E8 zk_session_queues_drained_sum 1.6420962E8 zk_sync_processor_queue_size{quantile="0.5"} 1.0 zk_sync_processor_queue_size_count 1.64160254E8 zk_sync_processor_queue_size_sum 7.17684017E8 zk_request_throttle_wait_count 0.0 zk_read_commit_proc_issued{quantile="0.5"} 1.0 zk_read_commit_proc_issued_count 1.645013114E9 zk_read_commit_proc_issued_sum 1.774025414E9 zk_requests_in_session_queue{quantile="0.5"} 3.0 zk_requests_in_session_queue_count 3.95154112E8 zk_requests_in_session_queue_sum 2.511043545E9 zk_inflight_diff_count{quantile="0.5"} NaN zk_inflight_diff_count_count 10.0 zk_inflight_diff_count_sum 7.0 zk_learner_request_processor_queue_size{quantile="0.5"} NaN zk_learner_request_processor_queue_size_count 0.0 zk_learner_request_processor_queue_size_sum 0.0 zk_om_commit_process_time_ms{quantile="0.5"} NaN zk_om_commit_process_time_ms{quantile="0.9"} NaN zk_om_commit_process_time_ms{quantile="0.99"} NaN zk_om_commit_process_time_ms_count 0.0 zk_om_commit_process_time_ms_sum 0.0 zk_non_mtls_local_conn_count 0.0 zk_unsuccessful_handshake 0.0 zk_open_file_descriptor_count 165.0 zk_approximate_data_size 3.30080507E8 zk_num_alive_connections 18.0 zk_proposal_latency{quantile="0.5"} NaN zk_proposal_latency{quantile="0.9"} NaN zk_proposal_latency{quantile="0.99"} NaN zk_proposal_latency_count 0.0 zk_proposal_latency_sum 0.0 zk_fsynctime{quantile="0.5"} 0.0 zk_fsynctime_count 7.4608303E7 zk_fsynctime_sum 1.33159E7 zk_propagation_latency{quantile="0.5"} 2.0 zk_propagation_latency{quantile="0.9"} 7.0 zk_propagation_latency{quantile="0.99"} 26.0 zk_propagation_latency_count 1.64213632E8 zk_propagation_latency_sum 6.71632903E8 zk_close_session_prep_time{quantile="0.5"} 0.0 zk_close_session_prep_time{quantile="0.9"} 0.0 zk_close_session_prep_time{quantile="0.99"} 0.0 zk_close_session_prep_time_count 56664.0 zk_close_session_prep_time_sum 1414.0 zk_write_batch_time_in_commit_processor{quantile="0.5"} 0.0 zk_write_batch_time_in_commit_processor_count 1.6421658E8 zk_write_batch_time_in_commit_processor_sum 8636850.0 zk_ensemble_auth_success 0.0 zk_sessionless_connections_expired 0.0 zk_sync_processor_queue_and_flush_time_ms{quantile="0.5"} 1.0 zk_sync_processor_queue_and_flush_time_ms{quantile="0.9"} 3.0 zk_sync_processor_queue_and_flush_time_ms{quantile="0.99"} 43.0 zk_sync_processor_queue_and_flush_time_ms_count 1.64158856E8 zk_sync_processor_queue_and_flush_time_ms_sum 5.39440932E8 zk_jvm_classes_loaded 3842.0 zk_jvm_classes_loaded_total 3986.0 zk_jvm_classes_unloaded_total 144.0 zk_looking_count 1.0 zk_sync_processor_batch_size{quantile="0.5"} 1.0 zk_sync_processor_batch_size_count 7.4611134E7 zk_sync_processor_batch_size_sum 1.6415944E8 zk_last_client_response_size 16.0 zk_node_children_watch_count{quantile="0.5"} 1.0 zk_node_children_watch_count_count 5456579.0 zk_node_children_watch_count_sum 6979727.0 zk_learners 2.0 zk_learner_commit_received_count 0.0 zk_server_write_committed_time_ms{quantile="0.5"} NaN zk_server_write_committed_time_ms{quantile="0.9"} NaN zk_server_write_committed_time_ms{quantile="0.99"} NaN zk_server_write_committed_time_ms_count 0.0 zk_server_write_committed_time_ms_sum 0.0 zk_outstanding_tls_handshake 0.0 zk_inflight_snap_count{quantile="0.5"} NaN zk_inflight_snap_count_count 0.0 zk_inflight_snap_count_sum 0.0 zk_response_packet_get_children_cache_misses 6.4758394E7 zk_read_commit_proc_req_queued{quantile="0.5"} 1.0 zk_read_commit_proc_req_queued_count 1.645014827E9 zk_read_commit_proc_req_queued_sum 3.783841394E9 zk_readlatency{quantile="0.5"} 0.0 zk_readlatency{quantile="0.9"} 1.0 zk_readlatency{quantile="0.99"} 9.0 zk_readlatency_count 1.840708291E9 zk_readlatency_sum 1.216654897E9 zk_max_client_response_size 2441548.0 zk_auth_failed_count 0.0 zk_ensemble_auth_fail 0.0 zk_node_deleted_watch_count{quantile="0.5"} NaN zk_node_deleted_watch_count_count 21116.0 zk_node_deleted_watch_count_sum 21153.0 zk_observer_sync_time{quantile="0.5"} NaN zk_observer_sync_time_count 0.0 zk_observer_sync_time_sum 0.0 zk_quit_leading_due_to_disloyal_voter 0.0 zk_stale_replies 14.0 zk_ensemble_auth_skip 0.0 zk_unavailable_time{quantile="0.5"} NaN zk_unavailable_time_count 1.0 zk_unavailable_time_sum 11119.0 zk_sync_processor_request_queued 1.64516414E8 zk_response_bytes 1.584690555995E12 zk_reads_issued_from_session_queue{quantile="0.5"} 0.0 zk_reads_issued_from_session_queue_count 1.64212988E8 zk_reads_issued_from_session_queue_sum 6.6707686E7 zk_outstanding_requests 3.0 zk_insecure_admin_count 0.0 zk_jvm_gc_collection_seconds_count{gc="G1 Young Generation"} 13762.0 zk_jvm_gc_collection_seconds_sum{gc="G1 Young Generation"} 4029.203 zk_jvm_gc_collection_seconds_count{gc="G1 Old Generation"} 0.0 zk_jvm_gc_collection_seconds_sum{gc="G1 Old Generation"} 0.0 zk_response_packet_cache_hits 5.553762E7 zk_local_sessions 0.0 zk_write_commit_proc_issued{quantile="0.5"} 1.0 zk_write_commit_proc_issued_count 1.64215535E8 zk_write_commit_proc_issued_sum 1.64211098E8 zk_max_proposal_size 473302.0 zk_stale_requests_dropped 0.0 Follower-1 (No issue seen): CONFIG: # TEMPLATE_VERSION:3.5.8 # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # the port at which the clients will connect clientPort=2181 # the directory where the snapshot is stored. dataDir=/data/d98d54dd-dd9f-49cf-85e4-ddd922a635d8 # disable the per-ip limit on the number of connections since this is a non-production config maxClientCnxns=0 # Standalone mode or a Distributed mode standaloneEnabled=false tickTime=2000 initLimit=300 syncLimit=10 #SASL configuration quorum.auth.enableSasl=true quorum.auth.learnerRequireSasl=true quorum.auth.serverRequireSasl=true 4lw.commands.whitelist=stat, ruok, mntr metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider metricsProvider.httpPort=4888 metricsProvider.exportJvmInfo=true server.0=169.254.1.4:2888:3888 server.1=169.254.2.4:2888:3888 server.2=169.254.3.4:2888:3888 nms.instance.id=1 # Auto purge logs and snapshots. Keep latest 20 snapshots and logs. Run purge per 3 hours. autopurge.snapRetainCount=20 autopurge.purgeInterval=3 # It is the maximum value that client may request and the server will accept. # NOTE: high maxSessionTimeout will cause ephemeral node could not be deleted correctly. maxSessionTimeout=180000 # To avoid seeks ZooKeeper allocates space in the transaction log file in # blocks of preAllocSize kilobytes. The default block size is 64M. One reason # for changing the size of the blocks is to reduce the block size if snapshots # are taken more often. (Also, see snapCount). preAllocSize=131072 # ZooKeeper logs transactions to a transaction log. After snapCount transactions # are written to a log file a snapshot is started and a new transaction log file # is started. The default snapCount is 100000. snapCount=3000000 leaderServes=yes MNTR: zk_version 3.8.4-9316c2a7a97e1666d8f4593f34dd6fc36ecc436c, built on 2024-02-12 22:16 UTC zk_server_state follower zk_peer_state following - broadcast zk_reads_after_write_in_session_queue{quantile="0.5"} 0.0 zk_reads_after_write_in_session_queue_count 6.9919616E7 zk_reads_after_write_in_session_queue_sum 1.23756278E8 zk_jvm_pause_time_ms{quantile="0.5"} NaN zk_jvm_pause_time_ms{quantile="0.9"} NaN zk_jvm_pause_time_ms{quantile="0.99"} NaN zk_jvm_pause_time_ms_count 0.0 zk_jvm_pause_time_ms_sum 0.0 zk_readlatency{quantile="0.5"} 0.0 zk_readlatency{quantile="0.9"} 1.0 zk_readlatency{quantile="0.99"} 7.0 zk_readlatency_count 1.890519218E9 zk_readlatency_sum 1.349121202E9 zk_bytes_received_count 2.5701291799E11 zk_write_final_proc_time_ms{quantile="0.5"} 0.0 zk_write_final_proc_time_ms{quantile="0.9"} 0.0 zk_write_final_proc_time_ms{quantile="0.99"} 1.0 zk_write_final_proc_time_ms_count 1.6413934E8 zk_write_final_proc_time_ms_sum 7661034.0 zk_om_commit_process_time_ms{quantile="0.5"} NaN zk_om_commit_process_time_ms{quantile="0.9"} NaN zk_om_commit_process_time_ms{quantile="0.99"} NaN zk_om_commit_process_time_ms_count 0.0 zk_om_commit_process_time_ms_sum 0.0 zk_connection_drop_probability 0.0 zk_min_latency 0.0 zk_dbinittime{quantile="0.5"} NaN zk_dbinittime_count 2.0 zk_dbinittime_sum 28783.0 zk_uptime 6.0038332E8 zk_connection_revalidate_count 1.0 zk_startup_txns_loaded{quantile="0.5"} NaN zk_startup_txns_loaded_count 2.0 zk_startup_txns_loaded_sum 1564103.0 zk_max_client_response_size 177972.0 zk_process_cpu_seconds_total 1182508.0 zk_process_start_time_seconds 1.738244917342E9 zk_process_open_fds 155.0 zk_process_max_fds 16000.0 zk_process_virtual_memory_bytes 3.6762136576E10 zk_process_resident_memory_bytes 1.0775683072E10 zk_fsynctime{quantile="0.5"} 0.0 zk_fsynctime_count 6.5627002E7 zk_fsynctime_sum 1.6069461E7 zk_znode_count 1844397.0 zk_jvm_buffer_pool_used_bytes{pool="mapped"} 0.0 zk_jvm_buffer_pool_used_bytes{pool="direct"} 1.5888202E7 zk_jvm_buffer_pool_capacity_bytes{pool="mapped"} 0.0 zk_jvm_buffer_pool_capacity_bytes{pool="direct"} 1.5888202E7 zk_jvm_buffer_pool_used_buffers{pool="mapped"} 0.0 zk_jvm_buffer_pool_used_buffers{pool="direct"} 258.0 zk_snap_count 0.0 zk_sessionless_connections_expired 0.0 zk_non_mtls_remote_conn_count 0.0 zk_outstanding_changes_removed 0.0 zk_open_file_descriptor_count 155.0 zk_close_session_prep_time{quantile="0.5"} NaN zk_close_session_prep_time{quantile="0.9"} NaN zk_close_session_prep_time{quantile="0.99"} NaN zk_close_session_prep_time_count 0.0 zk_close_session_prep_time_sum 0.0 zk_quorum_size 3.0 zk_approximate_data_size 3.30080507E8 zk_node_children_watch_count{quantile="0.5"} 2.0 zk_node_children_watch_count_count 7977680.0 zk_node_children_watch_count_sum 8980059.0 zk_packets_received 1.961215622E9 zk_outstanding_requests 0.0 zk_outstanding_tls_handshake 0.0 zk_jvm_threads_current 253.0 zk_jvm_threads_daemon 209.0 zk_jvm_threads_peak 254.0 zk_jvm_threads_started_total 333.0 zk_jvm_threads_deadlocked 0.0 zk_jvm_threads_deadlocked_monitor 0.0 zk_jvm_threads_state{state="TIMED_WAITING"} 8.0 zk_jvm_threads_state{state="NEW"} 0.0 zk_jvm_threads_state{state="BLOCKED"} 0.0 zk_jvm_threads_state{state="RUNNABLE"} 43.0 zk_jvm_threads_state{state="TERMINATED"} 0.0 zk_jvm_threads_state{state="WAITING"} 202.0 zk_prep_processor_queue_size{quantile="0.5"} NaN zk_prep_processor_queue_size_count 0.0 zk_prep_processor_queue_size_sum 0.0 zk_local_write_committed_time_ms{quantile="0.5"} 0.0 zk_local_write_committed_time_ms{quantile="0.9"} 2.0 zk_local_write_committed_time_ms{quantile="0.99"} 12.0 zk_local_write_committed_time_ms_count 6.9918973E7 zk_local_write_committed_time_ms_sum 8.6771729E7 zk_jvm_info{version="11.0.7+10",vendor="AdoptOpenJDK",runtime="OpenJDK Runtime Environment"} 1.0 zk_max_latency 6227.0 zk_request_throttle_wait_count 0.0 zk_proposal_process_time{quantile="0.5"} NaN zk_proposal_process_time_count 0.0 zk_proposal_process_time_sum 0.0 zk_om_proposal_process_time_ms{quantile="0.5"} NaN zk_om_proposal_process_time_ms{quantile="0.9"} NaN zk_om_proposal_process_time_ms{quantile="0.99"} NaN zk_om_proposal_process_time_ms_count 0.0 zk_om_proposal_process_time_ms_sum 0.0 zk_response_packet_get_children_cache_misses 6.5247416E7 zk_read_commitproc_time_ms{quantile="0.5"} 0.0 zk_read_commitproc_time_ms{quantile="0.9"} 1.0 zk_read_commitproc_time_ms{quantile="0.99"} 6.0 zk_read_commitproc_time_ms_count 1.890518902E9 zk_read_commitproc_time_ms_sum 1.098122082E9 zk_digest_mismatches_count 0.0 zk_propagation_latency{quantile="0.5"} 1.0 zk_propagation_latency{quantile="0.9"} 8.0 zk_propagation_latency{quantile="0.99"} 27.0 zk_propagation_latency_count 1.64097085E8 zk_propagation_latency_sum 2.243966436E9 zk_outstanding_changes_queued 0.0 zk_cnxn_closed_without_zk_server_running 0.0 zk_dead_watchers_queued 0.0 zk_commit_propagation_latency{quantile="0.5"} 1.0 zk_commit_propagation_latency{quantile="0.9"} 5.0 zk_commit_propagation_latency{quantile="0.99"} 20.0 zk_commit_propagation_latency_count 1.64002112E8 zk_commit_propagation_latency_sum 1.882800214E9 zk_auth_failed_count 0.0 zk_connection_request_count 18967.0 zk_learner_proposal_received_count 1.64476851E8 zk_prep_processor_request_queued 0.0 zk_read_commit_proc_issued{quantile="0.5"} 1.0 zk_read_commit_proc_issued_count 1.692549281E9 zk_read_commit_proc_issued_sum 1.766767472E9 zk_startup_snap_load_time{quantile="0.5"} NaN zk_startup_snap_load_time_count 2.0 zk_startup_snap_load_time_sum 14656.0 zk_read_commit_proc_req_queued{quantile="0.5"} 1.0 zk_read_commit_proc_req_queued_count 1.692552327E9 zk_read_commit_proc_req_queued_sum 4.644177987E9 zk_sync_processor_queue_flush_time_ms{quantile="0.5"} 0.0 zk_sync_processor_queue_flush_time_ms{quantile="0.9"} 1.0 zk_sync_processor_queue_flush_time_ms{quantile="0.99"} 4.0 zk_sync_processor_queue_flush_time_ms_count 6.5627063E7 zk_sync_processor_queue_flush_time_ms_sum 4.0702923E7 zk_connection_drop_count 19007.0 zk_node_deleted_watch_count{quantile="0.5"} NaN zk_node_deleted_watch_count_count 7444.0 zk_node_deleted_watch_count_sum 7444.0 zk_jvm_gc_collection_seconds_count{gc="G1 Young Generation"} 28051.0 zk_jvm_gc_collection_seconds_sum{gc="G1 Young Generation"} 10729.679 zk_jvm_gc_collection_seconds_count{gc="G1 Old Generation"} 0.0 zk_jvm_gc_collection_seconds_sum{gc="G1 Old Generation"} 0.0 zk_quit_leading_due_to_disloyal_voter 0.0 zk_requests_in_session_queue{quantile="0.5"} 9.0 zk_requests_in_session_queue_count 2.63600194E8 zk_requests_in_session_queue_sum 3.207832455E9 zk_sync_process_time{quantile="0.5"} 0.0 zk_sync_process_time_count 1.64086236E8 zk_sync_process_time_sum 3282160.0 zk_learner_request_processor_queue_size{quantile="0.5"} 0.0 zk_learner_request_processor_queue_size_count 1.960364769E9 zk_learner_request_processor_queue_size_sum 1.077921616E9 zk_requests_not_forwarded_to_commit_processor 0.0 zk_response_packet_get_children_cache_hits 8.08161502E8 zk_skip_learner_request_to_next_processor_count 0.0 zk_dead_watchers_cleared 0.0 zk_startup_txns_load_time{quantile="0.5"} NaN zk_startup_txns_load_time_count 2.0 zk_startup_txns_load_time_sum 14057.0 zk_stale_requests_dropped 0.0 zk_proposal_latency{quantile="0.5"} 0.0 zk_proposal_latency{quantile="0.9"} 3.0 zk_proposal_latency{quantile="0.99"} 19.0 zk_proposal_latency_count 1.58909932E8 zk_proposal_latency_sum 1.192914908E9 zk_sync_processor_batch_size{quantile="0.5"} 1.0 zk_sync_processor_batch_size_count 6.5626823E7 zk_sync_processor_batch_size_sum 1.64086676E8 zk_server_write_committed_time_ms{quantile="0.5"} 0.0 zk_server_write_committed_time_ms{quantile="0.9"} 2.0 zk_server_write_committed_time_ms{quantile="0.99"} 15.0 zk_server_write_committed_time_ms_count 9.4217366E7 zk_server_write_committed_time_ms_sum 2.63867576E8 zk_jvm_memory_bytes_used{area="heap"} 4.19130432E9 zk_jvm_memory_bytes_used{area="nonheap"} 4.5924064E7 zk_jvm_memory_bytes_committed{area="heap"} 9.974054912E9 zk_jvm_memory_bytes_committed{area="nonheap"} 5.3723136E7 zk_jvm_memory_bytes_max{area="heap"} 9.974054912E9 zk_jvm_memory_bytes_max{area="nonheap"} -1.0 zk_jvm_memory_bytes_init{area="heap"} 9.974054912E9 zk_jvm_memory_bytes_init{area="nonheap"} 7667712.0 zk_jvm_memory_pool_bytes_used{pool="CodeHeap 'non-nmethods'"} 1222656.0 zk_jvm_memory_pool_bytes_used{pool="Metaspace"} 2.7189376E7 zk_jvm_memory_pool_bytes_used{pool="CodeHeap 'profiled nmethods'"} 8042624.0 zk_jvm_memory_pool_bytes_used{pool="Compressed Class Space"} 2683872.0 zk_jvm_memory_pool_bytes_used{pool="G1 Eden Space"} 1.828716544E9 zk_jvm_memory_pool_bytes_used{pool="G1 Old Gen"} 2.274507392E9 zk_jvm_memory_pool_bytes_used{pool="G1 Survivor Space"} 8.8080384E7 zk_jvm_memory_pool_bytes_used{pool="CodeHeap 'non-profiled nmethods'"} 6785536.0 zk_jvm_memory_pool_bytes_committed{pool="CodeHeap 'non-nmethods'"} 4718592.0 zk_jvm_memory_pool_bytes_committed{pool="Metaspace"} 2.8491776E7 zk_jvm_memory_pool_bytes_committed{pool="CodeHeap 'profiled nmethods'"} 1.0289152E7 zk_jvm_memory_pool_bytes_committed{pool="Compressed Class Space"} 3145728.0 zk_jvm_memory_pool_bytes_committed{pool="G1 Eden Space"} 2.386558976E9 zk_jvm_memory_pool_bytes_committed{pool="G1 Old Gen"} 7.499415552E9 zk_jvm_memory_pool_bytes_committed{pool="G1 Survivor Space"} 8.8080384E7 zk_jvm_memory_pool_bytes_committed{pool="CodeHeap 'non-profiled nmethods'"} 7077888.0 zk_jvm_memory_pool_bytes_max{pool="CodeHeap 'non-nmethods'"} 8773632.0 zk_jvm_memory_pool_bytes_max{pool="Metaspace"} -1.0 zk_jvm_memory_pool_bytes_max{pool="CodeHeap 'profiled nmethods'"} 1.21442304E8 zk_jvm_memory_pool_bytes_max{pool="Compressed Class Space"} 1.073741824E9 zk_jvm_memory_pool_bytes_max{pool="G1 Eden Space"} -1.0 zk_jvm_memory_pool_bytes_max{pool="G1 Old Gen"} 9.974054912E9 zk_jvm_memory_pool_bytes_max{pool="G1 Survivor Space"} -1.0 zk_jvm_memory_pool_bytes_max{pool="CodeHeap 'non-profiled nmethods'"} 1.21442304E8 zk_jvm_memory_pool_bytes_init{pool="CodeHeap 'non-nmethods'"} 2555904.0 zk_jvm_memory_pool_bytes_init{pool="Metaspace"} 0.0 zk_jvm_memory_pool_bytes_init{pool="CodeHeap 'profiled nmethods'"} 2555904.0 zk_jvm_memory_pool_bytes_init{pool="Compressed Class Space"} 0.0 zk_jvm_memory_pool_bytes_init{pool="G1 Eden Space"} 5.20093696E8 zk_jvm_memory_pool_bytes_init{pool="G1 Old Gen"} 9.453961216E9 zk_jvm_memory_pool_bytes_init{pool="G1 Survivor Space"} 0.0 zk_jvm_memory_pool_bytes_init{pool="CodeHeap 'non-profiled nmethods'"} 2555904.0 zk_looking_count 1.0 zk_socket_closing_time{quantile="0.5"} NaN zk_socket_closing_time_count 0.0 zk_socket_closing_time_sum 0.0 zk_connection_rejected 0.0 zk_learner_commit_received_count 1.64476853E8 zk_local_sessions 0.0 zk_dead_watchers_cleaner_latency{quantile="0.5"} NaN zk_dead_watchers_cleaner_latency{quantile="0.9"} NaN zk_dead_watchers_cleaner_latency{quantile="0.99"} NaN zk_dead_watchers_cleaner_latency_count 0.0 zk_dead_watchers_cleaner_latency_sum 0.0 zk_request_commit_queued 1.64516541E8 zk_netty_queued_buffer_capacity{quantile="0.5"} NaN zk_netty_queued_buffer_capacity_count 0.0 zk_netty_queued_buffer_capacity_sum 0.0 zk_observer_sync_time{quantile="0.5"} NaN zk_observer_sync_time_count 0.0 zk_observer_sync_time_sum 0.0 zk_response_packet_cache_hits 3.6215279E7 zk_sync_processor_queue_and_flush_time_ms{quantile="0.5"} 1.0 zk_sync_processor_queue_and_flush_time_ms{quantile="0.9"} 3.0 zk_sync_processor_queue_and_flush_time_ms{quantile="0.99"} 129.0 zk_sync_processor_queue_and_flush_time_ms_count 1.64079892E8 zk_sync_processor_queue_and_flush_time_ms_sum 1.056144562E9 zk_connection_token_deficit{quantile="0.5"} 0.0 zk_connection_token_deficit_count 18958.0 zk_connection_token_deficit_sum 0.0 zk_ephemerals_count 20802.0 zk_read_per_namespace{key="zookeeper",quantile="0.5"} NaN zk_read_per_namespace_count{key="zookeeper"} 9.0 zk_read_per_namespace_sum{key="zookeeper"} 2007.0 zk_read_per_namespace{key="interprocess_lockvedge_serial_list_handler",quantile="0.5"} NaN zk_read_per_namespace_count{key="interprocess_lockvedge_serial_list_handler"} 18.0 zk_read_per_namespace_sum{key="interprocess_lockvedge_serial_list_handler"} 2861.0 zk_read_per_namespace{key="resource",quantile="0.5"} NaN zk_read_per_namespace_count{key="resource"} 14.0 zk_read_per_namespace_sum{key="resource"} 1567.0 zk_read_per_namespace{key="cluster-oracle",quantile="0.5"} NaN zk_read_per_namespace_count{key="cluster-oracle"} 9.0 zk_read_per_namespace_sum{key="cluster-oracle"} 4713.0 zk_read_per_namespace{key="bucket_9",quantile="0.5"} 242.0 zk_read_per_namespace_count{key="bucket_9"} 2930973.0 zk_read_per_namespace_sum{key="bucket_9"} 6.00846372E8 zk_read_per_namespace{key="bucket_8",quantile="0.5"} 242.0 zk_read_per_namespace_count{key="bucket_8"} 3235055.0 zk_read_per_namespace_sum{key="bucket_8"} 6.63333471E8 zk_read_per_namespace{key="bucket_7",quantile="0.5"} 242.0 zk_read_per_namespace_count{key="bucket_7"} 2639474.0 zk_read_per_namespace_sum{key="bucket_7"} 5.40944166E8 zk_read_per_namespace{key="bucket_6",quantile="0.5"} 242.0 zk_read_per_namespace_count{key="bucket_6"} 2516973.0 zk_read_per_namespace_sum{key="bucket_6"} 5.15771342E8 zk_read_per_namespace{key="bucket_5",quantile="0.5"} 242.0 zk_read_per_namespace_count{key="bucket_5"} 2445955.0 zk_read_per_namespace_sum{key="bucket_5"} 5.01175592E8 zk_read_per_namespace{key="bucket_4",quantile="0.5"} 242.0 zk_read_per_namespace_count{key="bucket_4"} 2887894.0 zk_read_per_namespace_sum{key="bucket_4"} 5.91994093E8 zk_read_per_namespace{key="bucket_3",quantile="0.5"} 297.0 zk_read_per_namespace_count{key="bucket_3"} 2646404.0 zk_read_per_namespace_sum{key="bucket_3"} 6.12750901E8 zk_read_per_namespace{key="bucket_2",quantile="0.5"} 242.0 zk_read_per_namespace_count{key="bucket_2"} 2357366.0 zk_read_per_namespace_sum{key="bucket_2"} 4.82971812E8 zk_read_per_namespace{key="vmanage",quantile="0.5"} 171.0 zk_read_per_namespace_count{key="vmanage"} 9155.0 zk_read_per_namespace_sum{key="vmanage"} 1644262.0 zk_read_per_namespace{key="bucket_1",quantile="0.5"} 242.0 zk_read_per_namespace_count{key="bucket_1"} 2357524.0 zk_read_per_namespace_sum{key="bucket_1"} 4.83003751E8 zk_read_per_namespace{key="bucket_0",quantile="0.5"} 242.0 zk_read_per_namespace_count{key="bucket_0"} 2978115.0 zk_read_per_namespace_sum{key="bucket_0"} 6.10533834E8 zk_read_per_namespace{key="olap-db",quantile="0.5"} NaN zk_read_per_namespace_count{key="olap-db"} 1.0 zk_read_per_namespace_sum{key="olap-db"} 86.0 zk_read_per_namespace{key="clickhouse",quantile="0.5"} 190.0 zk_read_per_namespace_count{key="clickhouse"} 1.18962658E9 zk_read_per_namespace_sum{key="clickhouse"} 1.928614712895E12 zk_read_per_namespace{key="interprocess_lock",quantile="0.5"} NaN zk_read_per_namespace_count{key="interprocess_lock"} 2439.0 zk_read_per_namespace_sum{key="interprocess_lock"} 417997.0 zk_write_batch_time_in_commit_processor{quantile="0.5"} 0.0 zk_write_batch_time_in_commit_processor_count 1.64139273E8 zk_write_batch_time_in_commit_processor_sum 8260897.0 zk_reads_issued_from_session_queue{quantile="0.5"} 0.0 zk_reads_issued_from_session_queue_count 1.64136348E8 zk_reads_issued_from_session_queue_sum 1.23755127E8 zk_unavailable_time{quantile="0.5"} NaN zk_unavailable_time_count 1.0 zk_unavailable_time_sum 28373.0 zk_inflight_snap_count{quantile="0.5"} NaN zk_inflight_snap_count_count 0.0 zk_inflight_snap_count_sum 0.0 zk_unrecoverable_error_count 0.0 zk_proposal_count 0.0 zk_commit_count 0.0 zk_commit_commit_proc_req_queued{quantile="0.5"} 0.0 zk_commit_commit_proc_req_queued_count 1.692549418E9 zk_commit_commit_proc_req_queued_sum 2.793332606E9 zk_response_bytes 2.170778353304E12 zk_prep_process_time{quantile="0.5"} NaN zk_prep_process_time_count 0.0 zk_prep_process_time_sum 0.0 zk_jvm_classes_loaded 3776.0 zk_jvm_classes_loaded_total 3921.0 zk_jvm_classes_unloaded_total 145.0 zk_avg_latency 0.9229 zk_write_commitproc_time_ms{quantile="0.5"} 2.0 zk_write_commitproc_time_ms{quantile="0.9"} 8.0 zk_write_commitproc_time_ms{quantile="0.99"} 23.0 zk_write_commitproc_time_ms_count 6.9919457E7 zk_write_commitproc_time_ms_sum 4.21273442E8 zk_node_created_watch_count{quantile="0.5"} NaN zk_node_created_watch_count_count 0.0 zk_node_created_watch_count_sum 0.0 zk_time_waiting_empty_pool_in_commit_processor_read_ms{quantile="0.5"} 0.0 zk_time_waiting_empty_pool_in_commit_processor_read_ms_count 1.64138715E8 zk_time_waiting_empty_pool_in_commit_processor_read_ms_sum 1.1105498E7 zk_sync_processor_request_queued 1.64516413E8 zk_large_requests_rejected 0.0 zk_write_commit_proc_issued{quantile="0.5"} 1.0 zk_write_commit_proc_issued_count 1.64138001E8 zk_write_commit_proc_issued_sum 1.64136921E8 zk_pending_session_queue_size{quantile="0.5"} 1.0 zk_pending_session_queue_size_count 6.9921322E7 zk_pending_session_queue_size_sum 9.1635671E7 zk_unsuccessful_handshake 0.0 zk_read_final_proc_time_ms{quantile="0.5"} 0.0 zk_read_final_proc_time_ms{quantile="0.9"} 0.0 zk_read_final_proc_time_ms{quantile="0.99"} 1.0 zk_read_final_proc_time_ms_count 1.890519378E9 zk_read_final_proc_time_ms_sum 5.3724542E7 zk_insecure_admin_count 0.0 zk_proposal_ack_creation_latency{quantile="0.5"} 1.0 zk_proposal_ack_creation_latency{quantile="0.9"} 7.0 zk_proposal_ack_creation_latency{quantile="0.99"} 146.0 zk_proposal_ack_creation_latency_count 1.63498072E8 zk_proposal_ack_creation_latency_sum 2.574304145E9 zk_jvm_memory_pool_allocated_bytes_total{pool="CodeHeap 'profiled nmethods'"} 1.5584384E7 zk_jvm_memory_pool_allocated_bytes_total{pool="G1 Old Gen"} 3.01118276944E11 zk_jvm_memory_pool_allocated_bytes_total{pool="G1 Eden Space"} 3.511737647104E13 zk_jvm_memory_pool_allocated_bytes_total{pool="CodeHeap 'non-profiled nmethods'"} 9635840.0 zk_jvm_memory_pool_allocated_bytes_total{pool="G1 Survivor Space"} 1.31583705088E11 zk_jvm_memory_pool_allocated_bytes_total{pool="Compressed Class Space"} 2760584.0 zk_jvm_memory_pool_allocated_bytes_total{pool="Metaspace"} 2.7468552E7 zk_jvm_memory_pool_allocated_bytes_total{pool="CodeHeap 'non-nmethods'"} 8120064.0 zk_snapshottime{quantile="0.5"} NaN zk_snapshottime_count 74.0 zk_snapshottime_sum 673752.0 zk_follower_sync_time{quantile="0.5"} NaN zk_follower_sync_time_count 1.0 zk_follower_sync_time_sum 23435.0 zk_revalidate_count 0.0 zk_global_sessions 27.0 zk_write_per_namespace{key="zookeeper",quantile="0.5"} NaN zk_write_per_namespace_count{key="zookeeper"} 2.0 zk_write_per_namespace_sum{key="zookeeper"} 310.0 zk_write_per_namespace{key="interprocess_lockvedge_serial_list_handler",quantile="0.5"} NaN zk_write_per_namespace_count{key="interprocess_lockvedge_serial_list_handler"} 502.0 zk_write_per_namespace_sum{key="interprocess_lockvedge_serial_list_handler"} 41256.0 zk_write_per_namespace{key="resource",quantile="0.5"} NaN zk_write_per_namespace_count{key="resource"} 96.0 zk_write_per_namespace_sum{key="resource"} 4594.0 zk_write_per_namespace{key="bucket_9",quantile="0.5"} NaN zk_write_per_namespace_count{key="bucket_9"} 43.0 zk_write_per_namespace_sum{key="bucket_9"} 3515.0 zk_write_per_namespace{key="cluster-oracle",quantile="0.5"} NaN zk_write_per_namespace_count{key="cluster-oracle"} 144.0 zk_write_per_namespace_sum{key="cluster-oracle"} 34774.0 zk_write_per_namespace{key="bucket_8",quantile="0.5"} NaN zk_write_per_namespace_count{key="bucket_8"} 43.0 zk_write_per_namespace_sum{key="bucket_8"} 3515.0 zk_write_per_namespace{key="bucket_7",quantile="0.5"} NaN zk_write_per_namespace_count{key="bucket_7"} 45.0 zk_write_per_namespace_sum{key="bucket_7"} 3681.0 zk_write_per_namespace{key="bucket_6",quantile="0.5"} NaN zk_write_per_namespace_count{key="bucket_6"} 45.0 zk_write_per_namespace_sum{key="bucket_6"} 3681.0 zk_write_per_namespace{key="bucket_COMPUTE_0",quantile="0.5"} NaN zk_write_per_namespace_count{key="bucket_COMPUTE_0"} 18.0 zk_write_per_namespace_sum{key="bucket_COMPUTE_0"} 1378.0 zk_write_per_namespace{key="bucket_5",quantile="0.5"} NaN zk_write_per_namespace_count{key="bucket_5"} 49.0 zk_write_per_namespace_sum{key="bucket_5"} 4013.0 zk_write_per_namespace{key="bucket_4",quantile="0.5"} NaN zk_write_per_namespace_count{key="bucket_4"} 45.0 zk_write_per_namespace_sum{key="bucket_4"} 3681.0 zk_write_per_namespace{key="bucket_3",quantile="0.5"} NaN zk_write_per_namespace_count{key="bucket_3"} 50.0 zk_write_per_namespace_sum{key="bucket_3"} 4114.0 zk_write_per_namespace{key="vmanage",quantile="0.5"} 106.0 zk_write_per_namespace_count{key="vmanage"} 55413.0 zk_write_per_namespace_sum{key="vmanage"} 5262598.0 zk_write_per_namespace{key="bucket_2",quantile="0.5"} NaN zk_write_per_namespace_count{key="bucket_2"} 43.0 zk_write_per_namespace_sum{key="bucket_2"} 3515.0 zk_write_per_namespace{key="bucket_1",quantile="0.5"} NaN zk_write_per_namespace_count{key="bucket_1"} 49.0 zk_write_per_namespace_sum{key="bucket_1"} 4013.0 zk_write_per_namespace{key="bucket_0",quantile="0.5"} NaN zk_write_per_namespace_count{key="bucket_0"} 43.0 zk_write_per_namespace_sum{key="bucket_0"} 3515.0 zk_write_per_namespace{key="cloudDock",quantile="0.5"} NaN zk_write_per_namespace_count{key="cloudDock"} 62.0 zk_write_per_namespace_sum{key="cloudDock"} 3792.0 zk_write_per_namespace{key="clickhouse",quantile="0.5"} 133.0 zk_write_per_namespace_count{key="clickhouse"} 3.26810863E8 zk_write_per_namespace_sum{key="clickhouse"} 5.2720962572E10 zk_write_per_namespace{key="interprocess_lock",quantile="0.5"} 51.0 zk_write_per_namespace_count{key="interprocess_lock"} 39822.0 zk_write_per_namespace_sum{key="interprocess_lock"} 3592382.0 zk_concurrent_request_processing_in_commit_processor{quantile="0.5"} 1.0 zk_concurrent_request_processing_in_commit_processor_count 3.3535647E7 zk_concurrent_request_processing_in_commit_processor_sum 9.0461931E7 zk_ensemble_auth_fail 0.0 zk_stale_sessions_expired 0.0 zk_sync_processor_queue_size{quantile="0.5"} 1.0 zk_sync_processor_queue_size_count 1.64085298E8 zk_sync_processor_queue_size_sum 2.452539936E9 zk_watch_count 16936.0 zk_last_client_response_size 114.0 zk_diff_count 0.0 zk_quorum_ack_latency{quantile="0.5"} NaN zk_quorum_ack_latency{quantile="0.9"} NaN zk_quorum_ack_latency{quantile="0.99"} NaN zk_quorum_ack_latency_count 0.0 zk_quorum_ack_latency_sum 0.0 zk_add_dead_watcher_stall_time 0.0 zk_commit_process_time{quantile="0.5"} 0.0 zk_commit_process_time_count 1.692549757E9 zk_commit_process_time_sum 3.5564252E7 zk_inflight_diff_count{quantile="0.5"} NaN zk_inflight_diff_count_count 0.0 zk_inflight_diff_count_sum 0.0 zk_ensemble_auth_success 0.0 zk_write_commit_proc_req_queued{quantile="0.5"} 0.0 zk_write_commit_proc_req_queued_count 1.692551129E9 zk_write_commit_proc_req_queued_sum 1.937588501E9 zk_updatelatency{quantile="0.5"} 2.0 zk_updatelatency{quantile="0.9"} 8.0 zk_updatelatency{quantile="0.99"} 23.0 zk_updatelatency_count 6.9920156E7 zk_updatelatency_sum 4.40734027E8 zk_max_file_descriptor_count 16000.0 zk_ensemble_auth_skip 0.0 zk_num_alive_connections 8.0 zk_node_changed_watch_count{quantile="0.5"} 1.0 zk_node_changed_watch_count_count 26001.0 zk_node_changed_watch_count_sum 26001.0 zk_election_time{quantile="0.5"} NaN zk_election_time_count 1.0 zk_election_time_sum 91.0 zk_stale_requests 0.0 zk_request_throttle_queue_time_ms{quantile="0.5"} 0.0 zk_request_throttle_queue_time_ms{quantile="0.9"} 0.0 zk_request_throttle_queue_time_ms{quantile="0.99"} 1.0 zk_request_throttle_queue_time_ms_count 1.960367196E9 zk_request_throttle_queue_time_ms_sum 1.07246375E8 zk_synced_observers 0.0 zk_prep_processor_queue_time_ms{quantile="0.5"} NaN zk_prep_processor_queue_time_ms{quantile="0.9"} NaN zk_prep_processor_queue_time_ms{quantile="0.99"} NaN zk_prep_processor_queue_time_ms_count 0.0 zk_prep_processor_queue_time_ms_sum 0.0 zk_leader_unavailable_time{quantile="0.5"} NaN zk_leader_unavailable_time_count 0.0 zk_leader_unavailable_time_sum 0.0 zk_sync_processor_queue_time_ms{quantile="0.5"} 0.0 zk_sync_processor_queue_time_ms{quantile="0.9"} 1.0 zk_sync_processor_queue_time_ms{quantile="0.99"} 62.0 zk_sync_processor_queue_time_ms_count 1.64085526E8 zk_sync_processor_queue_time_ms_sum 4.95322939E8 zk_min_client_response_size 16.0 zk_watch_bytes 9.17008694E8 zk_non_mtls_local_conn_count 0.0 zk_session_queues_drained{quantile="0.5"} 0.0 zk_session_queues_drained_count 1.64137254E8 zk_session_queues_drained_sum 6.9919214E7 zk_tls_handshake_exceeded 0.0 zk_packets_sent 1.970227397E9 zk_response_packet_cache_misses 1.35984303E8 zk_throttled_ops 0.0 zk_stale_replies 134.0 Follower-2 (Issue seen): CONFIG: # TEMPLATE_VERSION:3.5.8 # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # the port at which the clients will connect clientPort=2181 # the directory where the snapshot is stored. dataDir=/data/d98d54dd-dd9f-49cf-85e4-ddd922a635d8 # disable the per-ip limit on the number of connections since this is a non-production config maxClientCnxns=0 # Standalone mode or a Distributed mode standaloneEnabled=false tickTime=2000 initLimit=300 syncLimit=10 #SASL configuration quorum.auth.enableSasl=true quorum.auth.learnerRequireSasl=true quorum.auth.serverRequireSasl=true 4lw.commands.whitelist=stat, ruok, mntr metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider metricsProvider.httpPort=4888 metricsProvider.exportJvmInfo=true server.0=169.254.1.4:2888:3888 server.1=169.254.2.4:2888:3888 server.2=169.254.3.4:2888:3888 nms.instance.id=2 # Auto purge logs and snapshots. Keep latest 20 snapshots and logs. Run purge per 3 hours. autopurge.snapRetainCount=20 autopurge.purgeInterval=3 # It is the maximum value that client may request and the server will accept. # NOTE: high maxSessionTimeout will cause ephemeral node could not be deleted correctly. maxSessionTimeout=180000 # To avoid seeks ZooKeeper allocates space in the transaction log file in # blocks of preAllocSize kilobytes. The default block size is 64M. One reason # for changing the size of the blocks is to reduce the block size if snapshots # are taken more often. (Also, see snapCount). preAllocSize=131072 # ZooKeeper logs transactions to a transaction log. After snapCount transactions # are written to a log file a snapshot is started and a new transaction log file # is started. The default snapCount is 100000. snapCount=3000000 leaderServes=yes MNTR: zk_version 3.8.4-9316c2a7a97e1666d8f4593f34dd6fc36ecc436c, built on 2024-02-12 22:16 UTC zk_server_state follower zk_peer_state following - broadcast zk_reads_after_write_in_session_queue{quantile="0.5"} 0.0 zk_reads_after_write_in_session_queue_count 10563.0 zk_reads_after_write_in_session_queue_sum 0.0 zk_jvm_pause_time_ms{quantile="0.5"} NaN zk_jvm_pause_time_ms{quantile="0.9"} NaN zk_jvm_pause_time_ms{quantile="0.99"} NaN zk_jvm_pause_time_ms_count 0.0 zk_jvm_pause_time_ms_sum 0.0 zk_connection_drop_probability 0.0 zk_outstanding_tls_handshake 0.0 zk_readlatency{quantile="0.5"} 0.0 zk_readlatency{quantile="0.9"} 1.0 zk_readlatency{quantile="0.99"} 2.0 zk_readlatency_count 4.1419678E7 zk_readlatency_sum 6305982.0 zk_bytes_received_count 5.348331669E9 zk_write_final_proc_time_ms{quantile="0.5"} 0.0 zk_write_final_proc_time_ms{quantile="0.9"} 0.0 zk_write_final_proc_time_ms{quantile="0.99"} 1.0 zk_write_final_proc_time_ms_count 2.923343E7 zk_write_final_proc_time_ms_sum 878764.0 zk_last_client_response_size 16.0 zk_om_commit_process_time_ms{quantile="0.5"} NaN zk_om_commit_process_time_ms{quantile="0.9"} NaN zk_om_commit_process_time_ms{quantile="0.99"} NaN zk_om_commit_process_time_ms_count 0.0 zk_om_commit_process_time_ms_sum 0.0 zk_dbinittime{quantile="0.5"} NaN zk_dbinittime_count 1.0 zk_dbinittime_sum 40479.0 zk_connection_revalidate_count 1.0 zk_ephemerals_count 20812.0 zk_startup_txns_loaded{quantile="0.5"} NaN zk_startup_txns_loaded_count 1.0 zk_startup_txns_loaded_sum 1928525.0 zk_process_cpu_seconds_total 20005.09 zk_process_start_time_seconds 1.738735713453E9 zk_process_open_fds 150.0 zk_process_max_fds 16000.0 zk_process_virtual_memory_bytes 4.1122377728E10 zk_process_resident_memory_bytes 8.148135936E9 zk_fsynctime{quantile="0.5"} 0.0 zk_fsynctime_count 1.4801953E7 zk_fsynctime_sum 1522449.0 zk_jvm_buffer_pool_used_bytes{pool="mapped"} 0.0 zk_jvm_buffer_pool_used_bytes{pool="direct"} 9414544.0 zk_jvm_buffer_pool_capacity_bytes{pool="mapped"} 0.0 zk_jvm_buffer_pool_capacity_bytes{pool="direct"} 9414544.0 zk_jvm_buffer_pool_used_buffers{pool="mapped"} 0.0 zk_jvm_buffer_pool_used_buffers{pool="direct"} 258.0 zk_snap_count 0.0 zk_sessionless_connections_expired 0.0 zk_outstanding_changes_removed 0.0 zk_approximate_data_size 3.28800301E8 zk_close_session_prep_time{quantile="0.5"} NaN zk_close_session_prep_time{quantile="0.9"} NaN zk_close_session_prep_time{quantile="0.99"} NaN zk_close_session_prep_time_count 0.0 zk_close_session_prep_time_sum 0.0 zk_node_children_watch_count{quantile="0.5"} NaN zk_node_children_watch_count_count 0.0 zk_node_children_watch_count_sum 0.0 zk_jvm_threads_current 285.0 zk_jvm_threads_daemon 237.0 zk_jvm_threads_peak 322.0 zk_jvm_threads_started_total 342.0 zk_jvm_threads_deadlocked 0.0 zk_jvm_threads_deadlocked_monitor 0.0 zk_jvm_threads_state{state="TIMED_WAITING"} 9.0 zk_jvm_threads_state{state="NEW"} 0.0 zk_jvm_threads_state{state="BLOCKED"} 0.0 zk_jvm_threads_state{state="RUNNABLE"} 40.0 zk_jvm_threads_state{state="TERMINATED"} 0.0 zk_jvm_threads_state{state="WAITING"} 236.0 zk_prep_processor_queue_size{quantile="0.5"} NaN zk_prep_processor_queue_size_count 0.0 zk_prep_processor_queue_size_sum 0.0 zk_local_write_committed_time_ms{quantile="0.5"} 0.0 zk_local_write_committed_time_ms{quantile="0.9"} 0.0 zk_local_write_committed_time_ms{quantile="0.99"} 1.0 zk_local_write_committed_time_ms_count 10563.0 zk_local_write_committed_time_ms_sum 494.0 zk_global_sessions 35.0 zk_min_client_response_size 16.0 zk_jvm_info{version="11.0.7+10",vendor="AdoptOpenJDK",runtime="OpenJDK Runtime Environment"} 1.0 zk_synced_observers 0.0 zk_request_throttle_wait_count 0.0 zk_min_latency 0.0 zk_proposal_process_time{quantile="0.5"} NaN zk_proposal_process_time_count 0.0 zk_proposal_process_time_sum 0.0 zk_om_proposal_process_time_ms{quantile="0.5"} NaN zk_om_proposal_process_time_ms{quantile="0.9"} NaN zk_om_proposal_process_time_ms{quantile="0.99"} NaN zk_om_proposal_process_time_ms_count 0.0 zk_om_proposal_process_time_ms_sum 0.0 zk_response_packet_get_children_cache_misses 12.0 zk_read_commitproc_time_ms{quantile="0.5"} 0.0 zk_read_commitproc_time_ms{quantile="0.9"} 0.0 zk_read_commitproc_time_ms{quantile="0.99"} 1.0 zk_read_commitproc_time_ms_count 4.1419676E7 zk_read_commitproc_time_ms_sum 2896778.0 zk_digest_mismatches_count 3.1074832E7 zk_propagation_latency{quantile="0.5"} 1.0 zk_propagation_latency{quantile="0.9"} 4.0 zk_propagation_latency{quantile="0.99"} 10.0 zk_propagation_latency_count 2.9233324E7 zk_propagation_latency_sum 5.82056967E8 zk_local_sessions 0.0 zk_outstanding_changes_queued 0.0 zk_cnxn_closed_without_zk_server_running 0.0 zk_dead_watchers_queued 0.0 zk_commit_propagation_latency{quantile="0.5"} 1.0 zk_commit_propagation_latency{quantile="0.9"} 3.0 zk_commit_propagation_latency{quantile="0.99"} 10.0 zk_commit_propagation_latency_count 2.9233069E7 zk_commit_propagation_latency_sum 5.76665555E8 zk_num_alive_connections 4.0 zk_connection_request_count 3524.0 zk_learner_proposal_received_count 2.9221577E7 zk_prep_processor_request_queued 0.0 zk_read_commit_proc_issued{quantile="0.5"} 1.0 zk_read_commit_proc_issued_count 6.8124114E7 zk_read_commit_proc_issued_sum 4.1419676E7 zk_startup_snap_load_time{quantile="0.5"} NaN zk_startup_snap_load_time_count 1.0 zk_startup_snap_load_time_sum 6898.0 zk_read_commit_proc_req_queued{quantile="0.5"} 1.0 zk_read_commit_proc_req_queued_count 6.8124114E7 zk_read_commit_proc_req_queued_sum 4.1411724E7 zk_sync_processor_queue_flush_time_ms{quantile="0.5"} 0.0 zk_sync_processor_queue_flush_time_ms{quantile="0.9"} 1.0 zk_sync_processor_queue_flush_time_ms{quantile="0.99"} 1.0 zk_sync_processor_queue_flush_time_ms_count 1.480194E7 zk_sync_processor_queue_flush_time_ms_sum 6288767.0 zk_connection_drop_count 3520.0 zk_node_deleted_watch_count{quantile="0.5"} NaN zk_node_deleted_watch_count_count 0.0 zk_node_deleted_watch_count_sum 0.0 zk_jvm_gc_collection_seconds_count{gc="G1 Young Generation"} 308.0 zk_jvm_gc_collection_seconds_sum{gc="G1 Young Generation"} 17.199 zk_jvm_gc_collection_seconds_count{gc="G1 Old Generation"} 0.0 zk_jvm_gc_collection_seconds_sum{gc="G1 Old Generation"} 0.0 zk_quit_leading_due_to_disloyal_voter 0.0 zk_outstanding_requests 0.0 zk_requests_in_session_queue{quantile="0.5"} 1.0 zk_requests_in_session_queue_count 21126.0 zk_requests_in_session_queue_sum 21126.0 zk_sync_process_time{quantile="0.5"} 0.0 zk_sync_process_time_count 2.9235426E7 zk_sync_process_time_sum 455866.0 zk_learner_request_processor_queue_size{quantile="0.5"} 0.0 zk_learner_request_processor_queue_size_count 4.1430242E7 zk_learner_request_processor_queue_size_sum 5211288.0 zk_requests_not_forwarded_to_commit_processor 0.0 zk_response_packet_get_children_cache_hits 5.0 zk_skip_learner_request_to_next_processor_count 0.0 zk_dead_watchers_cleared 0.0 zk_startup_txns_load_time{quantile="0.5"} NaN zk_startup_txns_load_time_count 1.0 zk_startup_txns_load_time_sum 30406.0 zk_stale_requests_dropped 0.0 zk_proposal_latency{quantile="0.5"} 0.0 zk_proposal_latency{quantile="0.9"} 1.0 zk_proposal_latency{quantile="0.99"} 4.0 zk_proposal_latency_count 2.6780316E7 zk_proposal_latency_sum 1.6871684E7 zk_sync_processor_batch_size{quantile="0.5"} 1.0 zk_sync_processor_batch_size_count 1.480194E7 zk_sync_processor_batch_size_sum 2.9235428E7 zk_server_write_committed_time_ms{quantile="0.5"} 0.0 zk_server_write_committed_time_ms{quantile="0.9"} 0.0 zk_server_write_committed_time_ms{quantile="0.99"} 1.0 zk_server_write_committed_time_ms_count 2.9222867E7 zk_server_write_committed_time_ms_sum 3969068.0 zk_jvm_memory_bytes_used{area="heap"} 3.659588232E9 zk_jvm_memory_bytes_used{area="nonheap"} 4.4673656E7 zk_jvm_memory_bytes_committed{area="heap"} 9.974054912E9 zk_jvm_memory_bytes_committed{area="nonheap"} 5.2281344E7 zk_jvm_memory_bytes_max{area="heap"} 9.974054912E9 zk_jvm_memory_bytes_max{area="nonheap"} -1.0 zk_jvm_memory_bytes_init{area="heap"} 9.974054912E9 zk_jvm_memory_bytes_init{area="nonheap"} 7667712.0 zk_jvm_memory_pool_bytes_used{pool="CodeHeap 'non-nmethods'"} 1221888.0 zk_jvm_memory_pool_bytes_used{pool="Metaspace"} 2.7312272E7 zk_jvm_memory_pool_bytes_used{pool="CodeHeap 'profiled nmethods'"} 7994880.0 zk_jvm_memory_pool_bytes_used{pool="Compressed Class Space"} 2760296.0 zk_jvm_memory_pool_bytes_used{pool="G1 Eden Space"} 2.07618048E9 zk_jvm_memory_pool_bytes_used{pool="G1 Old Gen"} 1.499521672E9 zk_jvm_memory_pool_bytes_used{pool="G1 Survivor Space"} 8.388608E7 zk_jvm_memory_pool_bytes_used{pool="CodeHeap 'non-profiled nmethods'"} 5384320.0 zk_jvm_memory_pool_bytes_committed{pool="CodeHeap 'non-nmethods'"} 4784128.0 zk_jvm_memory_pool_bytes_committed{pool="Metaspace"} 2.8229632E7 zk_jvm_memory_pool_bytes_committed{pool="CodeHeap 'profiled nmethods'"} 1.0551296E7 zk_jvm_memory_pool_bytes_committed{pool="Compressed Class Space"} 3145728.0 zk_jvm_memory_pool_bytes_committed{pool="G1 Eden Space"} 6.199181312E9 zk_jvm_memory_pool_bytes_committed{pool="G1 Old Gen"} 3.69098752E9 zk_jvm_memory_pool_bytes_committed{pool="G1 Survivor Space"} 8.388608E7 zk_jvm_memory_pool_bytes_committed{pool="CodeHeap 'non-profiled nmethods'"} 5570560.0 zk_jvm_memory_pool_bytes_max{pool="CodeHeap 'non-nmethods'"} 8773632.0 zk_jvm_memory_pool_bytes_max{pool="Metaspace"} -1.0 zk_jvm_memory_pool_bytes_max{pool="CodeHeap 'profiled nmethods'"} 1.21442304E8 zk_jvm_memory_pool_bytes_max{pool="Compressed Class Space"} 1.073741824E9 zk_jvm_memory_pool_bytes_max{pool="G1 Eden Space"} -1.0 zk_jvm_memory_pool_bytes_max{pool="G1 Old Gen"} 9.974054912E9 zk_jvm_memory_pool_bytes_max{pool="G1 Survivor Space"} -1.0 zk_jvm_memory_pool_bytes_max{pool="CodeHeap 'non-profiled nmethods'"} 1.21442304E8 zk_jvm_memory_pool_bytes_init{pool="CodeHeap 'non-nmethods'"} 2555904.0 zk_jvm_memory_pool_bytes_init{pool="Metaspace"} 0.0 zk_jvm_memory_pool_bytes_init{pool="CodeHeap 'profiled nmethods'"} 2555904.0 zk_jvm_memory_pool_bytes_init{pool="Compressed Class Space"} 0.0 zk_jvm_memory_pool_bytes_init{pool="G1 Eden Space"} 5.20093696E8 zk_jvm_memory_pool_bytes_init{pool="G1 Old Gen"} 9.453961216E9 zk_jvm_memory_pool_bytes_init{pool="G1 Survivor Space"} 0.0 zk_jvm_memory_pool_bytes_init{pool="CodeHeap 'non-profiled nmethods'"} 2555904.0 zk_looking_count 3.0 zk_socket_closing_time{quantile="0.5"} NaN zk_socket_closing_time_count 2.0 zk_socket_closing_time_sum 0.0 zk_connection_rejected 0.0 zk_learner_commit_received_count 2.9221577E7 zk_dead_watchers_cleaner_latency{quantile="0.5"} NaN zk_dead_watchers_cleaner_latency{quantile="0.9"} NaN zk_dead_watchers_cleaner_latency{quantile="0.99"} NaN zk_dead_watchers_cleaner_latency_count 0.0 zk_dead_watchers_cleaner_latency_sum 0.0 zk_request_commit_queued 2.923343E7 zk_netty_queued_buffer_capacity{quantile="0.5"} NaN zk_netty_queued_buffer_capacity_count 0.0 zk_netty_queued_buffer_capacity_sum 0.0 zk_observer_sync_time{quantile="0.5"} NaN zk_observer_sync_time_count 0.0 zk_observer_sync_time_sum 0.0 zk_response_packet_cache_hits 62167.0 zk_sync_processor_queue_and_flush_time_ms{quantile="0.5"} 1.0 zk_sync_processor_queue_and_flush_time_ms{quantile="0.9"} 1.0 zk_sync_processor_queue_and_flush_time_ms{quantile="0.99"} 93.0 zk_sync_processor_queue_and_flush_time_ms_count 2.9234248E7 zk_sync_processor_queue_and_flush_time_ms_sum 6.974033E7 zk_connection_token_deficit{quantile="0.5"} 0.0 zk_connection_token_deficit_count 3524.0 zk_connection_token_deficit_sum 0.0 zk_watch_count 3.0 zk_read_per_namespace{key="zookeeper",quantile="0.5"} NaN zk_read_per_namespace_count{key="zookeeper"} 2.0 zk_read_per_namespace_sum{key="zookeeper"} 446.0 zk_read_per_namespace{key="clickhouse",quantile="0.5"} 411.0 zk_read_per_namespace_count{key="clickhouse"} 2.7593539E7 zk_read_per_namespace_sum{key="clickhouse"} 8.298735638E9 zk_read_per_namespace{key="cluster-oracle",quantile="0.5"} NaN zk_read_per_namespace_count{key="cluster-oracle"} 26.0 zk_read_per_namespace_sum{key="cluster-oracle"} 13738.0 zk_write_batch_time_in_commit_processor{quantile="0.5"} 0.0 zk_write_batch_time_in_commit_processor_count 2.923343E7 zk_write_batch_time_in_commit_processor_sum 915391.0 zk_reads_issued_from_session_queue{quantile="0.5"} 0.0 zk_reads_issued_from_session_queue_count 2.923343E7 zk_reads_issued_from_session_queue_sum 0.0 zk_quorum_size 3.0 zk_unavailable_time{quantile="0.5"} NaN zk_unavailable_time_count 1.0 zk_unavailable_time_sum 19344.0 zk_inflight_snap_count{quantile="0.5"} NaN zk_inflight_snap_count_count 0.0 zk_inflight_snap_count_sum 0.0 zk_unrecoverable_error_count 2.0 zk_proposal_count 0.0 zk_commit_count 0.0 zk_commit_commit_proc_req_queued{quantile="0.5"} 0.0 zk_commit_commit_proc_req_queued_count 6.8124114E7 zk_commit_commit_proc_req_queued_sum 1.33120799E8 zk_response_bytes 6.086652564E9 zk_prep_process_time{quantile="0.5"} NaN zk_prep_process_time_count 0.0 zk_prep_process_time_sum 0.0 zk_packets_sent 4.1435305E7 zk_max_client_response_size 3184.0 zk_jvm_classes_loaded 3924.0 zk_jvm_classes_loaded_total 3924.0 zk_jvm_classes_unloaded_total 0.0 zk_write_commitproc_time_ms{quantile="0.5"} 1.0 zk_write_commitproc_time_ms{quantile="0.9"} 2.0 zk_write_commitproc_time_ms{quantile="0.99"} 3.0 zk_write_commitproc_time_ms_count 10563.0 zk_write_commitproc_time_ms_sum 29632.0 zk_node_created_watch_count{quantile="0.5"} NaN zk_node_created_watch_count_count 0.0 zk_node_created_watch_count_sum 0.0 zk_time_waiting_empty_pool_in_commit_processor_read_ms{quantile="0.5"} 0.0 zk_time_waiting_empty_pool_in_commit_processor_read_ms_count 2.923343E7 zk_time_waiting_empty_pool_in_commit_processor_read_ms_sum 57126.0 zk_sync_processor_request_queued 2.9255806E7 zk_large_requests_rejected 0.0 zk_write_commit_proc_issued{quantile="0.5"} 1.0 zk_write_commit_proc_issued_count 2.923343E7 zk_write_commit_proc_issued_sum 2.923343E7 zk_pending_session_queue_size{quantile="0.5"} 1.0 zk_pending_session_queue_size_count 10563.0 zk_pending_session_queue_size_sum 10563.0 zk_unsuccessful_handshake 0.0 zk_open_file_descriptor_count 150.0 zk_read_final_proc_time_ms{quantile="0.5"} 0.0 zk_read_final_proc_time_ms{quantile="0.9"} 0.0 zk_read_final_proc_time_ms{quantile="0.99"} 1.0 zk_read_final_proc_time_ms_count 4.1419676E7 zk_read_final_proc_time_ms_sum 790732.0 zk_insecure_admin_count 0.0 zk_proposal_ack_creation_latency{quantile="0.5"} 1.0 zk_proposal_ack_creation_latency{quantile="0.9"} 2.0 zk_proposal_ack_creation_latency{quantile="0.99"} 101.0 zk_proposal_ack_creation_latency_count 2.9046881E7 zk_proposal_ack_creation_latency_sum 6.34693815E8 zk_jvm_memory_pool_allocated_bytes_total{pool="CodeHeap 'profiled nmethods'"} 1.268736E7 zk_jvm_memory_pool_allocated_bytes_total{pool="G1 Old Gen"} 1.612165424E9 zk_jvm_memory_pool_allocated_bytes_total{pool="G1 Eden Space"} 1.575070203904E12 zk_jvm_memory_pool_allocated_bytes_total{pool="CodeHeap 'non-profiled nmethods'"} 6816512.0 zk_jvm_memory_pool_allocated_bytes_total{pool="G1 Survivor Space"} 2.05520896E9 zk_jvm_memory_pool_allocated_bytes_total{pool="Compressed Class Space"} 2760296.0 zk_jvm_memory_pool_allocated_bytes_total{pool="Metaspace"} 2.7312272E7 zk_jvm_memory_pool_allocated_bytes_total{pool="CodeHeap 'non-nmethods'"} 9874688.0 zk_snapshottime{quantile="0.5"} NaN zk_snapshottime_count 13.0 zk_snapshottime_sum 83413.0 zk_follower_sync_time{quantile="0.5"} NaN zk_follower_sync_time_count 1.0 zk_follower_sync_time_sum 6393.0 zk_revalidate_count 0.0 zk_write_per_namespace{key="zookeeper",quantile="0.5"} NaN zk_write_per_namespace_count{key="zookeeper"} 4.0 zk_write_per_namespace_sum{key="zookeeper"} 620.0 zk_write_per_namespace{key="vmanage",quantile="0.5"} 106.0 zk_write_per_namespace_count{key="vmanage"} 9379.0 zk_write_per_namespace_sum{key="vmanage"} 930876.0 zk_write_per_namespace{key="interprocess_lockvedge_serial_list_handler",quantile="0.5"} NaN zk_write_per_namespace_count{key="interprocess_lockvedge_serial_list_handler"} 40.0 zk_write_per_namespace_sum{key="interprocess_lockvedge_serial_list_handler"} 2904.0 zk_write_per_namespace{key="resource",quantile="0.5"} NaN zk_write_per_namespace_count{key="resource"} 72.0 zk_write_per_namespace_sum{key="resource"} 3526.0 zk_write_per_namespace{key="clickhouse",quantile="0.5"} 133.0 zk_write_per_namespace_count{key="clickhouse"} 6.2345195E7 zk_write_per_namespace_sum{key="clickhouse"} 1.0044229618E10 zk_write_per_namespace{key="interprocess_lock",quantile="0.5"} 51.0 zk_write_per_namespace_count{key="interprocess_lock"} 7114.0 zk_write_per_namespace_sum{key="interprocess_lock"} 644562.0 zk_write_per_namespace{key="cluster-oracle",quantile="0.5"} NaN zk_write_per_namespace_count{key="cluster-oracle"} 1.0 zk_write_per_namespace_sum{key="cluster-oracle"} 126.0 zk_write_per_namespace{key="bucket_COMPUTE_0",quantile="0.5"} NaN zk_write_per_namespace_count{key="bucket_COMPUTE_0"} 2.0 zk_write_per_namespace_sum{key="bucket_COMPUTE_0"} 182.0 zk_concurrent_request_processing_in_commit_processor{quantile="0.5"} 1.0 zk_concurrent_request_processing_in_commit_processor_count 539244.0 zk_concurrent_request_processing_in_commit_processor_sum 768873.0 zk_ensemble_auth_fail 0.0 zk_stale_sessions_expired 0.0 zk_sync_processor_queue_size{quantile="0.5"} 0.0 zk_sync_processor_queue_size_count 2.9235429E7 zk_sync_processor_queue_size_sum 1.77968932E8 zk_diff_count 0.0 zk_quorum_ack_latency{quantile="0.5"} NaN zk_quorum_ack_latency{quantile="0.9"} NaN zk_quorum_ack_latency{quantile="0.99"} NaN zk_quorum_ack_latency_count 0.0 zk_quorum_ack_latency_sum 0.0 zk_max_latency 770.0 zk_add_dead_watcher_stall_time 0.0 zk_commit_process_time{quantile="0.5"} 0.0 zk_commit_process_time_count 6.8124114E7 zk_commit_process_time_sum 1346881.0 zk_inflight_diff_count{quantile="0.5"} NaN zk_inflight_diff_count_count 0.0 zk_inflight_diff_count_sum 0.0 zk_ensemble_auth_success 0.0 zk_write_commit_proc_req_queued{quantile="0.5"} 0.0 zk_write_commit_proc_req_queued_count 6.8124114E7 zk_write_commit_proc_req_queued_sum 29984.0 zk_uptime 1.09548726E8 zk_updatelatency{quantile="0.5"} 2.0 zk_updatelatency{quantile="0.9"} 3.0 zk_updatelatency{quantile="0.99"} 3.0 zk_updatelatency_count 10561.0 zk_updatelatency_sum 32919.0 zk_non_mtls_remote_conn_count 0.0 zk_packets_received 4.1438092E7 zk_avg_latency 0.153 zk_ensemble_auth_skip 0.0 zk_node_changed_watch_count{quantile="0.5"} NaN zk_node_changed_watch_count_count 0.0 zk_node_changed_watch_count_sum 0.0 zk_max_file_descriptor_count 16000.0 zk_election_time{quantile="0.5"} NaN zk_election_time_count 3.0 zk_election_time_sum 104.0 zk_znode_count 1839551.0 zk_stale_requests 0.0 zk_request_throttle_queue_time_ms{quantile="0.5"} 0.0 zk_request_throttle_queue_time_ms{quantile="0.9"} 0.0 zk_request_throttle_queue_time_ms{quantile="0.99"} 1.0 zk_request_throttle_queue_time_ms_count 4.1430239E7 zk_request_throttle_queue_time_ms_sum 1347436.0 zk_prep_processor_queue_time_ms{quantile="0.5"} NaN zk_prep_processor_queue_time_ms{quantile="0.9"} NaN zk_prep_processor_queue_time_ms{quantile="0.99"} NaN zk_prep_processor_queue_time_ms_count 0.0 zk_prep_processor_queue_time_ms_sum 0.0 zk_leader_unavailable_time{quantile="0.5"} NaN zk_leader_unavailable_time_count 0.0 zk_leader_unavailable_time_sum 0.0 zk_sync_processor_queue_time_ms{quantile="0.5"} 0.0 zk_sync_processor_queue_time_ms{quantile="0.9"} 1.0 zk_sync_processor_queue_time_ms{quantile="0.99"} 42.0 zk_sync_processor_queue_time_ms_count 2.9235428E7 zk_sync_processor_queue_time_ms_sum 3.3358476E7 zk_watch_bytes 0.0 zk_session_queues_drained{quantile="0.5"} 0.0 zk_session_queues_drained_count 2.923343E7 zk_session_queues_drained_sum 10563.0 zk_tls_handshake_exceeded 0.0 zk_response_packet_cache_misses 1.3734602E7 zk_non_mtls_local_conn_count 0.0 zk_throttled_ops 0.0 zk_auth_failed_count 0.0 zk_stale_replies 0.0 Thanks, Rounak From: Rounak Kakkar (rkakkar) <rkak...@cisco.com> Date: Thursday, 6 February 2025 at 17:58 To: Josef Roehrl <josef.roe...@fuseforward.com>, u...@zookeeper.apache.org <u...@zookeeper.apache.org>, dev@zookeeper.apache.org <dev@zookeeper.apache.org>, Steph van Schalkwyk <svanschalk...@gmail.com> Cc: Jitesh Mulchandani (jmulchan) <jmulc...@cisco.com>, Nilavazhagi M (nilm) <n...@cisco.com>, Liwei Wang (liwewang) <liwew...@cisco.com> Subject: Inconsistency for a ZK Path in the cluster Hello Team, We are using Zookeeper in cluster mode. For one of the paths, we notice a data inconsistency between a follower node and the other nodes in the system. We observe that the version for the path increases alongside content updates. However, there are four additional entries in that path in the follower where the issue is present. The version consistently lags four updates behind both the leader and the other followers. If we try to delete one of the extra entries from that follower node, it fails with the error: Node does not exist. What could cause this issue? Could you please help check this? Logs: Leader: vmanage@60f9084accc9:/var/lib/zookeeper$ /var/lib/zookeeper/bin/zkServer.sh status ZooKeeper JMX enabled by default Using config: /var/lib/zookeeper/bin/../conf/zoo.cfg Client port found: 2181. Client address: localhost. Client SSL: false. Mode: leader vmanage@60f9084accc9:/var/lib/zookeeper$ [zk: 127.0.0.1:2181(CONNECTED) 1] stat -w /clickhouse/tables/shard_2/aggregated_apps_dpi_app_summary_7f1396c5_a4d8_48c9_a3f3_90271305c114/replicas/replica_2/parts cZxid = 0x13000611b0 ctime = Thu Jun 06 13:04:52 UTC 2024 mZxid = 0x13000611b0 mtime = Thu Jun 06 13:04:52 UTC 2024 pZxid = 0x9109c98fb2 cversion = 2332172 dataVersion = 0 aclVersion = 0 ephemeralOwner = 0x0 dataLength = 0 numChildren = 22 [zk: 127.0.0.1:2181(CONNECTED) 2] ls /clickhouse/tables/shard_2/aggregated_apps_dpi_app_summary_7f1396c5_a4d8_48c9_a3f3_90271305c114/replicas/replica_2/parts [20250206_0_685_142, 20250206_0_690_143, 20250206_0_695_144, 20250206_0_700_145, 20250206_686_686_0, 20250206_687_687_0, 20250206_688_688_0, 20250206_689_689_0, 20250206_690_690_0, 20250206_691_691_0, 20250206_692_692_0, 20250206_693_693_0, 20250206_694_694_0, 20250206_695_695_0, 20250206_696_696_0, 20250206_697_697_0, 20250206_698_698_0, 20250206_699_699_0, 20250206_700_700_0, 20250206_701_701_0, 20250206_702_702_0, 20250206_703_703_0] [zk: 127.0.0.1:2181(CONNECTED) 3] stat -w /clickhouse/tables/shard_2/aggregated_apps_dpi_app_summary_7f1396c5_a4d8_48c9_a3f3_90271305c114/replicas/replica_2/parts cZxid = 0x13000611b0 ctime = Thu Jun 06 13:04:52 UTC 2024 mZxid = 0x13000611b0 mtime = Thu Jun 06 13:04:52 UTC 2024 pZxid = 0x9109c9b371 cversion = 2332178 dataVersion = 0 aclVersion = 0 ephemeralOwner = 0x0 dataLength = 0 numChildren = 26 [zk: 127.0.0.1:2181(CONNECTED) 4] ls /clickhouse/tables/shard_2/aggregated_apps_dpi_app_summary_7f1396c5_a4d8_48c9_a3f3_90271305c114/replicas/replica_2/parts [20250206_0_685_142, 20250206_0_690_143, 20250206_0_695_144, 20250206_0_700_145, 20250206_0_705_146, 20250206_686_686_0, 20250206_687_687_0, 20250206_688_688_0, 20250206_689_689_0, 20250206_690_690_0, 20250206_691_691_0, 20250206_692_692_0, 20250206_693_693_0, 20250206_694_694_0, 20250206_695_695_0, 20250206_696_696_0, 20250206_697_697_0, 20250206_698_698_0, 20250206_699_699_0, 20250206_700_700_0, 20250206_701_701_0, 20250206_702_702_0, 20250206_703_703_0, 20250206_704_704_0, 20250206_705_705_0, 20250206_706_706_0] Follower-1 (No issue seen): vmanage@aacf5ef6555b:/var/lib/zookeeper$ /var/lib/zookeeper/bin/zkServer.sh status ZooKeeper JMX enabled by default Using config: /var/lib/zookeeper/bin/../conf/zoo.cfg Client port found: 2181. Client address: localhost. Client SSL: false. Mode: follower vmanage@aacf5ef6555b:/var/lib/zookeeper$ [zk: 127.0.0.1:2181(CONNECTED) 1] stat -w /clickhouse/tables/shard_2/aggregated_apps_dpi_app_summary_7f1396c5_a4d8_48c9_a3f3_90271305c114/replicas/replica_2/parts cZxid = 0x13000611b0 ctime = Thu Jun 06 13:04:52 UTC 2024 mZxid = 0x13000611b0 mtime = Thu Jun 06 13:04:52 UTC 2024 pZxid = 0x9109c98fb2 cversion = 2332172 dataVersion = 0 aclVersion = 0 ephemeralOwner = 0x0 dataLength = 0 numChildren = 22 [zk: 127.0.0.1:2181(CONNECTED) 2] ls /clickhouse/tables/shard_2/aggregated_apps_dpi_app_summary_7f1396c5_a4d8_48c9_a3f3_90271305c114/replicas/replica_2/parts [20250206_0_685_142, 20250206_0_690_143, 20250206_0_695_144, 20250206_0_700_145, 20250206_686_686_0, 20250206_687_687_0, 20250206_688_688_0, 20250206_689_689_0, 20250206_690_690_0, 20250206_691_691_0, 20250206_692_692_0, 20250206_693_693_0, 20250206_694_694_0, 20250206_695_695_0, 20250206_696_696_0, 20250206_697_697_0, 20250206_698_698_0, 20250206_699_699_0, 20250206_700_700_0, 20250206_701_701_0, 20250206_702_702_0, 20250206_703_703_0] [zk: 127.0.0.1:2181(CONNECTED) 3] stat -w /clickhouse/tables/shard_2/aggregated_apps_dpi_app_summary_7f1396c5_a4d8_48c9_a3f3_90271305c114/replicas/replica_2/parts cZxid = 0x13000611b0 ctime = Thu Jun 06 13:04:52 UTC 2024 mZxid = 0x13000611b0 mtime = Thu Jun 06 13:04:52 UTC 2024 pZxid = 0x9109c9b371 cversion = 2332178 dataVersion = 0 aclVersion = 0 ephemeralOwner = 0x0 dataLength = 0 numChildren = 26 [zk: 127.0.0.1:2181(CONNECTED) 4] ls /clickhouse/tables/shard_2/aggregated_apps_dpi_app_summary_7f1396c5_a4d8_48c9_a3f3_90271305c114/replicas/replica_2/parts [20250206_0_685_142, 20250206_0_690_143, 20250206_0_695_144, 20250206_0_700_145, 20250206_0_705_146, 20250206_686_686_0, 20250206_687_687_0, 20250206_688_688_0, 20250206_689_689_0, 20250206_690_690_0, 20250206_691_691_0, 20250206_692_692_0, 20250206_693_693_0, 20250206_694_694_0, 20250206_695_695_0, 20250206_696_696_0, 20250206_697_697_0, 20250206_698_698_0, 20250206_699_699_0, 20250206_700_700_0, 20250206_701_701_0, 20250206_702_702_0, 20250206_703_703_0, 20250206_704_704_0, 20250206_705_705_0, 20250206_706_706_0] Follower-2 (Issue seen): vmanage@fc890e903696:/var/lib/zookeeper$ /var/lib/zookeeper/bin/zkServer.sh status ZooKeeper JMX enabled by default Using config: /var/lib/zookeeper/bin/../conf/zoo.cfg Client port found: 2181. Client address: localhost. Client SSL: false. Mode: follower vmanage@fc890e903696:/var/lib/zookeeper$ [zk: 127.0.0.1:2181(CONNECTED) 1] stat -w /clickhouse/tables/shard_2/aggregated_apps_dpi_app_summary_7f1396c5_a4d8_48c9_a3f3_90271305c114/replicas/replica_2/parts cZxid = 0x13000611b0 ctime = Thu Jun 06 13:04:52 UTC 2024 mZxid = 0x13000611b0 mtime = Thu Jun 06 13:04:52 UTC 2024 pZxid = 0x9109c98fb2 cversion = 2332168 dataVersion = 0 aclVersion = 0 ephemeralOwner = 0x0 dataLength = 0 numChildren = 26 [zk: 127.0.0.1:2181(CONNECTED) 2] [zk: 127.0.0.1:2181(CONNECTED) 2] ls /clickhouse/tables/shard_2/aggregated_apps_dpi_app_summary_7f1396c5_a4d8_48c9_a3f3_90271305c114/replicas/replica_2/parts [20250130_0_4587_333, 20250130_0_4592_334, 20250130_4443_4565_11, 20250130_4560_4560_0, 20250206_0_685_142, 20250206_0_690_143, 20250206_0_695_144, 20250206_0_700_145, 20250206_686_686_0, 20250206_687_687_0, 20250206_688_688_0, 20250206_689_689_0, 20250206_690_690_0, 20250206_691_691_0, 20250206_692_692_0, 20250206_693_693_0, 20250206_694_694_0, 20250206_695_695_0, 20250206_696_696_0, 20250206_697_697_0, 20250206_698_698_0, 20250206_699_699_0, 20250206_700_700_0, 20250206_701_701_0, 20250206_702_702_0, 20250206_703_703_0] [zk: 127.0.0.1:2181(CONNECTED) 3] stat -w /clickhouse/tables/shard_2/aggregated_apps_dpi_app_summary_7f1396c5_a4d8_48c9_a3f3_90271305c114/replicas/replica_2/parts cZxid = 0x13000611b0 ctime = Thu Jun 06 13:04:52 UTC 2024 mZxid = 0x13000611b0 mtime = Thu Jun 06 13:04:52 UTC 2024 pZxid = 0x9109c9b371 cversion = 2332174 dataVersion = 0 aclVersion = 0 ephemeralOwner = 0x0 dataLength = 0 numChildren = 30 [zk: 127.0.0.1:2181(CONNECTED) 4] [zk: 127.0.0.1:2181(CONNECTED) 4] ls /clickhouse/tables/shard_2/aggregated_apps_dpi_app_summary_7f1396c5_a4d8_48c9_a3f3_90271305c114/replicas/replica_2/parts [20250130_0_4587_333, 20250130_0_4592_334, 20250130_4443_4565_11, 20250130_4560_4560_0, 20250206_0_685_142, 20250206_0_690_143, 20250206_0_695_144, 20250206_0_700_145, 20250206_0_705_146, 20250206_686_686_0, 20250206_687_687_0, 20250206_688_688_0, 20250206_689_689_0, 20250206_690_690_0, 20250206_691_691_0, 20250206_692_692_0, 20250206_693_693_0, 20250206_694_694_0, 20250206_695_695_0, 20250206_696_696_0, 20250206_697_697_0, 20250206_698_698_0, 20250206_699_699_0, 20250206_700_700_0, 20250206_701_701_0, 20250206_702_702_0, 20250206_703_703_0, 20250206_704_704_0, 20250206_705_705_0, 20250206_706_706_0] [zk: 127.0.0.1:2181(CONNECTED) 5] delete /clickhouse/tables/shard_2/aggregated_apps_dpi_app_summary_7f1396c5_a4d8_48c9_a3f3_90271305c114/replicas/replica_2/parts/20250130_0_4587_333 Node does not exist: /clickhouse/tables/shard_2/aggregated_apps_dpi_app_summary_7f1396c5_a4d8_48c9_a3f3_90271305c114/replicas/replica_2/parts/20250130_0_4587_333 Thanks, Rounak
