Hi I testing Elastcisearch in very simple setup:
2 nodes: 1 master, 1 slave (placed on different physical servers)
1 index with no sharding
Project (PHP code) access to elastcisearch cluster via haproxy.
Immediately after initialization node, when node starts to process requests,
there are short period (about 10 seconds), when node process queries very
slow: >1s
normal query time in my tests: <100ms
In this period there are CPU usage peak (java uses up to 600% vs 30% in
normal work)
In this period there are no disk overload (Disk subsystem on server is
enough fast, RAID1: 2 x ssd)
After this period, node process queries fast (<100ms).
In case of using fixed thread pools - behavior has not changed.
State of queries in this period (search2 - slave):
host ip bulk.active bulk.queue bulk.rejected index.active
index.queue index.rejected search.active search.queue search.rejected
search1 172.16.76.21 0 0 0 0
0 0 0 0 0
search2 172.16.76.22 0 0 0 0
0 0 48 75 0
Slow queries:
[2014-07-10 12:59:18,456][TRACE][index.search.slowlog.fetch] [Kafka]
[v_v2][0] took[1s], took_millis[1014], types[default-type], stats[],
search_type[QUERY_AND_FETCH], total_shards[1] ...
[2014-07-10 12:59:18,784][TRACE][index.search.slowlog.fetch] [Kafka]
[v_v2][0] took[1.3s], took_millis[1313], types[default-type], stats[],
search_type[QUERY_AND_FETCH], total_shards[1] ...
My questions:
- Is this well-known behavior of elasticsearch?
- If not - what settings can affect and eliminate this behavior?
Thanks in advance!
P.S.: cluster config in attachment.
--
You received this message because you are subscribed to the Google Groups
"elasticsearch" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
To view this discussion on the web visit
https://groups.google.com/d/msgid/elasticsearch/199c6774-87eb-4eca-acc4-6229fc613949%40googlegroups.com.
For more options, visit https://groups.google.com/d/optout.
{
"cluster_name" : "rabota-es",
"nodes" : {
"5kbU94sqTm-D-QAZFyg81Q" : {
"name" : "ZX Spectrum",
"transport_address" : "inet[/172.16.76.21:9300]",
"host" : "search1",
"ip" : "172.16.76.21",
"version" : "1.1.0",
"build" : "2181e11",
"http_address" : "inet[/172.16.76.21:9200]",
"attributes" : {
"master" : "true"
},
"settings" : {
"node" : {
"data" : "true",
"master" : "true",
"name" : "ZX Spectrum"
},
"index" : {
"number_of_replicas" : "1",
"store" : {
"type" : "mmapfs"
},
"translog" : {
"flush_threshold_ops" : "5000",
"flush_threshold_period" : "30m",
"flush_threshold_size" : "200mb",
"disable_flush" : "false"
},
"search" : {
"slowlog" : {
"threshold" : {
"fetch" : {
"trace" : "1s"
},
"query" : {
"trace" : "1s"
}
}
}
},
"refresh_interval" : "1s"
},
"bootstrap" : {
"mlockall" : "true"
},
"http" : {
"port" : "9200",
"max_content_length" : "10mb"
},
"transport" : {
"tcp" : {
"port" : "9300",
"connect_timeout" : "1s"
}
},
"name" : "ZX Spectrum",
"action" : {
"replication_type" : "sync"
},
"pidfile" : "/var/run/elasticsearch.pid",
"path" : {
"data" : "/data/elasticsearch",
"work" : "/tmp/elasticsearch",
"home" : "/usr/share/elasticsearch",
"conf" : "/etc/elasticsearch",
"logs" : "/data/logs/elasticsearch"
},
"cluster" : {
"name" : "rabota-es"
},
"config" : "/etc/elasticsearch/elasticsearch.yml",
"discovery" : {
"fd" : {
"ping_interval" : "0.5s",
"ping_timeout" : "0.5s",
"ping_retries" : "3"
},
"zen" : {
"minimum_master_nodes" : "1",
"ping" : {
"unicast" : {
"hosts" : [ "172.16.76.21:9300", "172.16.76.22:9300" ]
},
"multicast" : {
"enabled" : "false"
},
"timeout" : "1s"
},
"publish_timeout" : "1s"
}
},
"network" : {
"host" : "172.16.76.21"
}
},
"os" : {
"refresh_interval" : 1000,
"available_processors" : 16,
"cpu" : {
"vendor" : "Intel",
"model" : "Xeon",
"mhz" : 2268,
"total_cores" : 16,
"total_sockets" : 1,
"cores_per_socket" : 16,
"cache_size_in_bytes" : 8192
},
"mem" : {
"total_in_bytes" : 101374992384
},
"swap" : {
"total_in_bytes" : 0
}
},
"process" : {
"refresh_interval" : 1000,
"id" : 3413,
"max_file_descriptors" : 65535,
"mlockall" : false
},
"jvm" : {
"pid" : 3413,
"version" : "1.7.0_51",
"vm_name" : "Java HotSpot(TM) 64-Bit Server VM",
"vm_version" : "24.51-b03",
"vm_vendor" : "Oracle Corporation",
"start_time" : 1404971570323,
"mem" : {
"heap_init_in_bytes" : 268435456,
"heap_max_in_bytes" : 1037959168,
"non_heap_init_in_bytes" : 24313856,
"non_heap_max_in_bytes" : 136314880,
"direct_max_in_bytes" : 1037959168
},
"gc_collectors" : [ "ParNew", "ConcurrentMarkSweep" ],
"memory_pools" : [ "Code Cache", "Par Eden Space", "Par Survivor
Space", "CMS Old Gen", "CMS Perm Gen" ]
},
"thread_pool" : {
"generic" : {
"type" : "cached",
"keep_alive" : "30s"
},
"index" : {
"type" : "fixed",
"min" : 16,
"max" : 16,
"queue_size" : "200"
},
"get" : {
"type" : "fixed",
"min" : 16,
"max" : 16,
"queue_size" : "1k"
},
"snapshot" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m"
},
"merge" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m"
},
"suggest" : {
"type" : "fixed",
"min" : 16,
"max" : 16,
"queue_size" : "1k"
},
"bulk" : {
"type" : "fixed",
"min" : 16,
"max" : 16,
"queue_size" : "50"
},
"optimize" : {
"type" : "fixed",
"min" : 1,
"max" : 1
},
"warmer" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m"
},
"flush" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m"
},
"search" : {
"type" : "fixed",
"min" : 48,
"max" : 48,
"queue_size" : "1k"
},
"percolate" : {
"type" : "fixed",
"min" : 16,
"max" : 16,
"queue_size" : "1k"
},
"management" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m"
},
"refresh" : {
"type" : "scaling",
"min" : 1,
"max" : 8,
"keep_alive" : "5m"
}
},
"network" : {
"refresh_interval" : 5000,
"primary_interface" : {
"address" : "172.16.76.21",
"name" : "eth0",
"mac_address" : "00:25:90:29:7F:3E"
}
},
"transport" : {
"bound_address" : "inet[/172.16.76.21:9300]",
"publish_address" : "inet[/172.16.76.21:9300]"
},
"http" : {
"bound_address" : "inet[/172.16.76.21:9200]",
"publish_address" : "inet[/172.16.76.21:9200]",
"max_content_length_in_bytes" : 10485760
},
"plugins" : [ {
"name" : "HQ",
"version" : "NA",
"description" : "No description found.",
"url" : "/_plugin/HQ/",
"jvm" : false,
"site" : true
}, {
"name" : "paramedic",
"version" : "NA",
"description" : "No description found.",
"url" : "/_plugin/paramedic/",
"jvm" : false,
"site" : true
} ]
},
"kt4KhVr4Ty6kDSyHx_sDGg" : {
"name" : "Kafka",
"transport_address" : "inet[/172.16.76.22:9300]",
"host" : "search2",
"ip" : "172.16.76.22",
"version" : "1.1.0",
"build" : "2181e11",
"http_address" : "inet[/172.16.76.22:9200]",
"attributes" : {
"master" : "true"
},
"settings" : {
"node" : {
"data" : "true",
"master" : "true",
"name" : "Kafka"
},
"index" : {
"number_of_replicas" : "1",
"store" : {
"type" : "mmapfs"
},
"translog" : {
"flush_threshold_ops" : "5000",
"flush_threshold_period" : "30m",
"flush_threshold_size" : "200mb",
"disable_flush" : "false"
},
"search" : {
"slowlog" : {
"threshold" : {
"fetch" : {
"trace" : "1s"
},
"query" : {
"trace" : "1s"
}
}
}
},
"refresh_interval" : "1s"
},
"bootstrap" : {
"mlockall" : "true"
},
"http" : {
"port" : "9200",
"max_content_length" : "10mb"
},
"transport" : {
"tcp" : {
"port" : "9300",
"connect_timeout" : "1s"
}
},
"name" : "Kafka",
"action" : {
"replication_type" : "sync"
},
"pidfile" : "/var/run/elasticsearch.pid",
"path" : {
"data" : "/data/elasticsearch",
"work" : "/tmp/elasticsearch",
"home" : "/usr/share/elasticsearch",
"conf" : "/etc/elasticsearch",
"logs" : "/data/logs/elasticsearch"
},
"cluster" : {
"name" : "rabota-es"
},
"config" : "/etc/elasticsearch/elasticsearch.yml",
"discovery" : {
"fd" : {
"ping_interval" : "0.5s",
"ping_timeout" : "0.5s",
"ping_retries" : "3"
},
"zen" : {
"minimum_master_nodes" : "1",
"ping" : {
"unicast" : {
"hosts" : [ "172.16.76.21:9300", "172.16.76.22:9300" ]
},
"multicast" : {
"enabled" : "false"
},
"timeout" : "1s"
},
"publish_timeout" : "1s"
}
},
"network" : {
"host" : "172.16.76.22"
}
},
"os" : {
"refresh_interval" : 1000,
"available_processors" : 16,
"cpu" : {
"vendor" : "Intel",
"model" : "Xeon",
"mhz" : 2268,
"total_cores" : 16,
"total_sockets" : 1,
"cores_per_socket" : 16,
"cache_size_in_bytes" : 8192
},
"mem" : {
"total_in_bytes" : 101374992384
},
"swap" : {
"total_in_bytes" : 0
}
},
"process" : {
"refresh_interval" : 1000,
"id" : 26314,
"max_file_descriptors" : 65535,
"mlockall" : true
},
"jvm" : {
"pid" : 26314,
"version" : "1.7.0_51",
"vm_name" : "Java HotSpot(TM) 64-Bit Server VM",
"vm_version" : "24.51-b03",
"vm_vendor" : "Oracle Corporation",
"start_time" : 1404971815991,
"mem" : {
"heap_init_in_bytes" : 4294967296,
"heap_max_in_bytes" : 4181590016,
"non_heap_init_in_bytes" : 24313856,
"non_heap_max_in_bytes" : 136314880,
"direct_max_in_bytes" : 4294967296
},
"gc_collectors" : [ "ParNew", "ConcurrentMarkSweep" ],
"memory_pools" : [ "Code Cache", "Par Eden Space", "Par Survivor
Space", "CMS Old Gen", "CMS Perm Gen" ]
},
"thread_pool" : {
"generic" : {
"type" : "cached",
"keep_alive" : "30s"
},
"index" : {
"type" : "fixed",
"min" : 16,
"max" : 16,
"queue_size" : "200"
},
"get" : {
"type" : "fixed",
"min" : 16,
"max" : 16,
"queue_size" : "1k"
},
"snapshot" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m"
},
"merge" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m"
},
"suggest" : {
"type" : "fixed",
"min" : 16,
"max" : 16,
"queue_size" : "1k"
},
"bulk" : {
"type" : "fixed",
"min" : 16,
"max" : 16,
"queue_size" : "50"
},
"optimize" : {
"type" : "fixed",
"min" : 1,
"max" : 1
},
"warmer" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m"
},
"flush" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m"
},
"search" : {
"type" : "fixed",
"min" : 48,
"max" : 48,
"queue_size" : "1k"
},
"percolate" : {
"type" : "fixed",
"min" : 16,
"max" : 16,
"queue_size" : "1k"
},
"management" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m"
},
"refresh" : {
"type" : "scaling",
"min" : 1,
"max" : 8,
"keep_alive" : "5m"
}
},
"network" : {
"refresh_interval" : 5000,
"primary_interface" : {
"address" : "172.16.76.22",
"name" : "eth0",
"mac_address" : "00:25:90:29:7F:36"
}
},
"transport" : {
"bound_address" : "inet[/172.16.76.22:9300]",
"publish_address" : "inet[/172.16.76.22:9300]"
},
"http" : {
"bound_address" : "inet[/172.16.76.22:9200]",
"publish_address" : "inet[/172.16.76.22:9200]",
"max_content_length_in_bytes" : 10485760
},
"plugins" : [ {
"name" : "paramedic",
"version" : "NA",
"description" : "No description found.",
"url" : "/_plugin/paramedic/",
"jvm" : false,
"site" : true
}, {
"name" : "HQ",
"version" : "NA",
"description" : "No description found.",
"url" : "/_plugin/HQ/",
"jvm" : false,
"site" : true
}, {
"name" : "bigdesk",
"version" : "NA",
"description" : "No description found.",
"url" : "/_plugin/bigdesk/",
"jvm" : false,
"site" : true
} ]
}
}
}
elasticsearch.yml
Description: Binary data
# Run Elasticsearch as this user ID and group ID ES_USER=elasticsearch ES_GROUP=elasticsearch # Heap Size (defaults to 256m min, 1g max) ES_HEAP_SIZE=4g # Heap new generation #ES_HEAP_NEWSIZE= # max direct memory ES_DIRECT_SIZE=4g # Maximum number of open files, defaults to 65535. MAX_OPEN_FILES=65535 # Maximum locked memory size. Set to "unlimited" if you use the # bootstrap.mlockall option in elasticsearch.yml. You must also set # ES_HEAP_SIZE. MAX_LOCKED_MEMORY=unlimited # Maximum number of VMA (Virtual Memory Areas) a process can own MAX_MAP_COUNT=262144 # Elasticsearch log directory LOG_DIR=/var/log/elasticsearch # Elasticsearch data directory DATA_DIR=/var/lib/elasticsearch # Elasticsearch work directory WORK_DIR=/tmp/elasticsearch # Elasticsearch configuration directory CONF_DIR=/etc/elasticsearch # Elasticsearch configuration file (elasticsearch.yml) CONF_FILE=/etc/elasticsearch/elasticsearch.yml # Additional Java OPTS #ES_JAVA_OPTS= # Configure restart on package upgrade (true, every other setting will lead to not restarting) #RESTART_ON_UPGRADE=true
