Hi, I've performed a delete by query and the number of deleted_docs and index size_in_bytes does not decrease. I've also looked through the ES mailing group and tried the recommended optimize, but I'm still not seeing expected results. Here is my test case:
*curl http://localhost:9200/apacheal/_stats/ <http://localhost:9200/apacheal/_stats/>* {"ok":true,"_shards":{"total":10,"successful":5,"failed":0},"_all":{"primaries":{"docs":{"count": *106407607*,"deleted":*809677*},"store":{"size":"45.5gb","size_in_bytes": *48894062436* ,"throttle_time":"7.8h","throttle_time_in_millis":28246172},"indexing":{"index_total":56322858,"index_time":"8.4h","index_time_in_millis":30412645,"index_current":0,"delete_total":0,"delete_time":"0s","delete_time_in_millis":0,"delete_current":0},"get":{"total":0,"get_time":"0s","time_in_millis":0,"exists_total":0,"exists_time":"0s","exists_time_in_millis":0,"missing_total":0,"missing_time":"0s","missing_time_in_millis":0,"current":0}," search ":{"open_contexts":0,"query_total":2020,"query_time":"2.5h","query_time_in_millis":9351789,"query_current":0,"fetch_total":1536,"fetch_time":"4.4m","fetch_time_in_millis":267579,"fetch_current":0}},"total":{"docs":{"count":106407607,"deleted":809677},"store":{"size":"45.5gb","size_in_bytes":48894062436,"throttle_time":"7.8h","throttle_time_in_millis":28246172},"indexing":{"index_total":56322858,"index_time":"8.4h","index_time_in_millis":30412645,"index_current":0,"delete_total":0,"delete_time":"0s","delete_time_in_millis":0,"delete_current":0},"get":{"total":0,"get_time":"0s","time_in_millis":0,"exists_total":0,"exists_time":"0s","exists_time_in_millis":0,"missing_total":0,"missing_time":"0s","missing_time_in_millis":0,"current":0}," search ":{"open_contexts":0,"query_total":2020,"query_time":"2.5h","query_time_in_millis":9351789,"query_current":0,"fetch_total":1536,"fetch_time":"4.4m","fetch_time_in_millis":267579,"fetch_current":0}}},"indices":{"apacheal":{"primaries":{"docs":{"count":106407607,"deleted":809677},"store":{"size":"45.5gb","size_in_bytes":48894062436,"throttle_time":"7.8h","throttle_time_in_millis":28246172},"indexing":{"index_total":56322858,"index_time":"8.4h","index_time_in_millis":30412645,"index_current":0,"delete_total":0,"delete_time":"0s","delete_time_in_millis":0,"delete_current":0},"get":{"total":0,"get_time":"0s","time_in_millis":0,"exists_total":0,"exists_time":"0s","exists_time_in_millis":0,"missing_total":0,"missing_time":"0s","missing_time_in_millis":0,"current":0}," search ":{"open_contexts":0,"query_total":2020,"query_time":"2.5h","query_time_in_millis":9351789,"query_current":0,"fetch_total":1536,"fetch_time":"4.4m","fetch_time_in_millis":267579,"fetch_current":0}},"total":{"docs":{"count":106407607,"deleted":809677},"store":{"size":"45.5gb","size_in_bytes":48894062436,"throttle_time":"7.8h","throttle_time_in_millis":28246172},"indexing":{"index_total":56322858,"index_time":"8.4h","index_time_in_millis":30412645,"index_current":0,"delete_total":0,"delete_time":"0s","delete_time_in_millis":0,"delete_current":0},"get":{"total":0,"get_time":"0s","time_in_millis":0,"exists_total":0,"exists_time":"0s","exists_time_in_millis":0,"missing_total":0,"missing_time":"0s","missing_time_in_millis":0,"current":0}," search ":{"open_contexts":0,"query_total":2020,"query_time":"2.5h","query_time_in_millis":9351789,"query_current":0,"fetch_total":1536,"fetch_time":"4.4m","fetch_time_in_millis":267579,"fetch_current":0}}}}} *curl -XDELETE http://localhost:9200/apacheal/events/_query <http://localhost:9200/apacheal/events/_query> -d "{ \"range\":{\"timestamp\":{\"lte\":\"2014-02-12T23:59:59-05:00\"}} }"* {"ok":true,"_indices":{"apacheal":{"_shards":{"total":5,"successful":5,"failed":0}}}} *curl http:// <http://logs.flyptech.com:9200/apacheal/_stats/>localhost <http://localhost:9200/apacheal/_stats/>:9200/apacheal/_stats/* {"ok":true,"_shards":{"total":10,"successful":5,"failed":0},"_all":{"primaries":{"docs":{"count": *100703300*,"deleted":*4961760*},"store":{"size":"45.5gb","size_in_bytes": *48896498586* ,"throttle_time":"7.8h","throttle_time_in_millis":28246326},"indexing":{"index_total":56324061,"index_time":"8.4h","index_time_in_millis":30413483,"index_current":0,"delete_total":0,"delete_time":"0s","delete_time_in_millis":0,"delete_current":0},"get":{"total":0,"get_time":"0s","time_in_millis":0,"exists_total":0,"exists_time":"0s","exists_time_in_millis":0,"missing_total":0,"missing_time":"0s","missing_time_in_millis":0,"current":0}," search ":{"open_contexts":0,"query_total":2020,"query_time":"2.5h","query_time_in_millis":9351789,"query_current":0,"fetch_total":1536,"fetch_time":"4.4m","fetch_time_in_millis":267579,"fetch_current":0}},"total":{"docs":{"count":100703300,"deleted":4961760},"store":{"size":"45.5gb","size_in_bytes":48896498586,"throttle_time":"7.8h","throttle_time_in_millis":28246326},"indexing":{"index_total":56324061,"index_time":"8.4h","index_time_in_millis":30413483,"index_current":0,"delete_total":0,"delete_time":"0s","delete_time_in_millis":0,"delete_current":0},"get":{"total":0,"get_time":"0s","time_in_millis":0,"exists_total":0,"exists_time":"0s","exists_time_in_millis":0,"missing_total":0,"missing_time":"0s","missing_time_in_millis":0,"current":0}," search ":{"open_contexts":0,"query_total":2020,"query_time":"2.5h","query_time_in_millis":9351789,"query_current":0,"fetch_total":1536,"fetch_time":"4.4m","fetch_time_in_millis":267579,"fetch_current":0}}},"indices":{"apacheal":{"primaries":{"docs":{"count":100703300,"deleted":4961760},"store":{"size":"45.5gb","size_in_bytes":48896498586,"throttle_time":"7.8h","throttle_time_in_millis":28246326},"indexing":{"index_total":56324061,"index_time":"8.4h","index_time_in_millis":30413483,"index_current":0,"delete_total":0,"delete_time":"0s","delete_time_in_millis":0,"delete_current":0},"get":{"total":0,"get_time":"0s","time_in_millis":0,"exists_total":0,"exists_time":"0s","exists_time_in_millis":0,"missing_total":0,"missing_time":"0s","missing_time_in_millis":0,"current":0}," search ":{"open_contexts":0,"query_total":2020,"query_time":"2.5h","query_time_in_millis":9351789,"query_current":0,"fetch_total":1536,"fetch_time":"4.4m","fetch_time_in_millis":267579,"fetch_current":0}},"total":{"docs":{"count":100703300,"deleted":4961760},"store":{"size":"45.5gb","size_in_bytes":48896498586,"throttle_time":"7.8h","throttle_time_in_millis":28246326},"indexing":{"index_total":56324061,"index_time":"8.4h","index_time_in_millis":30413483,"index_current":0,"delete_total":0,"delete_time":"0s","delete_time_in_millis":0,"delete_current":0},"get":{"total":0,"get_time":"0s","time_in_millis":0,"exists_total":0,"exists_time":"0s","exists_time_in_millis":0,"missing_total":0,"missing_time":"0s","missing_time_in_millis":0,"current":0}," search ":{"open_contexts":0,"query_total":2020,"query_time":"2.5h","query_time_in_millis":9351789,"query_current":0,"fetch_total":1536,"fetch_time":"4.4m","fetch_time_in_millis":267579,"fetch_current":0}}}}} *RAN OPTIMIZE THROUGH UI* *curl http:// <http://logs.flyptech.com:9200/apacheal/_stats/>**localhost <http://localhost:9200/apacheal/_stats/>**:9200/apacheal/_stats/ <http://logs.flyptech.com:9200/apacheal/_stats/>* {"ok":true,"_shards":{"total":10,"successful":5,"failed":0},"_all":{"primaries":{"docs":{"count": *100705106*,"deleted":*4961760*},"store":{"size":"44.8gb","size_in_bytes": *48159890359* ,"throttle_time":"7.8h","throttle_time_in_millis":28246687},"indexing":{"index_total":56325867,"index_time":"8.4h","index_time_in_millis":30417576,"index_current":0,"delete_total":0,"delete_time":"0s","delete_time_in_millis":0,"delete_current":0},"get":{"total":0,"get_time":"0s","time_in_millis":0,"exists_total":0,"exists_time":"0s","exists_time_in_millis":0,"missing_total":0,"missing_time":"0s","missing_time_in_millis":0,"current":0}," search ":{"open_contexts":0,"query_total":2020,"query_time":"2.5h","query_time_in_millis":9351789,"query_current":0,"fetch_total":1536,"fetch_time":"4.4m","fetch_time_in_millis":267579,"fetch_current":0}},"total":{"docs":{"count":100705106,"deleted":4961760},"store":{"size":"44.8gb","size_in_bytes":48159890359,"throttle_time":"7.8h","throttle_time_in_millis":28246687},"indexing":{"index_total":56325867,"index_time":"8.4h","index_time_in_millis":30417576,"index_current":0,"delete_total":0,"delete_time":"0s","delete_time_in_millis":0,"delete_current":0},"get":{"total":0,"get_time":"0s","time_in_millis":0,"exists_total":0,"exists_time":"0s","exists_time_in_millis":0,"missing_total":0,"missing_time":"0s","missing_time_in_millis":0,"current":0}," search ":{"open_contexts":0,"query_total":2020,"query_time":"2.5h","query_time_in_millis":9351789,"query_current":0,"fetch_total":1536,"fetch_time":"4.4m","fetch_time_in_millis":267579,"fetch_current":0}}},"indices":{"apacheal":{"primaries":{"docs":{"count":100705106,"deleted":4961760},"store":{"size":"44.8gb","size_in_bytes":48159890359,"throttle_time":"7.8h","throttle_time_in_millis":28246687},"indexing":{"index_total":56325867,"index_time":"8.4h","index_time_in_millis":30417576,"index_current":0,"delete_total":0,"delete_time":"0s","delete_time_in_millis":0,"delete_current":0},"get":{"total":0,"get_time":"0s","time_in_millis":0,"exists_total":0,"exists_time":"0s","exists_time_in_millis":0,"missing_total":0,"missing_time":"0s","missing_time_in_millis":0,"current":0}," search ":{"open_contexts":0,"query_total":2020,"query_time":"2.5h","query_time_in_millis":9351789,"query_current":0,"fetch_total":1536,"fetch_time":"4.4m","fetch_time_in_millis":267579,"fetch_current":0}},"total":{"docs":{"count":100705106,"deleted":4961760},"store":{"size":"44.8gb","size_in_bytes":48159890359,"throttle_time":"7.8h","throttle_time_in_millis":28246687},"indexing":{"index_total":56325867,"index_time":"8.4h","index_time_in_millis":30417576,"index_current":0,"delete_total":0,"delete_time":"0s","delete_time_in_millis":0,"delete_current":0},"get":{"total":0,"get_time":"0s","time_in_millis":0,"exists_total":0,"exists_time":"0s","exists_time_in_millis":0,"missing_total":0,"missing_time":"0s","missing_time_in_millis":0,"current":0}," search ":{"open_contexts":0,"query_total":2020,"query_time":"2.5h","query_time_in_millis":9351789,"query_current":0,"fetch_total":1536,"fetch_time":"4.4m","fetch_time_in_millis":267579,"fetch_current":0}}}}} When I deleted, the document count went down by ~5.7 million records, which was expected, but the deleted only increase by ~ 4.2 million, which is not exactly expected. The size_in_bytes only decrease by ~ 2.4 MB, and then after the optimize only decrease by ~ 700 MB. This is not what I was expecting at all. Each of the records is about 500 bytes, so 500 bytes * 5.7 million records = 2.65 GB. Why is the deleted documents and size in bytes not decreasing significantly? Thanks for your help, Rob -- You received this message because you are subscribed to the Google Groups "elasticsearch" group. To unsubscribe from this group and stop receiving emails from it, send an email to [email protected]. To view this discussion on the web visit https://groups.google.com/d/msgid/elasticsearch/1bc61dd1-26ff-4e0a-89a7-c5f2ba95cc10%40googlegroups.com. For more options, visit https://groups.google.com/d/optout.
