Elasticsearch long GC -
running 1.5.2
per server specs: 2x8 (hyperthreaded) 32 cores, 128gb ram, 5.2tb ssds, windows 2008
running 4 node cluster elasticsearch default settings except noted.
#since day 1 configured use 30gb, using doc values expect quite bit of sorting. es_heap_size=30gb #not starting multiple nodes on same machine forgot turn off. cluster.routing.allocation.same_shard.host: true discovery.zen.minimum_master_nodes: 3 discovery.zen.ping.timeout: 6s discovery.zen.ping.multicast.enabled: true #configured below setting allow better bulking. looked @ marvel stats, 512 bulk pool configured since bulking multiple indexes @ same time no longer case. isn't needed reduce doubt it's root cause. bulking single index per bulk request. { "persistent": { "threadpool": { "bulk": { "size": "512", "queue_size": "512" } }, "indices": { "store": { "throttle": { "max_bytes_per_sec": "200mb" } } } }, "transient": { "cluster": { "routing": { "allocation": { "cluster_concurrent_rebalance": "8", "node_concurrent_recoveries": "8", "node_initial_primaries_recoveries": "16", "enable": "all" } } }, "indices": { "recovery": { "concurrent_streams": "9", "max_bytes_per_sec": "80mb" } } } } index refresh rate set 30s , translog threashold 1000mb with above settings able bulk index 3500 documents/sec 2.5 days straight reached 400 million documents verry stable cluster , running searches. not using field data, configured doc_values though expect quite bit of sorting in searches.
at time of long gc, doing bulk, no specific searches, nothing else, except marvel open in browser. away not testing searches.
field data size 20mb per node , 150mb filter cache per node.
then hapenned on single node:
[2015-05-14 10:21:43,607][warn ][monitor.jvm ] [xxx.xxx.xxx.xxx] [gc][old][301536][3170] duration [6.2m], collections [4]/[6.2m], total [6.2m]/[2.1h], memory [28.2gb]->[28.3gb]/[29.8gb], all_pools {[young] [480.3mb]->[406.4mb]/[1.4gb]}{[survivor] [0b]->[191.3mb]/[191.3mb]}{[old] [27.7gb]->[27.8gb]/[28.1gb]} now cluster in yellow state , balancing shards.
the latest vm stats:
{ "cluster_name": "xxxx", "nodes": { "ryl4xn03tnuxfkkfugm3fq": { "timestamp": 1431626086373, "name": "xxxx (10.0.0.xxxx)", "transport_address": "inet[/10.0.0.xxxx:9300]", "host": "xxxx", "ip": [ "inet[/10.0.0.xxxx:9300]", "none" ], "jvm": { "timestamp": 1431626085671, "uptime_in_millis": 326261959, "mem": { "heap_used_in_bytes": 27794721128, "heap_used_percent": 86, "heap_committed_in_bytes": 32011649024, "heap_max_in_bytes": 32011649024, "non_heap_used_in_bytes": 129160952, "non_heap_committed_in_bytes": 131510272, "pools": { "young": { "used_in_bytes": 1285037736, "max_in_bytes": 1605304320, "peak_used_in_bytes": 1605304320, "peak_max_in_bytes": 1605304320 }, "survivor": { "used_in_bytes": 6206720, "max_in_bytes": 200605696, "peak_used_in_bytes": 200605696, "peak_max_in_bytes": 200605696 }, "old": { "used_in_bytes": 26503476672, "max_in_bytes": 30205739008, "peak_used_in_bytes": 30202301920, "peak_max_in_bytes": 30205739008 } } }, "threads": { "count": 872, "peak_count": 4152 }, "gc": { "collectors": { "young": { "collection_count": 133741, "collection_time_in_millis": 14345888 }, "old": { "collection_count": 3801, "collection_time_in_millis": 5671656 } } }, "buffer_pools": { "direct": { "count": 2270, "used_in_bytes": 135625370, "total_capacity_in_bytes": 135625370 }, "mapped": { "count": 28033, "used_in_bytes": 1222266447033, "total_capacity_in_bytes": 1222266447033 } } } }, "gigmbll-rso0cieirol19q": { "timestamp": 1431626086313, "name": "xxxx (10.0.0.xxxx)", "transport_address": "inet[/10.0.0.xxxx:9300]", "host": "xxxx", "ip": [ "inet[/10.0.0.xxxx:9300]", "none" ], "jvm": { "timestamp": 1431626086313, "uptime_in_millis": 326247220, "mem": { "heap_used_in_bytes": 26759522704, "heap_used_percent": 83, "heap_committed_in_bytes": 32011649024, "heap_max_in_bytes": 32011649024, "non_heap_used_in_bytes": 130322080, "non_heap_committed_in_bytes": 132616192, "pools": { "young": { "used_in_bytes": 207756720, "max_in_bytes": 1605304320, "peak_used_in_bytes": 1605304320, "peak_max_in_bytes": 1605304320 }, "survivor": { "used_in_bytes": 34069744, "max_in_bytes": 200605696, "peak_used_in_bytes": 200605696, "peak_max_in_bytes": 200605696 }, "old": { "used_in_bytes": 26517696240, "max_in_bytes": 30205739008, "peak_used_in_bytes": 30201611848, "peak_max_in_bytes": 30205739008 } } }, "threads": { "count": 884, "peak_count": 2855 }, "gc": { "collectors": { "young": { "collection_count": 132317, "collection_time_in_millis": 14532604 }, "old": { "collection_count": 3728, "collection_time_in_millis": 6390133 } } }, "buffer_pools": { "direct": { "count": 2301, "used_in_bytes": 136604011, "total_capacity_in_bytes": 136604011 }, "mapped": { "count": 28077, "used_in_bytes": 1226500757758, "total_capacity_in_bytes": 1226500757758 } } } }, "jncxayftsv20iqamgpbm2q": { "timestamp": 1431626086599, "name": "xxxx (10.0.0.xxxx)", "transport_address": "inet[xxxx/10.0.0.xxxx:9300]", "host": "xxxx", "ip": [ "inet[xxxx/10.0.0.xxxx:9300]", "none" ], "jvm": { "timestamp": 1431626086599, "uptime_in_millis": 326274817, "mem": { "heap_used_in_bytes": 12784591992, "heap_used_percent": 39, "heap_committed_in_bytes": 32011649024, "heap_max_in_bytes": 32011649024, "non_heap_used_in_bytes": 133812080, "non_heap_committed_in_bytes": 135835648, "pools": { "young": { "used_in_bytes": 398354208, "max_in_bytes": 1605304320, "peak_used_in_bytes": 1605304320, "peak_max_in_bytes": 1605304320 }, "survivor": { "used_in_bytes": 84894616, "max_in_bytes": 200605696, "peak_used_in_bytes": 200605696, "peak_max_in_bytes": 200605696 }, "old": { "used_in_bytes": 12301347344, "max_in_bytes": 30205739008, "peak_used_in_bytes": 30200356360, "peak_max_in_bytes": 30205739008 } } }, "threads": { "count": 897, "peak_count": 2319 }, "gc": { "collectors": { "young": { "collection_count": 132882, "collection_time_in_millis": 14076027 }, "old": { "collection_count": 3175, "collection_time_in_millis": 7975099 } } }, "buffer_pools": { "direct": { "count": 3094, "used_in_bytes": 165837901, "total_capacity_in_bytes": 165837901 }, "mapped": { "count": 16336, "used_in_bytes": 716955739746, "total_capacity_in_bytes": 716955739746 } } } }, "bxb5ek48tted87wom3wqxg": { "timestamp": 1431626086497, "name": "xxxx (10.0.0.xxxx)", "transport_address": "inet[/10.0.0.xxxx:9300]", "host": "xxxx", "ip": [ "inet[/10.0.0.xxxx:9300]", "none" ], "jvm": { "timestamp": 1431626086497, "uptime_in_millis": 326255802, "mem": { "heap_used_in_bytes": 26938475512, "heap_used_percent": 84, "heap_committed_in_bytes": 32011649024, "heap_max_in_bytes": 32011649024, "non_heap_used_in_bytes": 137921040, "non_heap_committed_in_bytes": 139923456, "pools": { "young": { "used_in_bytes": 1469836080, "max_in_bytes": 1605304320, "peak_used_in_bytes": 1605304320, "peak_max_in_bytes": 1605304320 }, "survivor": { "used_in_bytes": 34356752, "max_in_bytes": 200605696, "peak_used_in_bytes": 200605696, "peak_max_in_bytes": 200605696 }, "old": { "used_in_bytes": 25434282680, "max_in_bytes": 30205739008, "peak_used_in_bytes": 30201487944, "peak_max_in_bytes": 30205739008 } } }, "threads": { "count": 909, "peak_count": 1414 }, "gc": { "collectors": { "young": { "collection_count": 128613, "collection_time_in_millis": 13985910 }, "old": { "collection_count": 3582, "collection_time_in_millis": 6905669 } } }, "buffer_pools": { "direct": { "count": 2419, "used_in_bytes": 141681639, "total_capacity_in_bytes": 141681639 }, "mapped": { "count": 27785, "used_in_bytes": 1220079289372, "total_capacity_in_bytes": 1220079289372 } } } } } }
Comments
Post a Comment