Filippo Giunchedi has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/344340 )
Change subject: prometheus: keep more chunks in memory ...................................................................... prometheus: keep more chunks in memory eqiad and codfw have a substantial amount of metrics (~600k) and about 15k samples/s ingested. Tune Prometheus to keep more chunks in memory by default to avoid having to throttle sample ingestion. See also https://prometheus.io/docs/operating/storage/ Bug: T148408 Change-Id: I503c4a8e1eecded54505a9177874350206876021 --- M hieradata/role/codfw/prometheus/ops.yaml M hieradata/role/eqiad/prometheus/ops.yaml M modules/prometheus/manifests/server.pp M modules/prometheus/templates/initscripts/[email protected] M modules/role/manifests/prometheus/ops.pp 5 files changed, 16 insertions(+), 6 deletions(-) Approvals: jenkins-bot: Verified Filippo Giunchedi: Looks good to me, approved diff --git a/hieradata/role/codfw/prometheus/ops.yaml b/hieradata/role/codfw/prometheus/ops.yaml index 4fd4e98..b451f1a 100644 --- a/hieradata/role/codfw/prometheus/ops.yaml +++ b/hieradata/role/codfw/prometheus/ops.yaml @@ -1,2 +1,4 @@ +prometheus::server::memory_chunks: '3145728' +prometheus::server::max_chunks_to_persist: '1048576' lvs::realserver::realserver_ips: - 10.2.1.25 diff --git a/hieradata/role/eqiad/prometheus/ops.yaml b/hieradata/role/eqiad/prometheus/ops.yaml index d24a2fb..2623e9f 100644 --- a/hieradata/role/eqiad/prometheus/ops.yaml +++ b/hieradata/role/eqiad/prometheus/ops.yaml @@ -1,3 +1,5 @@ prometheus::server::storage_retention: '960h0m0s' +prometheus::server::memory_chunks: '3145728' +prometheus::server::max_chunks_to_persist: '1048576' lvs::realserver::realserver_ips: - 10.2.2.25 diff --git a/modules/prometheus/manifests/server.pp b/modules/prometheus/manifests/server.pp index 8e5cc29..455384d 100644 --- a/modules/prometheus/manifests/server.pp +++ b/modules/prometheus/manifests/server.pp @@ -51,6 +51,8 @@ $base_path = "/srv/prometheus/${title}", $storage_retention = '730h0m0s', $storage_encoding = '1', + $max_chunks_to_persist = '524288', + $memory_chunks = '1048576', $global_config_extra = {}, $scrape_configs_extra = [], $rule_files_extra = [], diff --git a/modules/prometheus/templates/initscripts/[email protected] b/modules/prometheus/templates/initscripts/[email protected] index cc8e479..c1bf51c 100644 --- a/modules/prometheus/templates/initscripts/[email protected] +++ b/modules/prometheus/templates/initscripts/[email protected] @@ -3,7 +3,7 @@ [Service] User=prometheus -ExecStart=/usr/bin/prometheus -storage.local.path <%= @metrics_path %> -web.listen-address <%= @listen_address %> -web.external-url <%= @external_url %> -storage.local.retention <%= @storage_retention %> -config.file <%= @base_path %>/prometheus.yml -storage.local.chunk-encoding-version <%= @storage_encoding %> +ExecStart=/usr/bin/prometheus -storage.local.max-chunks-to-persist <%= @max_chunks_to_persist %> -storage.local.memory-chunks <%= @memory_chunks %> -storage.local.path <%= @metrics_path %> -web.listen-address <%= @listen_address %> -web.external-url <%= @external_url %> -storage.local.retention <%= @storage_retention %> -config.file <%= @base_path %>/prometheus.yml -storage.local.chunk-encoding-version <%= @storage_encoding %> Restart=on-failure RestartSec=2s TimeoutStopSec=10m diff --git a/modules/role/manifests/prometheus/ops.pp b/modules/role/manifests/prometheus/ops.pp index 213b0b5..7a2d520 100644 --- a/modules/role/manifests/prometheus/ops.pp +++ b/modules/role/manifests/prometheus/ops.pp @@ -7,6 +7,8 @@ $targets_path = '/srv/prometheus/ops/targets' $storage_retention = hiera('prometheus::server::storage_retention', '2190h0m0s') + $max_chunks_to_persist = hiera('prometheus::server::max_chunks_to_persist', '524288') + $memory_chunks = hiera('prometheus::server::memory_chunks', '1048576') $config_extra = { # All metrics will get an additional 'site' label when queried by @@ -268,14 +270,16 @@ } prometheus::server { 'ops': - storage_encoding => '2', - listen_address => '127.0.0.1:9900', - storage_retention => $storage_retention, - scrape_configs_extra => array_concat( + storage_encoding => '2', + listen_address => '127.0.0.1:9900', + storage_retention => $storage_retention, + max_chunks_to_persist => $max_chunks_to_persist, + memory_chunks => $memory_chunks, + scrape_configs_extra => array_concat( $mysql_jobs, $varnish_jobs, $memcached_jobs, $hhvm_jobs, $apache_jobs, $etcd_jobs, $etcdmirror_jobs ), - global_config_extra => $config_extra, + global_config_extra => $config_extra, } prometheus::web { 'ops': -- To view, visit https://gerrit.wikimedia.org/r/344340 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I503c4a8e1eecded54505a9177874350206876021 Gerrit-PatchSet: 2 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Filippo Giunchedi <[email protected]> Gerrit-Reviewer: Filippo Giunchedi <[email protected]> Gerrit-Reviewer: Giuseppe Lavagetto <[email protected]> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
