AMBARI-21542. AMS fail to start after IOP 4.2 to HDP 2.6.2 upgrade. (swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a7cc3801 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a7cc3801 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a7cc3801 Branch: refs/heads/branch-feature-AMBARI-21450 Commit: a7cc3801a317bf95d6037250ea223e67d54502d2 Parents: ab1d378 Author: Di Li <[email protected]> Authored: Thu Jul 20 20:46:09 2017 -0400 Committer: Siddharth Wagle <[email protected]> Committed: Thu Jul 20 18:48:55 2017 -0700 ---------------------------------------------------------------------- .../configuration/ams-grafana-env.xml | 93 ++++++ .../configuration/ams-grafana-ini.xml | 320 +++++++++++++++++++ .../configuration/ams-hbase-site.xml | 90 +++++- .../AMBARI_METRICS/configuration/ams-site.xml | 215 +++++++++++-- .../configuration/ams-ssl-client.xml | 51 +++ .../configuration/ams-ssl-server.xml | 80 +++++ 6 files changed, 798 insertions(+), 51 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/a7cc3801/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/configuration/ams-grafana-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/configuration/ams-grafana-env.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/configuration/ams-grafana-env.xml new file mode 100644 index 0000000..eaafc6b --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/configuration/ams-grafana-env.xml @@ -0,0 +1,93 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> +<configuration> + <property> + <name>metrics_grafana_log_dir</name> + <value>/var/log/ambari-metrics-grafana</value> + <display-name>Metrics Grafana log dir</display-name> + <description>Metrics Grafana log directory.</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>metrics_grafana_pid_dir</name> + <value>/var/run/ambari-metrics-grafana</value> + <display-name>Metrics Grafana pid dir</display-name> + <description>Metrics Grafana pid directory.</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>metrics_grafana_data_dir</name> + <value>/var/lib/ambari-metrics-grafana</value> + <display-name>Metrics Grafana data dir</display-name> + <description>Metrics Grafana data directory.</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>metrics_grafana_username</name> + <value>admin</value> + <display-name>Grafana Admin Username</display-name> + <description> + Metrics Grafana Username. This value cannot be modified by Ambari + except on initial install. Please make sure the username change in + Grafana is reflected in Ambari. + </description> + <value-attributes> + <type>db_user</type> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property require-input="true"> + <name>metrics_grafana_password</name> + <value/> + <property-type>PASSWORD</property-type> + <display-name>Grafana Admin Password</display-name> + <description> + Metrics Grafana password. This value cannot be modified by Ambari + except on initial install. Please make sure the password change in + Grafana is reflected back in Ambari. + </description> + <value-attributes> + <overridable>false</overridable> + <type>password</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <display-name>ams-grafana-env template</display-name> + <value> +# Set environment variables here. + +# AMS UI Server Home Dir +export AMS_GRAFANA_HOME_DIR={{ams_grafana_home_dir}} + +# AMS UI Server Data Dir +export AMS_GRAFANA_DATA_DIR={{ams_grafana_data_dir}} + +# AMS UI Server Log Dir +export AMS_GRAFANA_LOG_DIR={{ams_grafana_log_dir}} + +# AMS UI Server PID Dir +export AMS_GRAFANA_PID_DIR={{ams_grafana_pid_dir}} + </value> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/a7cc3801/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/configuration/ams-grafana-ini.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/configuration/ams-grafana-ini.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/configuration/ams-grafana-ini.xml new file mode 100644 index 0000000..3c87ab1 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/configuration/ams-grafana-ini.xml @@ -0,0 +1,320 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> +<configuration> + <property> + <name>port</name> + <value>3000</value> + <description>The http port to use</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>protocol</name> + <value>http</value> + <description>Protocol (http or https)</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>cert_file</name> + <value>/etc/ambari-metrics-grafana/conf/ams-grafana.crt</value> + <description>Path to grafana certificate (.crt) file.</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>cert_key</name> + <value>/etc/ambari-metrics-grafana/conf/ams-grafana.key</value> + <description>Path to grafana certificate key (.key) file.</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>ca_cert</name> + <value></value> + <description>Path to CA root certificate or bundle to be used to validate the Grafana certificate against. + For self signed certificates, this value can be the same as the value for 'cert_file'. + (If a path is not specified, the certificate validation is skipped)</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <display-name>ams-grafana-ini template</display-name> + <value> +##################### Grafana Configuration Example ##################### +# +# Everything has defaults so you only need to uncomment things you want to +# change + +# possible values : production, development +; app_mode = production + +#################################### Paths #################################### +[paths] +# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) +# +;data = /var/lib/grafana +data = {{ams_grafana_data_dir}} +# +# Directory where grafana can store logs +# +;logs = /var/log/grafana +logs = {{ams_grafana_log_dir}} + + +#################################### Server #################################### +[server] +# Protocol (http or https) +;protocol = http +protocol = {{ams_grafana_protocol}} +# The ip address to bind to, empty will bind to all interfaces +;http_addr = + +# The http port to use +;http_port = 3000 +http_port = {{ams_grafana_port}} + +# The public facing domain name used to access grafana from a browser +;domain = localhost + +# Redirect to correct domain if host header does not match domain +# Prevents DNS rebinding attacks +;enforce_domain = false + +# The full public facing url +;root_url = %(protocol)s://%(domain)s:%(http_port)s/ + +# Log web requests +;router_logging = false + +# the path relative working path +;static_root_path = public +static_root_path = /usr/lib/ambari-metrics-grafana/public + +# enable gzip +;enable_gzip = false + +# https certs & key file +;cert_file = +;cert_key = +cert_file = {{ams_grafana_cert_file}} +cert_key = {{ams_grafana_cert_key}} + +#################################### Database #################################### +[database] +# Either "mysql", "postgres" or "sqlite3", it's your choice +;type = sqlite3 +;host = 127.0.0.1:3306 +;name = grafana +;user = root +;password = + +# For "postgres" only, either "disable", "require" or "verify-full" +;ssl_mode = disable + +# For "sqlite3" only, path relative to data_path setting +;path = grafana.db + +#################################### Session #################################### +[session] +# Either "memory", "file", "redis", "mysql", "postgres", default is "file" +;provider = file + +# Provider config options +# memory: not have any config yet +# file: session dir path, is relative to grafana data_path +# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=grafana` +# mysql: go-sql-driver/mysql dsn config string, e.g. `user:password@tcp(127.0.0.1:3306)/database_name` +# postgres: user=a password=b host=localhost port=5432 dbname=c sslmode=disable +;provider_config = sessions + +# Session cookie name +;cookie_name = grafana_sess + +# If you use session in https only, default is false +;cookie_secure = false + +# Session life time, default is 86400 +;session_life_time = 86400 + +#################################### Analytics #################################### +[analytics] +# Server reporting, sends usage counters to stats.grafana.org every 24 hours. +# No ip addresses are being tracked, only simple counters to track +# running instances, dashboard and error counts. It is very helpful to us. +# Change this option to false to disable reporting. +;reporting_enabled = true + +# Google Analytics universal tracking code, only enabled if you specify an id here +;google_analytics_ua_id = + +#################################### Security #################################### +[security] +# default admin user, created on startup +admin_user = {{ams_grafana_admin_user}} + +# default admin password, can be changed before first start of grafana, or in profile settings +;admin_password = + +# used for signing +;secret_key = SW2YcwTIb9zpOOhoPsMm + +# Auto-login remember days +;login_remember_days = 7 +;cookie_username = grafana_user +;cookie_remember_name = grafana_remember + +# disable gravatar profile images +;disable_gravatar = false + +# data source proxy whitelist (ip_or_domain:port seperated by spaces) +;data_source_proxy_whitelist = + +#################################### Users #################################### +[users] +# disable user signup / registration +;allow_sign_up = true + +# Allow non admin users to create organizations +;allow_org_create = true + +# Set to true to automatically assign new users to the default organization (id 1) +;auto_assign_org = true + +# Default role new users will be automatically assigned (if disabled above is set to true) +;auto_assign_org_role = Viewer + +# Background text for the user field on the login page +;login_hint = email or username + +#################################### Anonymous Auth ########################## +[auth.anonymous] +# enable anonymous access +enabled = true + +# specify organization name that should be used for unauthenticated users +org_name = Main Org. + +# specify role for unauthenticated users +;org_role = Admin + +#################################### Github Auth ########################## +[auth.github] +;enabled = false +;allow_sign_up = false +;client_id = some_id +;client_secret = some_secret +;scopes = user:email,read:org +;auth_url = https://github.com/login/oauth/authorize +;token_url = https://github.com/login/oauth/access_token +;api_url = https://api.github.com/user +;team_ids = +;allowed_organizations = + +#################################### Google Auth ########################## +[auth.google] +;enabled = false +;allow_sign_up = false +;client_id = some_client_id +;client_secret = some_client_secret +;scopes = https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/userinfo.email +;auth_url = https://accounts.google.com/o/oauth2/auth +;token_url = https://accounts.google.com/o/oauth2/token +;api_url = https://www.googleapis.com/oauth2/v1/userinfo +;allowed_domains = + +#################################### Auth Proxy ########################## +[auth.proxy] +;enabled = false +;header_name = X-WEBAUTH-USER +;header_property = username +;auto_sign_up = true + +#################################### Basic Auth ########################## +[auth.basic] +;enabled = true + +#################################### Auth LDAP ########################## +[auth.ldap] +;enabled = false +;config_file = /etc/grafana/ldap.toml + +#################################### SMTP / Emailing ########################## +[smtp] +;enabled = false +;host = localhost:25 +;user = +;password = +;cert_file = +;key_file = +;skip_verify = false +;from_address = [email protected] + +[emails] +;welcome_email_on_sign_up = false + +#################################### Logging ########################## +[log] +# Either "console", "file", default is "console" +# Use comma to separate multiple modes, e.g. "console, file" +;mode = console, file + +# Buffer length of channel, keep it as it is if you don't know what it is. +;buffer_len = 10000 + +# Either "Trace", "Debug", "Info", "Warn", "Error", "Critical", default is "Trace" +;level = Info + +# For "console" mode only +[log.console] +;level = + +# For "file" mode only +[log.file] +;level = +# This enables automated log rotate(switch of following options), default is true +;log_rotate = true + +# Max line number of single file, default is 1000000 +;max_lines = 1000000 + +# Max size shift of single file, default is 28 means 1 << 28, 256MB +;max_lines_shift = 28 + +# Segment log daily, default is true +;daily_rotate = true + +# Expired days of log file(delete after max days), default is 7 +;max_days = 7 + +#################################### AMPQ Event Publisher ########################## +[event_publisher] +;enabled = false +;rabbitmq_url = amqp://localhost/ +;exchange = grafana_events + +;#################################### Dashboard JSON files ########################## +[dashboards.json] +;enabled = false +;path = /var/lib/grafana/dashboards +path = /usr/lib/ambari-metrics-grafana/public/dashboards + </value> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/a7cc3801/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/configuration/ams-hbase-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/configuration/ams-hbase-site.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/configuration/ams-hbase-site.xml index 0432ffa..bf62b8e 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/configuration/ams-hbase-site.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/configuration/ams-hbase-site.xml @@ -20,7 +20,7 @@ * limitations under the License. */ --> -<configuration supports_do_not_extend="true"> +<configuration> <property> <name>hbase.rootdir</name> <value>file:///var/lib/ambari-metrics-collector/hbase</value> @@ -29,6 +29,16 @@ HBase to either local filesystem path if using Ambari Metrics in embedded mode or to a HDFS dir, example: hdfs://namenode.example.org:8020/amshbase. </description> + <depends-on> + <property> + <type>core-site</type> + <name>fs.defaultFS</name> + </property> + <property> + <type>ams-site</type> + <name>timeline.metrics.service.operation.mode</name> + </property> + </depends-on> <on-ambari-upgrade add="true"/> </property> <property> @@ -40,6 +50,9 @@ than '/tmp' (The '/tmp' directory is often cleared on machine restart). </description> + <value-attributes> + <type>directory</type> + </value-attributes> <on-ambari-upgrade add="true"/> </property> <property> @@ -57,6 +70,12 @@ standalone mode and true for distributed mode. If false, startup will run all HBase and ZooKeeper daemons together in the one JVM. </description> + <depends-on> + <property> + <type>ams-site</type> + <name>timeline.metrics.service.operation.mode</name> + </property> + </depends-on> <on-ambari-upgrade add="true"/> </property> <property> @@ -149,7 +168,7 @@ </property> <property> <name>hbase.normalizer.enabled</name> - <value>true</value> + <value>false</value> <description>If set to true, Master will try to keep region size within each table approximately the same.</description> <on-ambari-upgrade add="true"/> @@ -212,7 +231,7 @@ <name>hbase.hregion.max.filesize</name> <value>4294967296</value> <description> - Maximum HFile size. If the sum of the sizes of a regionâs HFiles has grown + Maximum HFile size. If the sum of the sizes of a region’s HFiles has grown to exceed this value, the region is split in two. Default is 10Gb. </description> <on-ambari-upgrade add="true"/> @@ -239,22 +258,42 @@ </property> <property> <name>phoenix.query.timeoutMs</name> - <value>1200000</value> + <value>300000</value> <description> Number of milliseconds after which a query will timeout on the client. - Default is 10 min. + Default is 5 min. + </description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>phoenix.query.keepAliveMs</name> + <value>300000</value> + <description> + Number of milliseconds after which a query will keep the connection to HBase alive. + Default is 5 min. </description> <on-ambari-upgrade add="true"/> </property> <property> <name>hbase.client.scanner.timeout.period</name> - <value>900000</value> + <value>300000</value> <description> Client scanner lease period in milliseconds. </description> <on-ambari-upgrade add="true"/> </property> <property> + <name>hbase.rpc.timeout</name> + <value>300000</value> + <description> + This is for the RPC layer to define how long HBase client applications + take for a remote call to time out. It uses pings to check connections + but will eventually throw a TimeoutException. + </description> + <display-name>HBase RPC Timeout</display-name> + <on-ambari-upgrade add="true"/> + </property> + <property> <name>hbase.regionserver.thread.compaction.large</name> <value>2</value> <description> @@ -272,7 +311,17 @@ </property> <property> <name>hbase.zookeeper.property.clientPort</name> - <value>61181</value> + <value>{{zookeeper_clientPort}}</value> + <depends-on> + <property> + <type>zoo.cfg</type> + <name>clientPort</name> + </property> + <property> + <type>ams-hbase-site</type> + <name>hbase.cluster.distributed</name> + </property> + </depends-on> <on-ambari-upgrade add="true"/> </property> <property> @@ -307,11 +356,6 @@ <on-ambari-upgrade add="true"/> </property> <property> - <name>phoenix.query.spoolThresholdBytes</name> - <value>12582912</value> - <on-ambari-upgrade add="true"/> - </property> - <property> <name>hbase.snapshot.enabled</name> <value>false</value> <description>Enable/Disable HBase snapshots.</description> @@ -324,6 +368,16 @@ <on-ambari-upgrade add="true"/> </property> <property> + <name>hbase.zookeeper.property.tickTime</name> + <value>6000</value> + <description> + The length of a single tick, which is the basic time unit used by + ZooKeeper, as measured in milliseconds. This property setting only + affects the ZK server started by AMS in embedded mode. Unit = ms. + </description> + <on-ambari-upgrade add="true"/> + </property> + <property> <name>zookeeper.session.timeout</name> <value>120000</value> <description>ZooKeeper session timeout in milliseconds.</description> @@ -392,8 +446,8 @@ Maximum living time (in milliseconds) of server caches. A cache entry expires after this amount of time has passed since last access. Consider adjusting this parameter when a server-side IOException( - âCould not find hash cache for joinIdâ) happens. Getting warnings like - âEarlier hash cache(s) might have expired on serversâ might also be a + “Could not find hash cache for joinId”) happens. Getting warnings like + “Earlier hash cache(s) might have expired on servers” might also be a sign that this number should be increased. </description> <on-ambari-upgrade add="true"/> @@ -415,7 +469,6 @@ Hadoop servers should be configured to allow short circuit read for the hbase user for this to take effect </description> - <on-ambari-upgrade add="true"/> <depends-on> <property> <type>ams-hbase-site</type> @@ -426,6 +479,11 @@ <name>hbase.rootdir</name> </property> </depends-on> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>zookeeper.znode.parent</name> + <value>/ams-hbase-unsecure</value> + <on-ambari-upgrade add="true"/> </property> - </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/a7cc3801/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/configuration/ams-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/configuration/ams-site.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/configuration/ams-site.xml index 0a25a9f..b9f534e 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/configuration/ams-site.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/configuration/ams-site.xml @@ -20,7 +20,7 @@ * limitations under the License. */ --> -<configuration supports_do_not_extend="true"> +<configuration> <property> <name>timeline.metrics.service.operation.mode</name> <value>embedded</value> @@ -56,10 +56,10 @@ Directory to store aggregator checkpoints. Change to a permanent location so that checkpoint ar not lost. </description> - <on-ambari-upgrade add="true"/> <value-attributes> <type>directory</type> </value-attributes> + <on-ambari-upgrade add="true"/> </property> <property> <name>timeline.metrics.host.aggregator.minute.interval</name> @@ -69,10 +69,10 @@ Time in seconds to sleep for the minute resolution host based aggregator. Default resolution is 5 minutes. </description> - <on-ambari-upgrade add="true"/> <value-attributes> <type>int</type> </value-attributes> + <on-ambari-upgrade add="true"/> </property> <property> <name>timeline.metrics.host.aggregator.hourly.interval</name> @@ -82,10 +82,10 @@ Time in seconds to sleep for the hourly resolution host based aggregator. Default resolution is 1 hour. </description> - <on-ambari-upgrade add="true"/> <value-attributes> <type>int</type> </value-attributes> + <on-ambari-upgrade add="true"/> </property> <property> <name>timeline.metrics.daily.aggregator.minute.interval</name> @@ -104,10 +104,10 @@ Time in seconds to sleep for the hourly resolution cluster wide aggregator. Default is 1 hour. </description> - <on-ambari-upgrade add="true"/> <value-attributes> <type>int</type> </value-attributes> + <on-ambari-upgrade add="true"/> </property> <property> <name>timeline.metrics.cluster.aggregator.daily.interval</name> @@ -126,10 +126,10 @@ Time in seconds to sleep for the minute resolution cluster wide aggregator. Default resolution is 5 minutes. </description> - <on-ambari-upgrade add="true"/> <value-attributes> <type>int</type> </value-attributes> + <on-ambari-upgrade add="true"/> </property> <property> <name>timeline.metrics.cluster.aggregator.second.interval</name> @@ -139,14 +139,14 @@ Time in seconds to sleep for the second resolution cluster wide aggregator. Default resolution is 2 minutes. </description> - <on-ambari-upgrade add="true"/> <value-attributes> <type>int</type> </value-attributes> + <on-ambari-upgrade add="true"/> </property> <property> <name>timeline.metrics.host.aggregator.daily.checkpointCutOffMultiplier</name> - <value>1</value> + <value>2</value> <description> Multiplier value * interval = Max allowed checkpoint lag. Effectively if aggregator checkpoint is greater than max allowed checkpoint delay, @@ -163,10 +163,10 @@ if aggregator checkpoint is greater than max allowed checkpoint delay, the checkpoint will be discarded by the aggregator. </description> - <on-ambari-upgrade add="true"/> <value-attributes> <type>int</type> </value-attributes> + <on-ambari-upgrade add="true"/> </property> <property> <name>timeline.metrics.host.aggregator.minute.checkpointCutOffMultiplier</name> @@ -177,10 +177,10 @@ if aggregator checkpoint is greater than max allowed checkpoint delay, the checkpoint will be discarded by the aggregator. </description> - <on-ambari-upgrade add="true"/> <value-attributes> <type>int</type> </value-attributes> + <on-ambari-upgrade add="true"/> </property> <property> <name>timeline.metrics.cluster.aggregator.hourly.checkpointCutOffMultiplier</name> @@ -191,10 +191,10 @@ if aggregator checkpoint is greater than max allowed checkpoint delay, the checkpoint will be discarded by the aggregator. </description> - <on-ambari-upgrade add="true"/> <value-attributes> <type>int</type> </value-attributes> + <on-ambari-upgrade add="true"/> </property> <property> <name>timeline.metrics.cluster.aggregator.second.checkpointCutOffMultiplier</name> @@ -205,10 +205,10 @@ if aggregator checkpoint is greater than max allowed checkpoint delay, the checkpoint will be discarded by the aggregator. </description> - <on-ambari-upgrade add="true"/> <value-attributes> <type>int</type> </value-attributes> + <on-ambari-upgrade add="true"/> </property> <property> <name>timeline.metrics.cluster.aggregator.minute.checkpointCutOffMultiplier</name> @@ -219,14 +219,14 @@ if aggregator checkpoint is greater than max allowed checkpoint delay, the checkpoint will be discarded by the aggregator. </description> - <on-ambari-upgrade add="true"/> <value-attributes> <type>int</type> </value-attributes> + <on-ambari-upgrade add="true"/> </property> <property> <name>timeline.metrics.cluster.aggregator.daily.checkpointCutOffMultiplier</name> - <value>1</value> + <value>2</value> <description> Multiplier value * interval = Max allowed checkpoint lag. Effectively if aggregator checkpoint is greater than max allowed checkpoint delay, @@ -302,16 +302,16 @@ <description> Lowest resolution of desired data for cluster level second aggregates. </description> - <on-ambari-upgrade add="true"/> <value-attributes> <type>int</type> </value-attributes> + <on-ambari-upgrade add="true"/> </property> <property> <name>timeline.metrics.host.aggregator.daily.ttl</name> <value>31536000</value> <description> - Host based daily resolution data purge interval. Default is 1 year. + Host based daily resolution data purge interval in seconds. Default is 1 year. </description> <on-ambari-upgrade add="true"/> </property> @@ -319,7 +319,7 @@ <name>timeline.metrics.host.aggregator.hourly.ttl</name> <value>2592000</value> <description> - Host based hourly resolution data purge interval. Default is 30 days. + Host based hourly resolution data purge interval in seconds. Default is 30 days. </description> <on-ambari-upgrade add="true"/> </property> @@ -327,23 +327,23 @@ <name>timeline.metrics.host.aggregator.minute.ttl</name> <value>604800</value> <description> - Host based minute resolution data purge interval. Default is 7 days. + Host based minute resolution data purge interval in seconds. Default is 7 days. </description> <on-ambari-upgrade add="true"/> </property> <property> <name>timeline.metrics.cluster.aggregator.second.ttl</name> - <value>2592000</value> + <value>259200</value> <description> - Cluster wide second resolution data purge interval. Default is 7 days. + Cluster wide second resolution data purge interval in seconds. Default is 3 days. </description> <on-ambari-upgrade add="true"/> </property> <property> <name>timeline.metrics.cluster.aggregator.minute.ttl</name> - <value>7776000</value> + <value>2592000</value> <description> - Cluster wide minute resolution data purge interval. Default is 30 days. + Cluster wide minute resolution data purge interval in seconds. Default is 30 days. </description> <on-ambari-upgrade add="true"/> </property> @@ -351,7 +351,7 @@ <name>timeline.metrics.cluster.aggregator.hourly.ttl</name> <value>31536000</value> <description> - Cluster wide hourly resolution data purge interval. Default is 1 year. + Cluster wide hourly resolution data purge interval in seconds. Default is 1 year. </description> <on-ambari-upgrade add="true"/> </property> @@ -359,7 +359,7 @@ <name>timeline.metrics.cluster.aggregator.daily.ttl</name> <value>63072000</value> <description> - Cluster wide daily resolution data purge interval. Default is 2 years. + Cluster wide daily resolution data purge interval in seconds. Default is 2 years. </description> <on-ambari-upgrade add="true"/> </property> @@ -367,8 +367,14 @@ <name>timeline.metrics.host.aggregator.ttl</name> <value>86400</value> <description> - 1 minute resolution data purge interval. Default is 1 day. + 1 minute resolution data purge interval in seconds. Default is 1 day. </description> + <depends-on> + <property> + <type>ams-site</type> + <name>timeline.metrics.service.operation.mode</name> + </property> + </depends-on> <on-ambari-upgrade add="true"/> </property> <property> @@ -399,10 +405,10 @@ 22 aggregate metrics/min * 2 * 60 * 6 : Retrieve 10 SECOND data for 2 hours. </description> <display-name>Metrics service default result limit</display-name> - <on-ambari-upgrade add="true"/> <value-attributes> <type>int</type> </value-attributes> + <on-ambari-upgrade add="true"/> </property> <property> <name>timeline.metrics.service.checkpointDelay</name> @@ -412,10 +418,10 @@ Time in seconds to sleep on the first run or when the checkpoint is too old. </description> - <on-ambari-upgrade add="true"/> <value-attributes> <type>int</type> </value-attributes> + <on-ambari-upgrade add="true"/> </property> <property> <name>timeline.metrics.service.resultset.fetchSize</name> @@ -424,10 +430,10 @@ <description> JDBC resultset prefect size for aggregator queries. </description> - <on-ambari-upgrade add="true"/> <value-attributes> <type>int</type> </value-attributes> + <on-ambari-upgrade add="true"/> </property> <!-- Phoenix properties that would manifest in the hbase-site.xml on the client side --> <property> @@ -446,6 +452,9 @@ Set directory for Phoenix spill files. If possible set this to a different mount point from the one for hbase.rootdir in embedded mode. </description> + <value-attributes> + <type>directory</type> + </value-attributes> <on-ambari-upgrade add="true"/> </property> <property> @@ -473,7 +482,6 @@ Pre-split regions using the split points corresponding to this property for the precision table that stores seconds aggregate data. </description> - <on-ambari-upgrade add="true"/> <depends-on> <property> <type>ams-hbase-site</type> @@ -492,6 +500,7 @@ <name>hbase_regionserver_heapsize</name> </property> </depends-on> + <on-ambari-upgrade add="true"/> </property> <property> <name>timeline.metrics.cluster.aggregate.splitpoints</name> @@ -500,7 +509,6 @@ Pre-split regions using the split points corresponding to this property for the aggregate table that stores seconds aggregate data across hosts. </description> - <on-ambari-upgrade add="true"/> <depends-on> <property> <type>ams-hbase-site</type> @@ -519,8 +527,8 @@ <name>hbase_regionserver_heapsize</name> </property> </depends-on> + <on-ambari-upgrade add="true"/> </property> - <property> <name>timeline.metrics.sink.report.interval</name> <value>60</value> @@ -532,13 +540,27 @@ </property> <property> <name>timeline.metrics.sink.collection.period</name> - <value>60</value> + <value>10</value> <description> The interval between two service metrics data exports. </description> <on-ambari-upgrade add="true"/> </property> <property> + <name>timeline.metrics.service.watcher.disabled</name> + <value>false</value> + <description> + Disable Timeline Metric Store watcher thread. Disabled by default in AMS distributed mode. + </description> + <depends-on> + <property> + <type>ams-site</type> + <name>timeline.metrics.service.operation.mode</name> + </property> + </depends-on> + <on-ambari-upgrade add="true"/> + </property> + <property> <name>timeline.metrics.service.watcher.initial.delay</name> <value>600</value> <description> @@ -567,12 +589,135 @@ <on-ambari-upgrade add="true"/> </property> <property> - <name>timeline.metrics.hbase.fifo.compaction.enabled</name> + <name>timeline.metrics.aggregators.skip.blockcache.enabled</name> + <value>false</value> + <description> + Skip block cache on aggregator queries to allow, HBase block + utilization only for user queries. + </description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>timeline.metrics.cache.commit.interval</name> + <value>3</value> + <description> + Time in seconds between committing metrics from cache + </description> + <value-attributes> + <type>int</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>timeline.metrics.cache.size</name> + <value>150</value> + <description> + Size of array blocking queue used to cache metrics + </description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>timeline.metrics.cache.enabled</name> <value>true</value> <description> - Enable Compaction policy for lower precision and minute aggregate tables. + If set to true PhoenixHBaseAccessor will use cache to store metrics before committing them + </description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>timeline.metrics.service.http.policy</name> + <value>HTTP_ONLY</value> + <description> + This configures the HTTP endpoint for Yarn Application History Server for + Ambari Metrics System. + The following values are supported: + - HTTP_ONLY : Service is provided only on http + - HTTPS_ONLY : Service is provided only on https + </description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>failover.strategy</name> + <value>round-robin</value> + <description> + Failover strategy for metric monitors + </description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>timeline.metrics.hbase.init.check.enabled</name> + <value>true</value> + <description> + Enable Initialization check for HBase tables during Metrics service startup. + </description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>timeline.metrics.cluster.aggregator.interpolation.enabled</name> + <value>true</value> + <description> + Enable Linear interpolation for missing slices of data, while aggregating. + </description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>timeline.metrics.service.metadata.filters</name> + <value>ContainerResource</value> + <description> + Commas separated list of regular expressions that match metric names + which prevents certain metrics from ending up in metadata cache. </description> <on-ambari-upgrade add="true"/> </property> - + <property> + <name>timeline.metrics.cluster.aggregation.sql.filters</name> + <value>sdisk\_%,boottime</value> + <description> + Commas separated list of Metric names or Phoenix 'LIKE' class expressions that match metric names + which prevents certain metrics from being aggregated across hosts. + </description> + <on-ambari-upgrade add="true"/> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + <property> + <name>timeline.metrics.downsampler.topn.metric.patterns</name> + <value>dfs.NNTopUserOpCounts.windowMs=60000.op=__%.user=%,dfs.NNTopUserOpCounts.windowMs=300000.op=__%.user=%,dfs.NNTopUserOpCounts.windowMs=1500000.op=__%.user=%</value> + <description> + Commas separated list of metric name regular expressions that are candidates for Top N downsampling. + </description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>timeline.metrics.downsampler.topn.value</name> + <value>10</value> + <description> + Top N value for to be used for top N downsampling. Default is 10. + </description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>timeline.metrics.downsampler.topn.function</name> + <value>max</value> + <description> + Top N function for to be used for top N downsampling (avg/max/sum) + </description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>cluster.zookeeper.quorum</name> + <value>{{cluster_zookeeper_quorum_hosts}}</value> + <description>Comma separated list of servers in the cluster ZooKeeper Quorum. + </description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>cluster.zookeeper.property.clientPort</name> + <value>{{cluster_zookeeper_clientPort}}</value> + <on-ambari-upgrade add="true"/> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/a7cc3801/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/configuration/ams-ssl-client.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/configuration/ams-ssl-client.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/configuration/ams-ssl-client.xml new file mode 100644 index 0000000..cac39de --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/configuration/ams-ssl-client.xml @@ -0,0 +1,51 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration> + <property> + <name>ssl.client.truststore.location</name> + <value>/etc/security/clientKeys/all.jks</value> + <description>Location of the trust store file.</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>ssl.client.truststore.type</name> + <value>jks</value> + <description>Optional. Default value is "jks".</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>ssl.client.truststore.password</name> + <value>bigdata</value> + <property-type>PASSWORD</property-type> + <description>Password to open the trust store file.</description> + <value-attributes> + <type>password</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>ssl.client.truststore.alias</name> + <value></value> + <description>Alias used to create certificate for AMS. (Default is hostname)</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/a7cc3801/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/configuration/ams-ssl-server.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/configuration/ams-ssl-server.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/configuration/ams-ssl-server.xml new file mode 100644 index 0000000..5d2745f --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/configuration/ams-ssl-server.xml @@ -0,0 +1,80 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration> + <property> + <name>ssl.server.truststore.location</name> + <value>/etc/security/serverKeys/all.jks</value> + <description>Location of the trust store file.</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>ssl.server.truststore.type</name> + <value>jks</value> + <description>Optional. Default value is "jks".</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>ssl.server.truststore.password</name> + <value>bigdata</value> + <property-type>PASSWORD</property-type> + <description>Password to open the trust store file.</description> + <value-attributes> + <type>password</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>ssl.server.truststore.reload.interval</name> + <value>10000</value> + <description>Truststore reload interval, in milliseconds.</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>ssl.server.keystore.type</name> + <value>jks</value> + <description>Optional. Default value is "jks".</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>ssl.server.keystore.location</name> + <value>/etc/security/serverKeys/keystore.jks</value> + <description>Location of the keystore file.</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>ssl.server.keystore.password</name> + <value>bigdata</value> + <property-type>PASSWORD</property-type> + <description>Password to open the keystore file.</description> + <value-attributes> + <type>password</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>ssl.server.keystore.keypassword</name> + <value>bigdata</value> + <property-type>PASSWORD</property-type> + <description>Password for private key in keystore file.</description> + <value-attributes> + <type>password</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration>
