Repository: ambari Updated Branches: refs/heads/branch-3.0-ams dfa64cb99 -> 0fcca47f6
AMBARI-22437 : Create an 'AD Manager' component in Ambari Metrics Service stack side. (avijayan) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0fcca47f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0fcca47f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0fcca47f Branch: refs/heads/branch-3.0-ams Commit: 0fcca47f6a178a1a155b4ed7ee0d89c1c7903c6a Parents: dfa64cb Author: Aravindan Vijayan <avija...@hortonworks.com> Authored: Tue Nov 14 09:46:00 2017 -0800 Committer: Aravindan Vijayan <avija...@hortonworks.com> Committed: Tue Nov 14 09:46:00 2017 -0800 ---------------------------------------------------------------------- .../conf/unix/ambari-metrics-admanager.sh | 194 +++++++++++++++++++ .../conf/unix/log4j.properties | 31 +++ .../pom.xml | 14 +- .../src/main/resources/config.yml | 6 +- .../app/AnomalyDetectionAppConfig.scala | 7 +- .../app/AnomalyDetectionAppModule.scala | 9 +- .../configuration/HBaseConfiguration.scala | 2 + .../MetricCollectorConfiguration.scala | 16 +- .../MetricDefinitionDBConfiguration.scala | 6 +- .../adservice/db/LevelDbStoreAccessor.scala | 56 ++++++ .../adservice/leveldb/LevelDBDatasource.scala | 17 +- .../adservice/metadata/ADMetadataProvider.scala | 17 +- .../metadata/MetricDefinitionServiceImpl.scala | 32 +-- .../adservice/resource/AnomalyResource.scala | 2 +- .../resource/MetricDefinitionResource.scala | 24 ++- .../subsystem/trend/TrendAnomalyInstance.scala | 17 ++ .../app/AnomalyDetectionAppConfigTest.scala | 14 +- .../app/DefaultADResourceSpecTest.scala | 4 +- .../leveldb/LevelDBDataSourceTest.scala | 4 +- .../configuration/ams-admanager-config.xml | 51 +++++ .../0.1.0/configuration/ams-admanager-env.xml | 12 +- .../0.1.0/configuration/ams-admanager-log4j.xml | 86 ++++++++ .../AMBARI_METRICS/0.1.0/metainfo.xml | 1 + .../AMBARI_METRICS/0.1.0/package/scripts/ams.py | 23 ++- .../0.1.0/package/scripts/params.py | 15 +- .../0.1.0/package/scripts/status_params.py | 2 +- .../package/templates/admanager_config.yaml.j2 | 20 ++ .../2.0.6/AMBARI_METRICS/test_admanager.py | 106 ++++++++++ .../python/stacks/2.0.6/configs/default.json | 13 ++ .../2.0.6/configs/default_ams_embedded.json | 13 ++ 30 files changed, 751 insertions(+), 63 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/ambari-metrics-admanager.sh ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/ambari-metrics-admanager.sh b/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/ambari-metrics-admanager.sh new file mode 100644 index 0000000..f1a1ae3 --- /dev/null +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/ambari-metrics-admanager.sh @@ -0,0 +1,194 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific + +PIDFILE=/var/run//var/run/ambari-metrics-anomaly-detection/ambari-metrics-admanager.pid +OUTFILE=/var/log/ambari-metrics-anomaly-detection/ambari-metrics-admanager.out + +CONF_DIR=/etc/ambari-metrics-anomaly-detection/conf +DAEMON_NAME=ams_admanager + +STOP_TIMEOUT=5 + +function write_pidfile +{ + local pidfile="$1" + echo $! > "${pidfile}" 2>/dev/null + if [[ $? -gt 0 ]]; then + echo "ERROR: Cannot write pid ${pidfile}." | tee -a $STARTUPFILE + exit 1; + fi +} + +function java_setup +{ + # Bail if we did not detect it + if [[ -z "${JAVA_HOME}" ]]; then + echo "ERROR: JAVA_HOME is not set and could not be found." + exit 1 + fi + + if [[ ! -d "${JAVA_HOME}" ]]; then + echo "ERROR: JAVA_HOME ${JAVA_HOME} does not exist." + exit 1 + fi + + JAVA="${JAVA_HOME}/bin/java" + + if [[ ! -x "$JAVA" ]]; then + echo "ERROR: $JAVA is not executable." + exit 1 + fi +} + +function daemon_status() +{ + # + # LSB 4.1.0 compatible status command (1) + # + # 0 = program is running + # 1 = dead, but still a pid (2) + # 2 = (not used by us) + # 3 = not running + # + # 1 - this is not an endorsement of the LSB + # + # 2 - technically, the specification says /var/run/pid, so + # we should never return this value, but we're giving + # them the benefit of a doubt and returning 1 even if + # our pid is not in in /var/run . + # + + local pidfile="$1" + shift + + local pid + + if [[ -f "${pidfile}" ]]; then + pid=$(cat "${pidfile}") + if ps -p "${pid}" > /dev/null 2>&1; then + return 0 + fi + return 1 + fi + return 3 +} + +function start() +{ + java_setup + + daemon_status "${PIDFILE}" + if [[ $? == 0 ]]; then + echo "AMS AD Manager is running as process $(cat "${PIDFILE}"). Exiting" | tee -a $STARTUPFILE + exit 0 + else + # stale pid file, so just remove it and continue on + rm -f "${PIDFILE}" >/dev/null 2>&1 + fi + + nohup "${JAVA}" "-Xms$AMS_AD_HEAPSIZE" "-Xmx$AMS_AD_HEAPSIZE" ${AMS_AD_OPTS} "-Dlog4j.configuration=file://$CONF_DIR/log4j.properties" "-jar" "/usr/lib/ambari-metrics-anomaly-detection/ambari-metrics-anomaly-detection-service.jar" "server" "${CONF_DIR}/config.yaml" "$@" > $OUTFILE 2>&1 & + PID=$! + write_pidfile "${PIDFILE}" + sleep 2 + + echo "Verifying ${DAEMON_NAME} process status..." + if [ -z "`ps ax -o pid | grep ${PID}`" ]; then + if [ -s ${OUTFILE} ]; then + echo "ERROR: ${DAEMON_NAME} start failed. For more details, see ${OUTFILE}:" + echo "====================" + tail -n 10 ${OUTFILE} + echo "====================" + else + echo "ERROR: ${DAEMON_NAME} start failed" + rm -f ${PIDFILE} + fi + echo "Anomaly Detection Manager out at: ${OUTFILE}" + exit -1 + fi + + rm -f $STARTUPFILE #Deleting startup file + echo "Anomaly Detection Manager successfully started." + } + +function stop() +{ + pidfile=${PIDFILE} + + if [[ -f "${pidfile}" ]]; then + pid=$(cat "$pidfile") + + kill "${pid}" >/dev/null 2>&1 + sleep "${STOP_TIMEOUT}" + + if kill -0 "${pid}" > /dev/null 2>&1; then + echo "WARNING: ${DAEMON_NAME} did not stop gracefully after ${STOP_TIMEOUT} seconds: Trying to kill with kill -9" + kill -9 "${pid}" >/dev/null 2>&1 + fi + + if ps -p "${pid}" > /dev/null 2>&1; then + echo "ERROR: Unable to kill ${pid}" + else + rm -f "${pidfile}" >/dev/null 2>&1 + fi + fi +} + +# execute ams-env.sh +if [[ -f "${CONF_DIR}/ams-admanager-env.sh" ]]; then + . "${CONF_DIR}/ams-admanager-env.sh" +else + echo "ERROR: Cannot execute ${CONF_DIR}/ams-admanager-env.sh." 2>&1 + exit 1 +fi + +# set these env variables only if they were not set by ams-env.sh +: ${AMS_AD_LOG_DIR:=/var/log/ambari-metrics-anomaly-detection} + +# set pid dir path +if [[ -n "${AMS_AD_PID_DIR}" ]]; then + PIDFILE=${AMS_AD_PID_DIR}/admanager.pid +fi + +# set out file path +if [[ -n "${AMS_AD_LOG_DIR}" ]]; then + OUTFILE=${AMS_AD_LOG_DIR}/ambari-metrics-admanager.out +fi + +#TODO manage 3 hbase daemons for start/stop/status +case "$1" in + + start) + start + + ;; + stop) + stop + + ;; + status) + daemon_status "${PIDFILE}" + if [[ $? == 0 ]]; then + echo "AMS AD Manager is running as process $(cat "${PIDFILE}")." + else + echo "AMS AD Manager is not running." + fi + ;; + restart) + stop + start + ;; + +esac http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/log4j.properties ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/log4j.properties b/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/log4j.properties new file mode 100644 index 0000000..9dba1da --- /dev/null +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/log4j.properties @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Define some default values that can be overridden by system properties +# Root logger option +log4j.rootLogger=INFO,file + +# Direct log messages to a log file +log4j.appender.file=org.apache.log4j.RollingFileAppender +log4j.appender.file.File=/var/log/ambari-metrics-anomaly-detection/ambari-metrics-admanager.log +log4j.appender.file.MaxFileSize=80MB +log4j.appender.file.MaxBackupIndex=60 +log4j.appender.file.layout=org.apache.log4j.PatternLayout +log4j.appender.file.layout.ConversionPattern=%d{ABSOLUTE} %5p [%t] %c{1}:%L - %m%n + + http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-metrics/ambari-metrics-anomaly-detection-service/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/pom.xml b/ambari-metrics/ambari-metrics-anomaly-detection-service/pom.xml index cfa8124..142f02f 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/pom.xml +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/pom.xml @@ -135,7 +135,7 @@ <version>3.1.0</version> <configuration> <createDependencyReducedPom>false</createDependencyReducedPom> - <minimizeJar>true</minimizeJar> + <!--<minimizeJar>true</minimizeJar>--> <filters> <filter> <artifact>*:*</artifact> @@ -231,6 +231,12 @@ <groupId>org.apache.kafka</groupId> <artifactId>connect-json</artifactId> <version>0.10.1.0</version> + <exclusions> + <exclusion> + <artifactId>jackson-databind</artifactId> + <groupId>com.fasterxml.jackson.core</groupId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.spark</groupId> @@ -262,6 +268,10 @@ <artifactId>jersey-json</artifactId> <groupId>com.sun.jersey</groupId> </exclusion> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -307,7 +317,6 @@ <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> - <scope>provided</scope> <exclusions> <exclusion> <groupId>commons-el</groupId> @@ -446,7 +455,6 @@ <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <version>21.0</version> - <scope>test</scope> </dependency> <dependency> <groupId>io.dropwizard.metrics</groupId> http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/config.yml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/config.yml b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/config.yml index 299a472..9402f6e 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/config.yml +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/config.yml @@ -21,10 +21,12 @@ logging: type: external metricDefinitionService: - inputDefinitionDirectory: /etc/ambari-metrics-anomaly-detection/conf + inputDefinitionDirectory: /etc/ambari-metrics-anomaly-detection/conf/definitionDirectory metricsCollector: - hostPortList: host1:6188,host2:6188 + hosts: host1,host2 + port: 6188 + protocol: http metadataEndpoint: /v1/timeline/metrics/metadata/keys adQueryService: http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppConfig.scala ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppConfig.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppConfig.scala index aa20223..93f6b28 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppConfig.scala +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppConfig.scala @@ -20,14 +20,16 @@ package org.apache.ambari.metrics.adservice.app import javax.validation.Valid -import org.apache.ambari.metrics.adservice.configuration._ +import org.apache.ambari.metrics.adservice.configuration.{HBaseConfiguration, _} + +import com.fasterxml.jackson.annotation.{JsonIgnore, JsonIgnoreProperties, JsonProperty} -import com.fasterxml.jackson.annotation.JsonProperty import io.dropwizard.Configuration /** * Top Level AD System Manager config items. */ +@JsonIgnoreProperties(ignoreUnknown=true) class AnomalyDetectionAppConfig extends Configuration { /* @@ -54,6 +56,7 @@ class AnomalyDetectionAppConfig extends Configuration { /* HBase Conf */ + @JsonIgnore def getHBaseConf : org.apache.hadoop.conf.Configuration = { HBaseConfiguration.getHBaseConf } http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppModule.scala ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppModule.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppModule.scala index 28b2880..a896563 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppModule.scala +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppModule.scala @@ -17,14 +17,16 @@ */ package org.apache.ambari.metrics.adservice.app -import org.apache.ambari.metrics.adservice.db.MetadataDatasource +import org.apache.ambari.metrics.adservice.db.{AdMetadataStoreAccessor, LevelDbStoreAccessor, MetadataDatasource} import org.apache.ambari.metrics.adservice.leveldb.LevelDBDataSource -import org.apache.ambari.metrics.adservice.resource.{AnomalyResource, RootResource} +import org.apache.ambari.metrics.adservice.metadata.{MetricDefinitionService, MetricDefinitionServiceImpl} +import org.apache.ambari.metrics.adservice.resource.{AnomalyResource, MetricDefinitionResource, RootResource} import org.apache.ambari.metrics.adservice.service.{ADQueryService, ADQueryServiceImpl} import com.codahale.metrics.health.HealthCheck import com.google.inject.AbstractModule import com.google.inject.multibindings.Multibinder + import io.dropwizard.setup.Environment class AnomalyDetectionAppModule(config: AnomalyDetectionAppConfig, env: Environment) extends AbstractModule { @@ -34,8 +36,11 @@ class AnomalyDetectionAppModule(config: AnomalyDetectionAppConfig, env: Environm val healthCheckBinder = Multibinder.newSetBinder(binder(), classOf[HealthCheck]) healthCheckBinder.addBinding().to(classOf[DefaultHealthCheck]) bind(classOf[AnomalyResource]) + bind(classOf[MetricDefinitionResource]) bind(classOf[RootResource]) + bind(classOf[AdMetadataStoreAccessor]).to(classOf[LevelDbStoreAccessor]) bind(classOf[ADQueryService]).to(classOf[ADQueryServiceImpl]) + bind(classOf[MetricDefinitionService]).to(classOf[MetricDefinitionServiceImpl]) bind(classOf[MetadataDatasource]).to(classOf[LevelDBDataSource]) } } http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/HBaseConfiguration.scala ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/HBaseConfiguration.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/HBaseConfiguration.scala index a7bbc66..a51a959 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/HBaseConfiguration.scala +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/HBaseConfiguration.scala @@ -19,12 +19,14 @@ package org.apache.ambari.metrics.adservice.configuration import java.net.{MalformedURLException, URISyntaxException} import org.apache.hadoop.conf.Configuration +import org.slf4j.{Logger, LoggerFactory} object HBaseConfiguration { val HBASE_SITE_CONFIGURATION_FILE: String = "hbase-site.xml" val hbaseConf: org.apache.hadoop.conf.Configuration = new Configuration(true) var isInitialized: Boolean = false + val LOG : Logger = LoggerFactory.getLogger("HBaseConfiguration") def initConfigs(): Unit = { if (!isInitialized) { http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/MetricCollectorConfiguration.scala ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/MetricCollectorConfiguration.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/MetricCollectorConfiguration.scala index 9418897..2530730 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/MetricCollectorConfiguration.scala +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/MetricCollectorConfiguration.scala @@ -28,13 +28,25 @@ import com.fasterxml.jackson.annotation.JsonProperty class MetricCollectorConfiguration { @NotNull - private var hostPortList: String = _ + private var hosts: String = _ + + @NotNull + private var port: String = _ + + @NotNull + private var protocol: String = _ @NotNull private var metadataEndpoint: String = _ @JsonProperty - def getHostPortList: String = hostPortList + def getHosts: String = hosts + + @JsonProperty + def getPort: String = port + + @JsonProperty + def getProtocol: String = protocol @JsonProperty def getMetadataEndpoint: String = metadataEndpoint http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/MetricDefinitionDBConfiguration.scala ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/MetricDefinitionDBConfiguration.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/MetricDefinitionDBConfiguration.scala index 79a350c..ef4e00c 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/MetricDefinitionDBConfiguration.scala +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/MetricDefinitionDBConfiguration.scala @@ -26,12 +26,14 @@ class MetricDefinitionDBConfiguration { @NotNull private var dbDirPath: String = _ + private var verifyChecksums: Boolean = true + private var performParanoidChecks: Boolean = false @JsonProperty("verifyChecksums") - def verifyChecksums: Boolean = true + def getVerifyChecksums: Boolean = verifyChecksums @JsonProperty("performParanoidChecks") - def performParanoidChecks: Boolean = false + def getPerformParanoidChecks: Boolean = performParanoidChecks @JsonProperty("dbDirPath") def getDbDirPath: String = dbDirPath http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/db/LevelDbStoreAccessor.scala ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/db/LevelDbStoreAccessor.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/db/LevelDbStoreAccessor.scala new file mode 100644 index 0000000..baad57d --- /dev/null +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/db/LevelDbStoreAccessor.scala @@ -0,0 +1,56 @@ +package org.apache.ambari.metrics.adservice.db + +import org.apache.ambari.metrics.adservice.metadata.MetricSourceDefinition + +import com.google.inject.Inject + +class LevelDbStoreAccessor extends AdMetadataStoreAccessor{ + + @Inject + var levelDbDataSource : MetadataDatasource = _ + + @Inject + def this(levelDbDataSource: MetadataDatasource) = { + this + this.levelDbDataSource = levelDbDataSource + } + + /** + * Return all saved component definitions from DB. + * + * @return + */ + override def getSavedInputDefinitions: List[MetricSourceDefinition] = { + List.empty[MetricSourceDefinition] + } + + /** + * Save a set of component definitions + * + * @param metricSourceDefinitions Set of component definitions + * @return Success / Failure + */ +override def saveInputDefinitions(metricSourceDefinitions: List[MetricSourceDefinition]): Boolean = { + true +} + + /** + * Save a component definition + * + * @param metricSourceDefinition component definition + * @return Success / Failure + */ + override def saveInputDefinition(metricSourceDefinition: MetricSourceDefinition): Boolean = { + true + } + + /** + * Delete a component definition + * + * @param definitionName component definition + * @return + */ + override def removeInputDefinition(definitionName: String): Boolean = { + true + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/leveldb/LevelDBDatasource.scala ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/leveldb/LevelDBDatasource.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/leveldb/LevelDBDatasource.scala index 6d185bf..a34a60a 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/leveldb/LevelDBDatasource.scala +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/leveldb/LevelDBDatasource.scala @@ -20,6 +20,8 @@ package org.apache.ambari.metrics.adservice.leveldb import java.io.File +import javax.inject.Inject + import org.apache.ambari.metrics.adservice.app.AnomalyDetectionAppConfig import org.apache.ambari.metrics.adservice.configuration.MetricDefinitionDBConfiguration import org.apache.ambari.metrics.adservice.db.MetadataDatasource @@ -29,11 +31,20 @@ import org.iq80.leveldb.impl.Iq80DBFactory import com.google.inject.Singleton @Singleton -class LevelDBDataSource(appConfig: AnomalyDetectionAppConfig) extends MetadataDatasource { +class LevelDBDataSource() extends MetadataDatasource { private var db: DB = _ @volatile var isInitialized: Boolean = false + var appConfig: AnomalyDetectionAppConfig = _ + + @Inject + def this(appConfig: AnomalyDetectionAppConfig) = { + this + this.appConfig = appConfig + initialize() + } + override def initialize(): Unit = { if (isInitialized) return @@ -41,8 +52,8 @@ class LevelDBDataSource(appConfig: AnomalyDetectionAppConfig) extends MetadataDa db = createDB(new LevelDbConfig { override val createIfMissing: Boolean = true - override val verifyChecksums: Boolean = configuration.verifyChecksums - override val paranoidChecks: Boolean = configuration.performParanoidChecks + override val verifyChecksums: Boolean = configuration.getVerifyChecksums + override val paranoidChecks: Boolean = configuration.getPerformParanoidChecks override val path: String = configuration.getDbDirPath }) isInitialized = true http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/metadata/ADMetadataProvider.scala ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/metadata/ADMetadataProvider.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/metadata/ADMetadataProvider.scala index 3bcf4b0..95b1b63 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/metadata/ADMetadataProvider.scala +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/metadata/ADMetadataProvider.scala @@ -32,7 +32,9 @@ import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper */ class ADMetadataProvider extends MetricMetadataProvider { - var metricCollectorHostPorts: Array[String] = Array.empty[String] + var metricCollectorHosts: Array[String] = Array.empty[String] + var metricCollectorPort: String = _ + var metricCollectorProtocol: String = _ var metricMetadataPath: String = "/v1/timeline/metrics/metadata/keys" val connectTimeout: Int = 10000 @@ -42,9 +44,11 @@ class ADMetadataProvider extends MetricMetadataProvider { def this(configuration: MetricCollectorConfiguration) { this - if (StringUtils.isNotEmpty(configuration.getHostPortList)) { - metricCollectorHostPorts = configuration.getHostPortList.split(",") + if (StringUtils.isNotEmpty(configuration.getHosts)) { + metricCollectorHosts = configuration.getHosts.split(",") } + metricCollectorPort = configuration.getPort + metricCollectorProtocol = configuration.getProtocol metricMetadataPath = configuration.getMetadataEndpoint } @@ -57,8 +61,8 @@ class ADMetadataProvider extends MetricMetadataProvider { for (metricDef <- metricSourceDefinition.metricDefinitions) { if (metricDef.isValid) { //Skip requesting metric keys for invalid definitions. - for (hostPort <- metricCollectorHostPorts) { - val metricKeys: Set[MetricKey] = getKeysFromMetricsCollector(hostPort + metricMetadataPath, metricDef) + for (host <- metricCollectorHosts) { + val metricKeys: Set[MetricKey] = getKeysFromMetricsCollector(metricCollectorProtocol, host, metricCollectorPort, metricMetadataPath, metricDef) if (metricKeys != null) { keysMap += (metricDef -> metricKeys) metricKeySet.++(metricKeys) @@ -76,8 +80,9 @@ class ADMetadataProvider extends MetricMetadataProvider { * @param metricDefinition * @return */ - def getKeysFromMetricsCollector(url: String, metricDefinition: MetricDefinition): Set[MetricKey] = { + def getKeysFromMetricsCollector(protocol: String, host: String, port: String, path: String, metricDefinition: MetricDefinition): Set[MetricKey] = { + val url: String = protocol + "://" + host + port + "/" + path val mapper = new ObjectMapper() with ScalaObjectMapper try { val connection = new URL(url).openConnection.asInstanceOf[HttpURLConnection] http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/metadata/MetricDefinitionServiceImpl.scala ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/metadata/MetricDefinitionServiceImpl.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/metadata/MetricDefinitionServiceImpl.scala index ffa9944..c34d2dd 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/metadata/MetricDefinitionServiceImpl.scala +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/metadata/MetricDefinitionServiceImpl.scala @@ -19,15 +19,16 @@ package org.apache.ambari.metrics.adservice.metadata import org.apache.ambari.metrics.adservice.app.AnomalyDetectionAppConfig import org.apache.ambari.metrics.adservice.db.AdMetadataStoreAccessor +import org.slf4j.{Logger, LoggerFactory} import com.google.inject.{Inject, Singleton} @Singleton class MetricDefinitionServiceImpl extends MetricDefinitionService { - @Inject - var adMetadataStoreAccessor: AdMetadataStoreAccessor = _ + val LOG : Logger = LoggerFactory.getLogger(classOf[MetricDefinitionServiceImpl]) + var adMetadataStoreAccessor: AdMetadataStoreAccessor = _ var configuration: AnomalyDetectionAppConfig = _ var metricMetadataProvider: MetricMetadataProvider = _ @@ -36,18 +37,10 @@ class MetricDefinitionServiceImpl extends MetricDefinitionService { var metricDefinitionMetricKeyMap: Map[MetricDefinition, Set[MetricKey]] = Map() @Inject - def this (anomalyDetectionAppConfig: AnomalyDetectionAppConfig) = { - this () - //TODO : Create AD Metadata instance here (or inject) - configuration = anomalyDetectionAppConfig - initializeService() - } - - def this (anomalyDetectionAppConfig: AnomalyDetectionAppConfig, adMetadataStoreAccessor: AdMetadataStoreAccessor) = { + def this (anomalyDetectionAppConfig: AnomalyDetectionAppConfig, metadataStoreAccessor: AdMetadataStoreAccessor) = { this () - //TODO : Create AD Metadata instance here (or inject). Pass in Schema information. + adMetadataStoreAccessor = metadataStoreAccessor configuration = anomalyDetectionAppConfig - this.adMetadataStoreAccessor = adMetadataStoreAccessor initializeService() } @@ -67,13 +60,13 @@ class MetricDefinitionServiceImpl extends MetricDefinitionService { //Load definitions from metadata store val definitionsFromStore: List[MetricSourceDefinition] = adMetadataStoreAccessor.getSavedInputDefinitions for (definition <- definitionsFromStore) { - validateAndSanitizeMetricSourceDefinition(definition) + sanitizeMetricSourceDefinition(definition) } //Load definitions from configs val definitionsFromConfig: List[MetricSourceDefinition] = getInputDefinitionsFromConfig for (definition <- definitionsFromConfig) { - validateAndSanitizeMetricSourceDefinition(definition) + sanitizeMetricSourceDefinition(definition) } //Union the 2 sources, with DB taking precedence. @@ -100,6 +93,9 @@ class MetricDefinitionServiceImpl extends MetricDefinitionService { @Override def getDefinitionByName(name: String): MetricSourceDefinition = { + if (!metricSourceDefinitionMap.contains(name)) { + LOG.warn("Metric Source Definition with name " + name + " not found") + } metricSourceDefinitionMap.apply(name) } @@ -187,7 +183,13 @@ class MetricDefinitionServiceImpl extends MetricDefinitionService { this.adMetadataStoreAccessor = adMetadataStoreAccessor } - def validateAndSanitizeMetricSourceDefinition(metricSourceDefinition: MetricSourceDefinition): Unit = { + + /** + * Look into the Metric Definitions inside a Metric Source definition, and push down source level appId & + * hosts to Metric definition if they do not have an override. + * @param metricSourceDefinition Input Metric Source Definition + */ + def sanitizeMetricSourceDefinition(metricSourceDefinition: MetricSourceDefinition): Unit = { val sourceLevelAppId: String = metricSourceDefinition.appId val sourceLevelHostList: List[String] = metricSourceDefinition.hosts http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/resource/AnomalyResource.scala ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/resource/AnomalyResource.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/resource/AnomalyResource.scala index c941ac3..98ce0c4 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/resource/AnomalyResource.scala +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/resource/AnomalyResource.scala @@ -23,7 +23,7 @@ import javax.ws.rs.{GET, Path, Produces} import org.joda.time.DateTime -@Path("/topNAnomalies") +@Path("/anomaly") class AnomalyResource { @GET http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/resource/MetricDefinitionResource.scala ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/resource/MetricDefinitionResource.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/resource/MetricDefinitionResource.scala index aacea79..16125fa 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/resource/MetricDefinitionResource.scala +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/resource/MetricDefinitionResource.scala @@ -17,12 +17,24 @@ package org.apache.ambari.metrics.adservice.resource +import javax.ws.rs.{GET, Path, Produces} +import javax.ws.rs.core.MediaType.APPLICATION_JSON + +import org.apache.ambari.metrics.adservice.metadata.{MetricDefinitionService, MetricSourceDefinition} +import org.apache.commons.lang.StringUtils + +import com.google.inject.Inject + +@Path("/metric-definition") class MetricDefinitionResource { - /* - GET component definition - POST component definition - DELETE component definition - PUT component definition - */ + @Inject + var metricDefinitionService: MetricDefinitionService = _ + + @GET + @Produces(Array(APPLICATION_JSON)) + def getMetricDefinition (definitionName: String) : MetricSourceDefinition = { + null + } + } http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/subsystem/trend/TrendAnomalyInstance.scala ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/subsystem/trend/TrendAnomalyInstance.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/subsystem/trend/TrendAnomalyInstance.scala index 125da34..3fc0d6f 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/subsystem/trend/TrendAnomalyInstance.scala +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/subsystem/trend/TrendAnomalyInstance.scala @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.ambari.metrics.adservice.subsystem.trend import org.apache.ambari.metrics.adservice.common.{Season, TimeRange} http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppConfigTest.scala ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppConfigTest.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppConfigTest.scala index 104ccea..989ba21 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppConfigTest.scala +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppConfigTest.scala @@ -44,11 +44,21 @@ class AnomalyDetectionAppConfigTest extends FunSuite { assert(config.isInstanceOf[AnomalyDetectionAppConfig]) - assert(config.getMetricDefinitionServiceConfiguration.getInputDefinitionDirectory == "/etc/ambari-metrics-anomaly-detection/conf") + assert(config.getMetricDefinitionServiceConfiguration.getInputDefinitionDirectory == + "/etc/ambari-metrics-anomaly-detection/conf/definitionDirectory") - assert(config.getMetricCollectorConfiguration.getHostPortList == "host1:6188,host2:6188") + assert(config.getMetricCollectorConfiguration.getHosts == "host1,host2") + + assert(config.getMetricCollectorConfiguration.getPort == "6188") assert(config.getAdServiceConfiguration.getAnomalyDataTtl == 604800) + + assert(config.getMetricDefinitionDBConfiguration.getDbDirPath == "/var/lib/ambari-metrics-anomaly-detection/") + + assert(config.getMetricDefinitionDBConfiguration.getVerifyChecksums) + + assert(!config.getMetricDefinitionDBConfiguration.getPerformParanoidChecks) + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/app/DefaultADResourceSpecTest.scala ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/app/DefaultADResourceSpecTest.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/app/DefaultADResourceSpecTest.scala index 65cf609..2a4999c 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/app/DefaultADResourceSpecTest.scala +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/app/DefaultADResourceSpecTest.scala @@ -32,10 +32,10 @@ import com.google.common.io.Resources class DefaultADResourceSpecTest extends FunSpec with Matchers { - describe("/topNAnomalies") { + describe("/anomaly") { it("Must return default message") { withAppRunning(classOf[AnomalyDetectionApp], Resources.getResource("config.yml").getPath) { rule => - val json = client.target(s"http://localhost:${rule.getLocalPort}/topNAnomalies") + val json = client.target(s"http://localhost:${rule.getLocalPort}/anomaly") .request().accept(APPLICATION_JSON).buildGet().invoke(classOf[String]) val now = DateTime.now.toString("MM-dd-yyyy hh:mm") assert(json == "{\"message\":\"Anomaly Detection Service!\"," + "\"today\":\"" + now + "\"}") http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/leveldb/LevelDBDataSourceTest.scala ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/leveldb/LevelDBDataSourceTest.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/leveldb/LevelDBDataSourceTest.scala index 2ddb7b8..9757d76 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/leveldb/LevelDBDataSourceTest.scala +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/leveldb/LevelDBDataSourceTest.scala @@ -36,8 +36,8 @@ class LevelDBDataSourceTest extends FunSuite with BeforeAndAfter with Matchers w val mdConfig : MetricDefinitionDBConfiguration = mock[MetricDefinitionDBConfiguration] when(appConfig.getMetricDefinitionDBConfiguration).thenReturn(mdConfig) - when(mdConfig.verifyChecksums).thenReturn(true) - when(mdConfig.performParanoidChecks).thenReturn(false) + when(mdConfig.getVerifyChecksums).thenReturn(true) + when(mdConfig.getPerformParanoidChecks).thenReturn(false) when(mdConfig.getDbDirPath).thenReturn(file.getAbsolutePath) db = new LevelDBDataSource(appConfig) http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-config.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-config.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-config.xml index 489850f..2c6bbf7 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-config.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-config.xml @@ -57,4 +57,55 @@ </value-attributes> <on-ambari-upgrade add="true"/> </property> + <property> + <name>ambari.metrics.admanager.input.definition.directory</name> + <value></value> + <display-name>AD Manager Input definition directory</display-name> + <description>AMS Anomaly Detection Manager definition directory</description> + <value-attributes> + <type>directory</type> + <overridable>false</overridable> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <display-name>ams-admanager-config template</display-name> + <value> + server: + applicationConnectors: + - type: http + port: {{ams_admanager_port}} + requestLog: + type: external + + logging: + type: external + + metricDefinitionService: + inputDefinitionDirectory: {{ams_ad_input_definition_directory}} + + metricsCollector: + hosts: {{ams_collector_hosts}} + port: {{metric_collector_port}} + protocol: {{metric_collector_protocol}} + metadataEndpoint: /v1/timeline/metrics/metadata/keys + + adQueryService: + anomalyDataTtl: 604800 + + metricDefinitionDB: + # force checksum verification of all data that is read from the file system on behalf of a particular read + verifyChecksums: true + # raise an error as soon as it detects an internal corruption + performParanoidChecks: false + # Path to Level DB directory + dbDirPath: {{ams_ad_data_dir}} + </value> + <value-attributes> + <type>content</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> </configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-env.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-env.xml index 99e93a6..a79796b 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-env.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-env.xml @@ -21,7 +21,7 @@ --> <configuration> <property> - <name>ams_admanager_log_dir</name> + <name>ams_ad_log_dir</name> <value>/var/log/ambari-metrics-anomaly-detection</value> <display-name>Anomaly Detection Manager log dir</display-name> <description>AMS Anomaly Detection Manager log directory.</description> @@ -31,7 +31,7 @@ <on-ambari-upgrade add="true"/> </property> <property> - <name>ams_admanager_pid_dir</name> + <name>ams_ad_pid_dir</name> <value>/var/run/ambari-metrics-anomaly-detection</value> <display-name>Anomaly Detection Manager pid dir</display-name> <description>AMS Anomaly Detection Manager pid directory.</description> @@ -41,7 +41,7 @@ <on-ambari-upgrade add="true"/> </property> <property> - <name>ams_admanager_data_dir</name> + <name>ams_ad_data_dir</name> <value>/var/lib/ambari-metrics-anomaly-detection</value> <display-name>Anomaly Detection Manager data dir</display-name> <description>AMS Anomaly Detection Manager data directory.</description> @@ -74,10 +74,10 @@ export JAVA_HOME={{java64_home}} # Anomaly Detection Manager Log directory for log4j - export AMS_AD_LOG_DIR={{ams_admanager_log_dir}} + export AMS_AD_LOG_DIR={{ams_ad_log_dir}} # Anomaly Detection Manager pid directory - export AMS_AD_PID_DIR={{ams_admanager_pid_dir}} + export AMS_AD_PID_DIR={{ams_ad_pid_dir}} # Anomaly Detection Manager heapsize export AMS_AD_HEAPSIZE={{ams_admanager_heapsize}} @@ -92,7 +92,7 @@ {% endif %} # Anomaly Detection Manager GC options - export AMS_AD_GC_OPTS="-XX:+UseConcMarkSweepGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{ams_admanager_log_dir}}/admanager-gc.log-`date +'%Y%m%d%H%M'`" + export AMS_AD_GC_OPTS="-XX:+UseConcMarkSweepGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{ams_ad_log_dir}}/admanager-gc.log-`date +'%Y%m%d%H%M'`" export AMS_AD_OPTS="$AMS_AD_OPTS $AMS_AD_GC_OPTS" </value> http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-log4j.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-log4j.xml new file mode 100644 index 0000000..b1f821e --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-log4j.xml @@ -0,0 +1,86 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>ams_ad_log_max_backup_size</name> + <value>80</value> + <description>The maximum size of backup file before the log is rotated</description> + <display-name>AMS AD Manager Log: backup file size</display-name> + <value-attributes> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>ams_ad_log_number_of_backup_files</name> + <value>60</value> + <description>The number of backup files</description> + <display-name>AMS AD Manager Log: # of backup files</display-name> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <display-name>ams-ad-log4j template</display-name> + <description>Custom log4j.properties</description> + <value> + # + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + # + + # Define some default values that can be overridden by system properties + ams.ad.log.dir=. + ams.ad.log.file=ambari-metrics-admanager.log + + # Root logger option + log4j.rootLogger=INFO,file + + # Direct log messages to a log file + log4j.appender.file=org.apache.log4j.RollingFileAppender + log4j.appender.file.File=${ams.ad.log.dir}/${ams.ad.log.file} + log4j.appender.file.MaxFileSize={{ams_ad_log_max_backup_size}}MB + log4j.appender.file.MaxBackupIndex={{ams_ad_log_number_of_backup_files}} + log4j.appender.file.layout=org.apache.log4j.PatternLayout + log4j.appender.file.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml index e80fae4..a80340c 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml @@ -137,6 +137,7 @@ <config-type>ams-hbase-site</config-type> <config-type>ams-admanager-config</config-type> <config-type>ams-admanager-env</config-type> + <config-type>ams-admanager-log4j</config-type> </configuration-dependencies> <logs> <log> http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py index 4b960ff..b22a870 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py @@ -527,14 +527,31 @@ def ams(name=None, action=None): File(format("{ams_ad_conf_dir}/ams-admanager-env.sh"), owner=params.ams_user, group=params.user_group, - content=InlineTemplate(params.ams_grafana_env_sh_template) + content=InlineTemplate(params.ams_admanager_env_sh_template) ) - File(format("{conf_dir}/config.yaml"), - content=Template("config.yaml.j2"), + File(format("{ams_ad_conf_dir}/config.yaml"), + content=InlineTemplate(params.ams_admanager_config_template), owner=params.ams_user, group=params.user_group ) + merged_ams_hbase_site = {} + merged_ams_hbase_site.update(params.config['configurations']['ams-hbase-site']) + if params.security_enabled: + merged_ams_hbase_site.update(params.config['configurations']['ams-hbase-security-site']) + + XmlConfig( "hbase-site.xml", + conf_dir = params.ams_ad_conf_dir, + configurations = merged_ams_hbase_site, + configuration_attributes=params.config['configuration_attributes']['ams-hbase-site'], + owner = params.ams_user, + ) + + if (params.ams_ad_log4j_props != None): + File(os.path.join(params.ams_ad_conf_dir, "log4j.properties"), + owner=params.ams_user, + content=params.ams_ad_log4j_props + ) if action != 'stop': for dir in ams_ad_directories: http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py index a2bfe68..b7004b8 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py @@ -189,12 +189,21 @@ ams_hbase_init_check_enabled = default("/configurations/ams-site/timeline.metric # AD Manager settings ams_ad_conf_dir = '/etc/ambari-metrics-anomaly-detection/conf' -ams_ad_log_dir = default("/configurations/ams-ad-env/ams_admanager_log_dir", 'var/log/ambari-metrics-anomaly-detection') -ams_ad_pid_dir = status_params.ams_admanager_pid_dir -ams_ad_data_dir = default("/configurations/ams-ad-env/ams_admanager_data_dir", '/var/lib/ambari-metrics-anomaly-detection') +ams_ad_log_dir = default("/configurations/ams-admanager-env/ams_ad_log_dir", '/var/log/ambari-metrics-anomaly-detection') +ams_ad_pid_dir = status_params.ams_ad_pid_dir +ams_ad_data_dir = default("/configurations/ams-admanager-env/ams_ad_data_dir", '/var/lib/ambari-metrics-anomaly-detection') +ams_ad_input_definition_directory = config['configurations']['ams-admanager-config']['ambari.metrics.admanager.input.definition.directory'] +ams_admanager_env_sh_template = config['configurations']['ams-admanager-env']['content'] +ams_admanager_config_template = config['configurations']['ams-admanager-config']['content'] ams_admanager_script = "/usr/sbin/ambari-metrics-admanager" ams_admanager_port = config['configurations']['ams-admanager-config']['ambari.metrics.admanager.application.port'] +ams_admanager_heapsize = config['configurations']['ams-admanager-env']['ams_admanager_heapsize'] + +if (('ams-admanager-log4j' in config['configurations']) and ('content' in config['configurations']['ams-admanager-log4j'])): + ams_ad_log4j_props = config['configurations']['ams-admanager-log4j']['content'] +else: + ams_ad_log4j_props = None #hadoop params http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status_params.py index 0a92cd2..c01b368 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status_params.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status_params.py @@ -33,7 +33,7 @@ hbase_user = ams_user ams_collector_pid_dir = config['configurations']['ams-env']['metrics_collector_pid_dir'] ams_monitor_pid_dir = config['configurations']['ams-env']['metrics_monitor_pid_dir'] ams_grafana_pid_dir = config['configurations']['ams-grafana-env']['metrics_grafana_pid_dir'] -ams_admanager_pid_dir = config['configurations']['ams-ad-env']['ams_admanager_pid_dir'] +ams_ad_pid_dir = config['configurations']['ams-admanager-env']['ams_ad_pid_dir'] monitor_pid_file = format("{ams_monitor_pid_dir}/ambari-metrics-monitor.pid") grafana_pid_file = format("{ams_grafana_pid_dir}/grafana-server.pid") http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/admanager_config.yaml.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/admanager_config.yaml.j2 b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/admanager_config.yaml.j2 index 787aa3e..a403978 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/admanager_config.yaml.j2 +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/admanager_config.yaml.j2 @@ -22,3 +22,23 @@ server: logging: type: external + +metricDefinitionService: + inputDefinitionDirectory: {{ams_ad_input_definition_directory}} + +metricsCollector: + hosts: {{ams_collector_hosts}} + port: {{metric_collector_port}} + protocol: {{metric_collector_protocol}} + metadataEndpoint: /v1/timeline/metrics/metadata/keys + +adQueryService: + anomalyDataTtl: 604800 + +metricDefinitionDB: + # force checksum verification of all data that is read from the file system on behalf of a particular read + verifyChecksums: true + # raise an error as soon as it detects an internal corruption + performParanoidChecks: false + # Path to Level DB directory + dbDirPath: {{ams_ad_data_dir}} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_admanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_admanager.py b/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_admanager.py new file mode 100644 index 0000000..dc2f4b0 --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_admanager.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +from mock.mock import MagicMock, patch +from stacks.utils.RMFTestCase import * +import os, sys + +@patch("tempfile.mkdtemp", new = MagicMock(return_value='/some_tmp_dir')) +@patch("os.path.exists", new = MagicMock(return_value=True)) +@patch("platform.linux_distribution", new = MagicMock(return_value="Linux")) +class TestADManager(RMFTestCase): + COMMON_SERVICES_PACKAGE_DIR = "AMBARI_METRICS/0.1.0/package/scripts" + STACK_VERSION = "2.0.6" + + file_path = os.path.dirname(os.path.abspath(__file__)) + file_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(file_path))))) + file_path = os.path.join(file_path, "main", "resources", "common-services", COMMON_SERVICES_PACKAGE_DIR) + + sys.path.append(file_path) + def test_start(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/ams_admanager.py", + classname = "AmsADManager", + command = "start", + config_file="default.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + self.maxDiff=None + self.assert_configure() + self.assertResourceCalled('Execute', ('chown', u'-R', u'ams', '/etc/ambari-metrics-anomaly-detection/conf'), + sudo = True + ) + self.assertResourceCalled('Execute', ('chown', u'-R', u'ams', '/var/log/ambari-metrics-anomaly-detection'), + sudo = True + ) + self.assertResourceCalled('Execute', ('chown', u'-R', u'ams', '/var/lib/ambari-metrics-anomaly-detection'), + sudo = True + ) + self.assertResourceCalled('Execute', ('chown', u'-R', u'ams', '/var/run/ambari-metrics-anomaly-detection'), + sudo = True + ) + self.assertResourceCalled('Execute', '/usr/sbin/ambari-metrics-admanager start', + user = 'ams' + ) + self.assertNoMoreResources() + + def assert_configure(self): + + ams_admanager_directories = [ + '/etc/ambari-metrics-anomaly-detection/conf', + '/var/log/ambari-metrics-anomaly-detection', + '/var/lib/ambari-metrics-anomaly-detection', + '/var/run/ambari-metrics-anomaly-detection' + ] + + for ams_admanager_directory in ams_admanager_directories: + self.assertResourceCalled('Directory', ams_admanager_directory, + owner = 'ams', + group = 'hadoop', + mode=0755, + create_parents = True, + recursive_ownership = True + ) + + self.assertResourceCalled('File', '/etc/ambari-metrics-anomaly-detection/conf/ams-admanager-env.sh', + owner = 'ams', + group = 'hadoop', + content = InlineTemplate(self.getConfig()['configurations']['ams-admanager-env']['content']) + ) + + self.assertResourceCalled('File', '/etc/ambari-metrics-anomaly-detection/conf/config.yaml', + owner = 'ams', + group = 'hadoop', + content = InlineTemplate(self.getConfig()['configurations']['ams-admanager-config']['content']), + ) + + merged_ams_hbase_site = {} + merged_ams_hbase_site.update(self.getConfig()['configurations']['ams-hbase-site']) + + self.assertResourceCalled('XmlConfig', 'hbase-site.xml', + owner = 'ams', + conf_dir = '/etc/ambari-metrics-anomaly-detection/conf', + configurations = merged_ams_hbase_site, + configuration_attributes = self.getConfig()['configuration_attributes']['ams-hbase-site'] + ) + self.assertResourceCalled('File', '/etc/ambari-metrics-anomaly-detection/conf/log4j.properties', + owner = 'ams', + content = '' + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-server/src/test/python/stacks/2.0.6/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json index 43c184c..2845f7c 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json @@ -971,6 +971,19 @@ }, "hadoop-metrics2.properties": { "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\r\n# contributor license agreements. See the NOTICE file distributed with\r\n# this work for additional information regarding copyright ownership.\r\n# The ASF licenses this file to You under the Apache License, Version 2.0\r\n# (the \"License\"); you may not use this file except in compliance with\r\n# the License. You may obtain a copy of the License at\r\n#\r\n# http:\/\/www.apache.org\/licenses\/LICENSE-2.0\r\n#\r\n# Unless required by applicable law or agreed to in writing, software\r\n# distributed under the License is distributed on an \"AS IS\" BASIS,\r\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r\n# See the License for the specific language governing permissions and\r\n# limitations under the License.\r\n\r\n# syntax: [prefix].[source|sink|jmx].[instance].[options]\r\n# See package.html for org.apache.hadoop.metrics2 for details\r\n\r\n{% if has_gang lia_server %}\r\n*.period=60\r\n\r\n*.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31\r\n*.sink.ganglia.period=10\r\n\r\n# default for supportsparse is false\r\n*.sink.ganglia.supportsparse=true\r\n\r\n.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both\r\n.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40\r\n\r\n# Hook up to the server\r\nnamenode.sink.ganglia.servers={{ganglia_server_host}}:8661\r\ndatanode.sink.ganglia.servers={{ganglia_server_host}}:8659\r\njobtracker.sink.ganglia.servers={{ganglia_server_host}}:8662\r\ntasktracker.sink.ganglia.servers={{ganglia_server_host}}:8658\r\nmaptask.sink.ganglia.servers={{ganglia_server_host}}:8660\r\nreducetask.sink.ganglia.servers={{ganglia_server_host}}:8660\r\nresourcemanager.sink.ganglia.servers={{ganglia_server_host}}:8664\r\nnodemanager.sink.ganglia.servers={{ganglia_server_host}}:8657\r\nhistoryserver.sink.ganglia.servers={{ganglia_server_host}}:8666\r\njo urnalnode.sink.ganglia.servers={{ganglia_server_host}}:8654\r\nnimbus.sink.ganglia.servers={{ganglia_server_host}}:8649\r\nsupervisor.sink.ganglia.servers={{ganglia_server_host}}:8650\r\n\r\nresourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue\r\n\r\n{% endif %}\r\n\r\n{% if has_metric_collector %}\r\n\r\n*.period={{metrics_collection_period}}\r\n*.sink.timeline.plugin.urls=file:\/\/\/usr\/lib\/ambari-metrics-hadoop-sink\/ambari-metrics-hadoop-sink.jar\r\n*.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink\r\n*.sink.timeline.period={{metrics_collection_period}}\r\n*.sink.timeline.sendInterval={{metrics_report_interval}}000\r\n*.sink.timeline.slave.host.name={{hostname}}\r\n*.sink.timeline.zookeeper.quorum={{zookeeper_quorum}}\r\n*.sink.timeline.protocol={{metric_collector_protocol}}\r\n*.sink.timeline.port={{metric_collector_port}}\r\n\r\n# HTTPS properties\r\n*.sink.timeline.truststore.path = {{metric_truststore_path}}\r\n*.sink.timeline.trusts tore.type = {{metric_truststore_type}}\r\n*.sink.timeline.truststore.password = {{metric_truststore_password}}\r\n\r\ndatanode.sink.timeline.collector={{metric_collector_hosts}}\r\nnamenode.sink.timeline.collector={{metric_collector_hosts}}\r\nresourcemanager.sink.timeline.collector={{metric_collector_hosts}}\r\nnodemanager.sink.timeline.collector={{metric_collector_hosts}}\r\njobhistoryserver.sink.timeline.collector={{metric_collector_hosts}}\r\njournalnode.sink.timeline.collector={{metric_collector_hosts}}\r\nmaptask.sink.timeline.collector={{metric_collector_hosts}}\r\nreducetask.sink.timeline.collector={{metric_collector_hosts}}\r\napplicationhistoryserver.sink.timeline.collector={{metric_collector_hosts}}\r\n\r\nresourcemanager.sink.timeline.tagsForPrefix.yarn=Queue\r\n\r\n{% if is_nn_client_port_configured %}\r\n# Namenode rpc ports customization\r\nnamenode.sink.timeline.metric.rpc.client.port={{nn_rpc_client_port}}\r\n{% endif %}\r\n{% if is_nn_dn_port_configured %}\r\nnamen ode.sink.timeline.metric.rpc.datanode.port={{nn_rpc_dn_port}}\r\n{% endif %}\r\n{% if is_nn_healthcheck_port_configured %}\r\nnamenode.sink.timeline.metric.rpc.healthcheck.port={{nn_rpc_healthcheck_port}}\r\n{% endif %}\r\n\r\n{% endif %}" + }, + "ams-admanager-env": { + "ams_ad_pid_dir": "/var/run/ambari-metrics-anomaly-detection", + "content": "\n" + }, + "ams-admanager-config": { + "content": "", + "ambari.metrics.admanager.input.definition.directory": "", + "ambari.metrics.admanager.spark.operation.mode": "stand-alone", + "ambari.metrics.admanager.application.port": "9090" + }, + "ams-admanager-log4j": { + "content": "" } }, "configuration_attributes": { http://git-wip-us.apache.org/repos/asf/ambari/blob/0fcca47f/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json index eaaeeb5..ec9ff2b 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json @@ -865,6 +865,19 @@ "timeline.metrics.daily.aggregator.minute.interval": "86400", "timeline.metrics.cluster.aggregator.minute.interval": "120", "timeline.metrics.host.aggregator.hourly.interval": "3600" + }, + "ams-admanager-env": { + "ams_ad_pid_dir": "/var/run/ambari-metrics-anomaly-detection", + "content": "\n" + }, + "ams-admanager-config": { + "content": "", + "ambari.metrics.admanager.input.definition.directory": "", + "ambari.metrics.admanager.spark.operation.mode": "stand-alone", + "ambari.metrics.admanager.application.port": "9090" + }, + "ams-admanager-log4j": { + "content": "" } }, "configuration_attributes": {