This is an automated email from the ASF dual-hosted git repository. avijayan pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/ambari.git
commit d46abc7ac19c4316463214022b28c95d63058764 Author: Aravindan Vijayan <[email protected]> AuthorDate: Thu Nov 30 16:07:56 2017 -0800 AMBARI-22567 : Integrate Spark lifecycle management into AMS AD Manager. (avijayan) --- ...trics-admanager.sh => ambari-metrics-admanager} | 75 ++++++++--- .../resources/config.yml => conf/unix/config.yaml} | 9 +- .../pom.xml | 44 ++++-- .../src/main/assemblies/empty.xml | 21 +++ .../adservice/app/AnomalyDetectionAppConfig.scala | 10 ++ .../MetricDefinitionServiceConfiguration.scala | 3 - .../configuration/SparkConfiguration.scala | 39 ++++++ .../adservice/db/PhoenixAnomalyStoreAccessor.scala | 5 +- .../PointInTimeAnomalyInstance.scala | 4 +- .../adservice/{common => model}/Range.scala | 2 +- .../adservice/{common => model}/Season.scala | 4 +- .../adservice/{common => model}/SeasonType.scala | 2 +- .../adservice/{common => model}/TimeRange.scala | 2 +- .../trend => model}/TrendAnomalyInstance.scala | 4 +- .../config.yml => test/resources/config.yaml} | 17 +-- .../app/AnomalyDetectionAppConfigTest.scala | 19 +-- .../adservice/app/DefaultADResourceSpecTest.scala | 2 +- .../adservice/{common => model}/RangeTest.scala | 7 +- .../adservice/{common => model}/SeasonTest.scala | 19 +-- ambari-metrics/ambari-metrics-assembly/pom.xml | 148 +++++++++++++++++++++ .../src/main/assembly/anomaly-detection.xml | 60 +++++++++ .../package/rpm/anomaly-detection/postinstall.sh | 27 ++++ ambari-metrics/pom.xml | 4 +- .../0.1.0/configuration/ams-admanager-config.xml | 4 + .../0.1.0/configuration/ams-admanager-env.xml | 6 +- .../0.1.0/configuration/ams-admanager-log4j.xml | 2 +- .../configuration/ams-admanager-spark-env.xml | 129 ++++++++++++++++++ .../AMBARI_METRICS/0.1.0/metainfo.xml | 1 + .../AMBARI_METRICS/0.1.0/package/scripts/ams.py | 14 +- .../0.1.0/package/scripts/ams_admanager.py | 4 +- .../AMBARI_METRICS/0.1.0/package/scripts/params.py | 22 ++- .../0.1.0/package/scripts/status_params.py | 2 +- 32 files changed, 613 insertions(+), 98 deletions(-) diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/ambari-metrics-admanager.sh b/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/ambari-metrics-admanager similarity index 70% rename from ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/ambari-metrics-admanager.sh rename to ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/ambari-metrics-admanager index f1a1ae3..98b7606 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/ambari-metrics-admanager.sh +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/ambari-metrics-admanager @@ -14,13 +14,44 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific -PIDFILE=/var/run//var/run/ambari-metrics-anomaly-detection/ambari-metrics-admanager.pid +PIDFILE=/var/run/ambari-metrics-anomaly-detection/ambari-metrics-admanager.pid OUTFILE=/var/log/ambari-metrics-anomaly-detection/ambari-metrics-admanager.out CONF_DIR=/etc/ambari-metrics-anomaly-detection/conf DAEMON_NAME=ams_admanager +SPARK_HOME=/usr/lib/ambari-metrics-anomaly-detection/spark -STOP_TIMEOUT=5 +SPARK_MASTER_PID=/var/run/ambari-metrics-anomaly-detection/spark-ams-org.apache.spark.deploy.master.Master.pid + +STOP_TIMEOUT=10 + +function spark_daemon +{ + local cmd=$1 + local pid + + if [[ "${cmd}" == "start" ]] + then + + ${SPARK_HOME}/sbin/start-master.sh + sleep 2 + master_pid=$(cat "$SPARK_MASTER_PID") + if [ -z "`ps ax | grep -w ${master_pid} | grep org.apache.spark.deploy.master.Master`" ]; then + echo "ERROR: Spark Master start failed. For more details, see outfile in log directory." + exit -1 + fi + + ${SPARK_HOME}/sbin/start-slave.sh spark://${SPARK_MASTER_HOST}:${SPARK_MASTER_PORT} + elif [[ "${cmd}" == "stop" ]] + then + ${SPARK_HOME}/sbin/stop-slave.sh + ${SPARK_HOME}/sbin/stop-master.sh + else + pid=${SPARK_MASTER_PID} + daemon_status "${pid}" + fi + +} function write_pidfile { @@ -55,22 +86,6 @@ function java_setup function daemon_status() { - # - # LSB 4.1.0 compatible status command (1) - # - # 0 = program is running - # 1 = dead, but still a pid (2) - # 2 = (not used by us) - # 3 = not running - # - # 1 - this is not an endorsement of the LSB - # - # 2 - technically, the specification says /var/run/pid, so - # we should never return this value, but we're giving - # them the benefit of a doubt and returning 1 even if - # our pid is not in in /var/run . - # - local pidfile="$1" shift @@ -90,6 +105,12 @@ function start() { java_setup + + if [[ "${AMS_AD_STANDALONE_SPARK_ENABLED}" == "true" || "${AMS_AD_STANDALONE_SPARK_ENABLED}" == "True" ]] + then + spark_daemon "start" + fi + daemon_status "${PIDFILE}" if [[ $? == 0 ]]; then echo "AMS AD Manager is running as process $(cat "${PIDFILE}"). Exiting" | tee -a $STARTUPFILE @@ -144,9 +165,12 @@ function stop() rm -f "${pidfile}" >/dev/null 2>&1 fi fi + + #Let's try to stop spark always since if the user has flipped the spark mode to 'yarn', the enabled flag becomes obsolete. + spark_daemon "stop" } -# execute ams-env.sh +# execute ams-admanager-env.sh if [[ -f "${CONF_DIR}/ams-admanager-env.sh" ]]; then . "${CONF_DIR}/ams-admanager-env.sh" else @@ -154,12 +178,21 @@ else exit 1 fi -# set these env variables only if they were not set by ams-env.sh +if [[ -f "${CONF_DIR}/ams-admanager-spark-env.sh" ]]; then + . "${CONF_DIR}/ams-admanager-spark-env.sh" +else + echo "ERROR: Cannot execute ${CONF_DIR}/ams-admanager-spark-env.sh." 2>&1 + exit 1 +fi + +# set these env variables only if they were not set by ams-admanager-env.sh : ${AMS_AD_LOG_DIR:=/var/log/ambari-metrics-anomaly-detection} +: ${AMS_AD_STANDALONE_SPARK_ENABLED:=true} # set pid dir path if [[ -n "${AMS_AD_PID_DIR}" ]]; then - PIDFILE=${AMS_AD_PID_DIR}/admanager.pid + PIDFILE=${AMS_AD_PID_DIR}/ambari-metrics-admanager.pid + SPARK_MASTER_PID=${AMS_AD_PID_DIR}/spark-${USER}-org.apache.spark.deploy.master.Master-1.pid fi # set out file path diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/config.yml b/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/config.yaml similarity index 91% copy from ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/config.yml copy to ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/config.yaml index 7de06b4..85e4004 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/config.yml +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/config.yaml @@ -38,9 +38,8 @@ metricDefinitionDB: # raise an error as soon as it detects an internal corruption performParanoidChecks: false # Path to Level DB directory - dbDirPath: /var/lib/ambari-metrics-anomaly-detection/ + dbDirPath: /tmp/ambari-metrics-anomaly-detection/db -#subsystemService: -# spark: -# pointInTime: -# trend: \ No newline at end of file +spark: + mode: standalone + masterHostPort: localhost:7077 \ No newline at end of file diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/pom.xml b/ambari-metrics/ambari-metrics-anomaly-detection-service/pom.xml index c6927dd..50d7ef6 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/pom.xml +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/pom.xml @@ -36,7 +36,7 @@ <scala.binary.version>2.11</scala.binary.version> <jackson.version>2.9.1</jackson.version> <dropwizard.version>1.2.0</dropwizard.version> - <spark.version>2.2.0</spark.version> + <spark.version>2.1.1</spark.version> <hadoop.version>2.7.3.2.6.0.3-8</hadoop.version> <hbase.version>1.1.2.2.6.0.3-8</hbase.version> <phoenix.version>4.7.0.2.6.0.3-8</phoenix.version> @@ -59,7 +59,7 @@ </pluginRepositories> <build> - <finalName>${project.artifactId}</finalName> + <finalName>${project.artifactId}-${project.version}</finalName> <resources> <resource> <filtering>true</filtering> @@ -157,14 +157,6 @@ </excludes> </filter> <filter> - <artifact>org.apache.phoenix:phoenix-core</artifact> - <excludes> - <exclude>org/joda/time/**</exclude> - <exclude>com/codahale/metrics/**</exclude> - <exclude>com/google/common/collect/**</exclude> - </excludes> - </filter> - <filter> <artifact>*:*</artifact> <excludes> <exclude>com/sun/jersey/**</exclude> @@ -191,6 +183,38 @@ </execution> </executions> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <version>1.7</version> + <executions> + <execution> + <phase>generate-resources</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <target name="Download Spark"> + <mkdir dir="${project.build.directory}/embedded"/> + <get + src="${spark.tar}" + dest="${project.build.directory}/embedded/spark.tar.gz" + usetimestamp="true" + /> + <untar + src="${project.build.directory}/embedded/spark.tar.gz" + dest="${project.build.directory}/embedded" + compression="gzip" + /> + <move + todir="${project.build.directory}/embedded/spark" > + <fileset dir="${project.build.directory}/embedded/${spark.folder}" includes="**"/> + </move> + </target> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/assemblies/empty.xml b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/assemblies/empty.xml new file mode 100644 index 0000000..35738b1 --- /dev/null +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/assemblies/empty.xml @@ -0,0 +1,21 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<assembly> + <id>empty</id> + <formats/> +</assembly> diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppConfig.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppConfig.scala index f9ed4b2..58efa97 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppConfig.scala +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppConfig.scala @@ -53,6 +53,12 @@ class AnomalyDetectionAppConfig extends Configuration { @Valid private val metricDefinitionDBConfiguration = new MetricDefinitionDBConfiguration + /** + * Spark configurations + */ + @Valid + private val sparkConfiguration = new SparkConfiguration + /* AMS HBase Conf */ @@ -76,4 +82,8 @@ class AnomalyDetectionAppConfig extends Configuration { @JsonProperty("metricDefinitionDB") def getMetricDefinitionDBConfiguration: MetricDefinitionDBConfiguration = metricDefinitionDBConfiguration + + @JsonProperty("spark") + def getSparkConfiguration: SparkConfiguration = sparkConfiguration + } diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/MetricDefinitionServiceConfiguration.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/MetricDefinitionServiceConfiguration.scala index b560713..a453f03 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/MetricDefinitionServiceConfiguration.scala +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/MetricDefinitionServiceConfiguration.scala @@ -17,8 +17,6 @@ package org.apache.ambari.metrics.adservice.configuration -import javax.validation.constraints.NotNull - import com.fasterxml.jackson.annotation.JsonProperty /** @@ -26,7 +24,6 @@ import com.fasterxml.jackson.annotation.JsonProperty */ class MetricDefinitionServiceConfiguration { - @NotNull private val inputDefinitionDirectory: String = "" @JsonProperty diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/SparkConfiguration.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/SparkConfiguration.scala new file mode 100644 index 0000000..30efdc7 --- /dev/null +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/SparkConfiguration.scala @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.metrics.adservice.configuration + +import javax.validation.constraints.NotNull + +import com.fasterxml.jackson.annotation.JsonProperty + +class SparkConfiguration { + + @NotNull + private var mode: String = _ + + @NotNull + private var masterHostPort: String = _ + + @JsonProperty + def getMode: String = mode + + @JsonProperty + def getMasterHostPort: String = masterHostPort + +} diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/db/PhoenixAnomalyStoreAccessor.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/db/PhoenixAnomalyStoreAccessor.scala index 147d1f7..53e6dee 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/db/PhoenixAnomalyStoreAccessor.scala +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/db/PhoenixAnomalyStoreAccessor.scala @@ -21,14 +21,11 @@ import java.sql.{Connection, PreparedStatement, ResultSet, SQLException} import java.util.concurrent.TimeUnit.SECONDS import org.apache.ambari.metrics.adservice.app.AnomalyDetectionAppConfig -import org.apache.ambari.metrics.adservice.common._ import org.apache.ambari.metrics.adservice.configuration.HBaseConfiguration import org.apache.ambari.metrics.adservice.metadata.{MetricDefinitionService, MetricKey} import org.apache.ambari.metrics.adservice.model.AnomalyDetectionMethod.AnomalyDetectionMethod import org.apache.ambari.metrics.adservice.model.AnomalyType.AnomalyType -import org.apache.ambari.metrics.adservice.model.{AnomalyDetectionMethod, AnomalyType, MetricAnomalyInstance} -import org.apache.ambari.metrics.adservice.subsystem.pointintime.PointInTimeAnomalyInstance -import org.apache.ambari.metrics.adservice.subsystem.trend.TrendAnomalyInstance +import org.apache.ambari.metrics.adservice.model._ import org.apache.hadoop.hbase.util.RetryCounterFactory import org.slf4j.{Logger, LoggerFactory} diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/subsystem/pointintime/PointInTimeAnomalyInstance.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/PointInTimeAnomalyInstance.scala similarity index 90% rename from ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/subsystem/pointintime/PointInTimeAnomalyInstance.scala rename to ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/PointInTimeAnomalyInstance.scala index 56ca2c1..470cc2c 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/subsystem/pointintime/PointInTimeAnomalyInstance.scala +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/PointInTimeAnomalyInstance.scala @@ -15,15 +15,13 @@ * limitations under the License. */ -package org.apache.ambari.metrics.adservice.subsystem.pointintime +package org.apache.ambari.metrics.adservice.model import java.util.Date -import org.apache.ambari.metrics.adservice.common.Season import org.apache.ambari.metrics.adservice.metadata.MetricKey import org.apache.ambari.metrics.adservice.model.AnomalyDetectionMethod.AnomalyDetectionMethod import org.apache.ambari.metrics.adservice.model.AnomalyType.AnomalyType -import org.apache.ambari.metrics.adservice.model.{AnomalyType, MetricAnomalyInstance} class PointInTimeAnomalyInstance(val metricKey: MetricKey, val timestamp: Long, diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/common/Range.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/Range.scala similarity index 96% rename from ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/common/Range.scala rename to ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/Range.scala index 003c18f..4ad35e7 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/common/Range.scala +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/Range.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ambari.metrics.adservice.common +package org.apache.ambari.metrics.adservice.model /** * Class to capture a Range in a Season. diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/common/Season.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/Season.scala similarity index 96% rename from ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/common/Season.scala rename to ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/Season.scala index f875e3b..84784bc 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/common/Season.scala +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/Season.scala @@ -15,14 +15,14 @@ * limitations under the License. */ -package org.apache.ambari.metrics.adservice.common +package org.apache.ambari.metrics.adservice.model import java.time.DayOfWeek import java.util.Calendar import javax.xml.bind.annotation.XmlRootElement -import org.apache.ambari.metrics.adservice.common.SeasonType.SeasonType +import org.apache.ambari.metrics.adservice.model.SeasonType.SeasonType import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.scala.DefaultScalaModule diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/common/SeasonType.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/SeasonType.scala similarity index 94% rename from ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/common/SeasonType.scala rename to ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/SeasonType.scala index 067972c..b510531 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/common/SeasonType.scala +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/SeasonType.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ambari.metrics.adservice.common +package org.apache.ambari.metrics.adservice.model object SeasonType extends Enumeration{ diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/common/TimeRange.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/TimeRange.scala similarity index 96% rename from ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/common/TimeRange.scala rename to ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/TimeRange.scala index 50df658..0be2564 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/common/TimeRange.scala +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/TimeRange.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ambari.metrics.adservice.common +package org.apache.ambari.metrics.adservice.model import java.util.Date diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/subsystem/trend/TrendAnomalyInstance.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/TrendAnomalyInstance.scala similarity index 90% rename from ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/subsystem/trend/TrendAnomalyInstance.scala rename to ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/TrendAnomalyInstance.scala index 7392d59..d67747c 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/subsystem/trend/TrendAnomalyInstance.scala +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/TrendAnomalyInstance.scala @@ -15,13 +15,11 @@ * limitations under the License. */ -package org.apache.ambari.metrics.adservice.subsystem.trend +package org.apache.ambari.metrics.adservice.model -import org.apache.ambari.metrics.adservice.common.{Season, TimeRange} import org.apache.ambari.metrics.adservice.metadata.MetricKey import org.apache.ambari.metrics.adservice.model.AnomalyDetectionMethod.AnomalyDetectionMethod import org.apache.ambari.metrics.adservice.model.AnomalyType.AnomalyType -import org.apache.ambari.metrics.adservice.model.{AnomalyType, MetricAnomalyInstance} case class TrendAnomalyInstance (metricKey: MetricKey, anomalousPeriod: TimeRange, diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/config.yml b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/resources/config.yaml similarity index 86% rename from ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/config.yml rename to ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/resources/config.yaml index 7de06b4..6b09499 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/config.yml +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/resources/config.yaml @@ -10,16 +10,6 @@ #See the License for the specific language governing permissions and #limitations under the License. -server: - applicationConnectors: - - type: http - port: 9999 - requestLog: - type: external - -logging: - type: external - metricDefinitionService: inputDefinitionDirectory: /etc/ambari-metrics-anomaly-detection/conf/definitionDirectory @@ -40,7 +30,6 @@ metricDefinitionDB: # Path to Level DB directory dbDirPath: /var/lib/ambari-metrics-anomaly-detection/ -#subsystemService: -# spark: -# pointInTime: -# trend: \ No newline at end of file +spark: + mode: standalone + masterHostPort: localhost:7077 \ No newline at end of file diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppConfigTest.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppConfigTest.scala index 989ba21..76391a0 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppConfigTest.scala +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppConfigTest.scala @@ -18,28 +18,31 @@ package org.apache.ambari.metrics.adservice.app import java.io.File +import java.net.URL import javax.validation.Validator import org.scalatest.FunSuite import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.datatype.guava.GuavaModule import io.dropwizard.configuration.YamlConfigurationFactory -import io.dropwizard.jackson.Jackson import io.dropwizard.jersey.validation.Validators class AnomalyDetectionAppConfigTest extends FunSuite { test("testConfiguration") { - val objectMapper: ObjectMapper = Jackson.newObjectMapper() + val classLoader = getClass.getClassLoader + val url: URL = classLoader.getResource("config.yaml") + val file = new File(url.getFile) + + val objectMapper: ObjectMapper = new ObjectMapper() + objectMapper.registerModule(new GuavaModule) val validator: Validator = Validators.newValidator val factory: YamlConfigurationFactory[AnomalyDetectionAppConfig] = new YamlConfigurationFactory[AnomalyDetectionAppConfig](classOf[AnomalyDetectionAppConfig], validator, objectMapper, "") - - val classLoader = getClass.getClassLoader - val file = new File(classLoader.getResource("config.yml").getFile) val config = factory.build(file) assert(config.isInstanceOf[AnomalyDetectionAppConfig]) @@ -48,17 +51,17 @@ class AnomalyDetectionAppConfigTest extends FunSuite { "/etc/ambari-metrics-anomaly-detection/conf/definitionDirectory") assert(config.getMetricCollectorConfiguration.getHosts == "host1,host2") - assert(config.getMetricCollectorConfiguration.getPort == "6188") assert(config.getAdServiceConfiguration.getAnomalyDataTtl == 604800) assert(config.getMetricDefinitionDBConfiguration.getDbDirPath == "/var/lib/ambari-metrics-anomaly-detection/") - assert(config.getMetricDefinitionDBConfiguration.getVerifyChecksums) - assert(!config.getMetricDefinitionDBConfiguration.getPerformParanoidChecks) + assert(config.getSparkConfiguration.getMode.equals("standalone")) + assert(config.getSparkConfiguration.getMasterHostPort.equals("localhost:7077")) + } } diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/app/DefaultADResourceSpecTest.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/app/DefaultADResourceSpecTest.scala index e38ea40..7330ff9 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/app/DefaultADResourceSpecTest.scala +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/app/DefaultADResourceSpecTest.scala @@ -36,7 +36,7 @@ class DefaultADResourceSpecTest extends FunSpec with Matchers { describe("/anomaly") { it("Must return default message") { - withAppRunning(classOf[AnomalyDetectionApp], Resources.getResource("config.yml").getPath) { rule => + withAppRunning(classOf[AnomalyDetectionApp], Resources.getResource("config.yaml").getPath) { rule => val json = client.target(s"http://localhost:${rule.getLocalPort}/anomaly";) .request().accept(APPLICATION_JSON).buildGet().invoke(classOf[String]) val dtf = java.time.format.DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm") diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/common/RangeTest.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/model/RangeTest.scala similarity index 85% rename from ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/common/RangeTest.scala rename to ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/model/RangeTest.scala index b610b97..16f4951 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/common/RangeTest.scala +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/model/RangeTest.scala @@ -15,14 +15,15 @@ * limitations under the License. */ -package org.apache.ambari.metrics.adservice.common +package org.apache.ambari.metrics.adservice.model +import org.apache.ambari.metrics.adservice.model import org.scalatest.FlatSpec class RangeTest extends FlatSpec { "A Range " should " return true for inner and boundary values" in { - val range : Range = Range(4,6) + val range : model.Range = model.Range(4,6) assert(range.withinRange(5)) assert(range.withinRange(6)) assert(range.withinRange(4)) @@ -30,7 +31,7 @@ class RangeTest extends FlatSpec { } it should "accept same lower and higher range values" in { - val range : Range = Range(4,4) + val range : model.Range = model.Range(4,4) assert(range.withinRange(4)) } diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/common/SeasonTest.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/model/SeasonTest.scala similarity index 75% rename from ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/common/SeasonTest.scala rename to ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/model/SeasonTest.scala index a823c73..a661c05 100644 --- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/common/SeasonTest.scala +++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/model/SeasonTest.scala @@ -15,10 +15,11 @@ * limitations under the License. */ -package org.apache.ambari.metrics.adservice.common +package org.apache.ambari.metrics.adservice.model import java.util.Calendar +import org.apache.ambari.metrics.adservice.model import org.scalatest.FunSuite class SeasonTest extends FunSuite { @@ -26,7 +27,7 @@ class SeasonTest extends FunSuite { test("testBelongsTo") { //Create Season for weekdays. Mon to Friday and 9AM - 5PM - var season : Season = Season(Range(Calendar.MONDAY,Calendar.FRIDAY), Range(9,17)) + var season : Season = Season(model.Range(Calendar.MONDAY,Calendar.FRIDAY), model.Range(9,17)) //Try with a timestamp on a Monday, @ 9AM. val c = Calendar.getInstance @@ -41,7 +42,7 @@ class SeasonTest extends FunSuite { assert(!season.belongsTo(c.getTimeInMillis)) //Create Season for Monday 11AM - 12Noon. - season = Season(Range(Calendar.MONDAY,Calendar.MONDAY), Range(11,12)) + season = Season(model.Range(Calendar.MONDAY,Calendar.MONDAY), model.Range(11,12)) c.set(2017, Calendar.OCTOBER, 30, 9, 0, 0) assert(!season.belongsTo(c.getTimeInMillis)) @@ -50,7 +51,7 @@ class SeasonTest extends FunSuite { //Create Season from Friday to Monday and 9AM - 5PM - season = Season(Range(Calendar.FRIDAY,Calendar.MONDAY), Range(9,17)) + season = Season(model.Range(Calendar.FRIDAY,Calendar.MONDAY), model.Range(9,17)) //Try with a timestamp on a Monday, @ 9AM. c.set(2017, Calendar.OCTOBER, 30, 9, 0, 0) @@ -67,23 +68,23 @@ class SeasonTest extends FunSuite { test("testEquals") { - var season1: Season = Season(Range(4,5), Range(2,3)) - var season2: Season = Season(Range(4,5), Range(2,3)) + var season1: Season = Season(model.Range(4,5), model.Range(2,3)) + var season2: Season = Season(model.Range(4,5), model.Range(2,3)) assert(season1 == season2) - var season3: Season = Season(Range(4,4), Range(2,3)) + var season3: Season = Season(model.Range(4,4), model.Range(2,3)) assert(!(season1 == season3)) } test("testSerialize") { - val season1 : Season = Season(Range(Calendar.MONDAY,Calendar.FRIDAY), Range(9,17)) + val season1 : Season = Season(model.Range(Calendar.MONDAY,Calendar.FRIDAY), model.Range(9,17)) val seasonString = Season.toJson(season1) val season2 : Season = Season.fromJson(seasonString) assert(season1 == season2) - val season3 : Season = Season(Range(Calendar.MONDAY,Calendar.THURSDAY), Range(9,17)) + val season3 : Season = Season(model.Range(Calendar.MONDAY,Calendar.THURSDAY), model.Range(9,17)) assert(!(season2 == season3)) } diff --git a/ambari-metrics/ambari-metrics-assembly/pom.xml b/ambari-metrics/ambari-metrics-assembly/pom.xml index 43ff285..b1a6430 100644 --- a/ambari-metrics/ambari-metrics-assembly/pom.xml +++ b/ambari-metrics/ambari-metrics-assembly/pom.xml @@ -42,6 +42,7 @@ <storm-sink-legacy.dir>${project.basedir}/../ambari-metrics-storm-sink-legacy</storm-sink-legacy.dir> <flume-sink.dir>${project.basedir}/../ambari-metrics-flume-sink</flume-sink.dir> <kafka-sink.dir>${project.basedir}/../ambari-metrics-kafka-sink</kafka-sink.dir> + <anomaly-detection.dir>${project.basedir}/../ambari-metrics-anomaly-detection-service</anomaly-detection.dir> <python.ver>python >= 2.6</python.ver> <python.devel>python-devel</python.devel> <deb.publisher>Apache</deb.publisher> @@ -56,6 +57,7 @@ <storm.sink.legacy.jar>ambari-metrics-storm-sink-legacy-with-common-${project.version}.jar</storm.sink.legacy.jar> <flume.sink.jar>ambari-metrics-flume-sink-with-common-${project.version}.jar</flume.sink.jar> <kafka.sink.jar>ambari-metrics-kafka-sink-with-common-${project.version}.jar</kafka.sink.jar> + <anomaly.detection.jar>ambari-metrics-anomaly-detection-service-${project.version}.jar</anomaly.detection.jar> </properties> <build> @@ -139,6 +141,22 @@ </configuration> </execution> <execution> + <id>anomaly-detection</id> + <phase>prepare-package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <attach>false</attach> + <finalName>ambari-metrics-anomaly-detection-${project.version}</finalName> + <appendAssemblyId>false</appendAssemblyId> + <descriptors> + <descriptor>${assemblydescriptor.anomaly-detection}</descriptor> + </descriptors> + <tarLongFileMode>gnu</tarLongFileMode> + </configuration> + </execution> + <execution> <id>hadoop-sink</id> <phase>prepare-package</phase> <goals> @@ -638,6 +656,81 @@ </configuration> </execution> + <!--ambari-metrics-anomaly-detection--> + <execution> + <id>ambari-metrics-anomaly-detection</id> + <phase>package</phase> + <goals> + <goal>rpm</goal> + </goals> + <configuration> + <name>ambari-metrics-anomaly-detection</name> + <copyright>2012, Apache Software Foundation</copyright> + <group>Development</group> + <description>Maven Recipe: RPM Package.</description> + <autoRequires>false</autoRequires> + + + <defaultFilemode>644</defaultFilemode> + <defaultDirmode>755</defaultDirmode> + <defaultUsername>root</defaultUsername> + <defaultGroupname>root</defaultGroupname> + + <postinstallScriptlet> + <scriptFile>${project.build.directory}/resources/rpm/anomaly-detection/postinstall.sh</scriptFile> + <fileEncoding>utf-8</fileEncoding> + </postinstallScriptlet> + + <mappings> + <mapping> + <!--jars--> + <directory>/usr/lib/ambari-metrics-anomaly-detection/</directory> + <sources> + <source> + <location> + ${anomaly-detection.dir}/target/ambari-metrics-anomaly-detection-service-${project.version}.jar + </location> + </source> + </sources> + </mapping> + <mapping> + <directory>/usr/lib/ambari-metrics-anomaly-detection/spark</directory> + <sources> + <source> + <location> + ${anomaly-detection.dir}/target/embedded/spark + </location> + </source> + </sources> + </mapping> + <mapping> + <directory>/usr/sbin</directory> + <filemode>755</filemode> + <username>root</username> + <groupname>root</groupname> + <directoryIncluded>false</directoryIncluded> + <sources> + <source> + <location>${anomaly-detection.dir}/conf/unix/ambari-metrics-admanager</location> + <filter>false</filter> + </source> + </sources> + </mapping> + <mapping> + <directory>/etc/ambari-metrics-anomaly-detection/conf</directory> + <configuration>true</configuration> + <sources> + <source> + <location>${anomaly-detection.dir}/conf/unix/config.yaml</location> + </source> + <source> + <location>${anomaly-detection.dir}/conf/unix/log4j.properties</location> + </source> + </sources> + </mapping> + </mappings> + </configuration> + </execution> </executions> </plugin> @@ -757,10 +850,13 @@ <path>/etc/ambari-metrics-collector/conf</path> <path>/etc/ambari-metrics-grafana/conf</path> <path>/etc/ams-hbase/conf</path> + <path>/etc/ambari-metrics-anomaly-detection/conf</path> <path>/var/run/ams-hbase</path> <path>/var/run/ambari-metrics-grafana</path> <path>/var/log/ambari-metrics-grafana</path> <path>/var/lib/ambari-metrics-collector</path> + <path>/usr/lib/ambari-metrics-anomaly-detection</path> + <path>/var/lib/ambari-metrics-anomaly-detection</path> <path>/var/lib/ambari-metrics-monitor/lib</path> <path>/var/lib/ambari-metrics-grafana</path> <path>/usr/lib/ambari-metrics-hadoop-sink</path> @@ -979,6 +1075,49 @@ </mapper> </data> + <!-- Anomaly Detection --> + <data> + <src>${anomaly-detection.dir}/target/${anomaly.detection.jar}</src> + <type>file</type> + <mapper> + <type>perm</type> + <dirmode>644</dirmode> + <prefix>/usr/lib/ambari-metrics-anomaly-detection</prefix> + </mapper> + </data> + <data> + <type>link</type> + <linkName>/usr/lib/ambari-metrics-anomaly-detection/ambari-metrics-anomaly-detection-service.jar</linkName> + <linkTarget>/usr/lib/ambari-metrics-anomaly-detection/${anomaly.detection.jar}</linkTarget> + <symlink>true</symlink> + </data> + <data> + <src>${anomaly-detection.dir}/target/embedded/spark</src> + <type>directory</type> + <mapper> + <type>perm</type> + <prefix>/usr/lib/ambari-metrics-anomaly-detection/spark</prefix> + <filemode>644</filemode> + </mapper> + </data> + <data> + <src>${anomaly-detection.dir}/conf/unix/config.yaml</src> + <type>file</type> + <mapper> + <type>perm</type> + <filemode>755</filemode> + <prefix>/etc/ambari-metrics-anomaly-detection/conf</prefix> + </mapper> + </data> + <data> + <src>${anomaly-detection.dir}/conf/unix/log4j.properties</src> + <type>file</type> + <mapper> + <type>perm</type> + <filemode>755</filemode> + <prefix>/etc/ambari-metrics-anomaly-detection/conf</prefix> + </mapper> + </data> <!-- hadoop sink --> <data> @@ -1075,6 +1214,8 @@ <assemblydescriptor.monitor>src/main/assembly/monitor.xml</assemblydescriptor.monitor> <assemblydescriptor.sink>src/main/assembly/sink.xml</assemblydescriptor.sink> <assemblydescriptor.grafana>src/main/assembly/grafana.xml</assemblydescriptor.grafana> + <assemblydescriptor.anomaly-detection>src/main/assembly/anomaly-detection.xml</assemblydescriptor.anomaly-detection> + <packagingFormat>jar</packagingFormat> </properties> <build> @@ -1354,6 +1495,13 @@ <artifactId>ambari-metrics-host-aggregator</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.apache.ambari</groupId> + <artifactId>ambari-metrics-anomaly-detection-service</artifactId> + <version>${project.version}</version> + <type>pom</type> + <optional>true</optional> + </dependency> </dependencies> diff --git a/ambari-metrics/ambari-metrics-assembly/src/main/assembly/anomaly-detection.xml b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/anomaly-detection.xml new file mode 100644 index 0000000..b05aaf3 --- /dev/null +++ b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/anomaly-detection.xml @@ -0,0 +1,60 @@ +<?xml version="1.0"?> + +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1 http://maven.apache.org/xsd/assembly-1.1.1.xsd";> + <id>anomaly-detection</id> + <formats> + <format>dir</format> + <format>tar.gz</format> + </formats> + + <fileSets> + <fileSet> + <directory>${anomaly-detection.dir}/target/embedded/spark</directory> + <outputDirectory>anomaly-detection/spark</outputDirectory> + </fileSet> + <fileSet> + <directory>${anomaly-detection.dir}/conf/unix</directory> + <outputDirectory>anomaly-detection/bin</outputDirectory> + <includes> + <include>ambari-metrics-admanager</include> + </includes> + </fileSet> + <fileSet> + <directory>${anomaly-detection.dir}/conf/unix</directory> + <outputDirectory>anomaly-detection/conf</outputDirectory> + <includes> + <include>config.yaml</include> + <include>log4j.properties</include> + </includes> + </fileSet> + </fileSets> + + <files> + <file> + <fileMode>644</fileMode> + <source>${anomaly-detection.dir}/target/ambari-metrics-anomaly-detection-service-${project.version}.jar + </source> + <outputDirectory>anomaly-detection</outputDirectory> + </file> + </files> +</assembly> \ No newline at end of file diff --git a/ambari-metrics/ambari-metrics-assembly/src/main/package/rpm/anomaly-detection/postinstall.sh b/ambari-metrics/ambari-metrics-assembly/src/main/package/rpm/anomaly-detection/postinstall.sh new file mode 100644 index 0000000..399c439 --- /dev/null +++ b/ambari-metrics/ambari-metrics-assembly/src/main/package/rpm/anomaly-detection/postinstall.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + + +ANOMALY_DETECTION_LINK_NAME="/usr/lib/ambari-metrics-anomaly-detection/ambari-metrics-anomaly-detection-service.jar" +ANOMALY_DETECTION_JAR="/usr/lib/ambari-metrics-anomaly-detection/${anomaly.detection.jar}" + +JARS=(${ANOMALY_DETECTION_JAR}) +LINKS=(${ANOMALY_DETECTION_LINK_NAME}) + +for index in ${!LINKS[*]} +do + rm -f ${LINKS[$index]} ; ln -s ${JARS[$index]} ${LINKS[$index]} +done diff --git a/ambari-metrics/pom.xml b/ambari-metrics/pom.xml index a8c71e6..98559e6 100644 --- a/ambari-metrics/pom.xml +++ b/ambari-metrics/pom.xml @@ -32,9 +32,9 @@ <module>ambari-metrics-timelineservice</module> <module>ambari-metrics-host-monitoring</module> <module>ambari-metrics-grafana</module> - <module>ambari-metrics-assembly</module> <module>ambari-metrics-host-aggregator</module> <module>ambari-metrics-anomaly-detection-service</module> + <module>ambari-metrics-assembly</module> </modules> <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> @@ -53,6 +53,8 @@ <grafana.tar>https://grafanarel.s3.amazonaws.com/builds/grafana-2.6.0.linux-x64.tar.gz</grafana.tar> <phoenix.tar>https://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.6.4.0/tars/phoenix/phoenix-4.7.0.2.6.4.0-91.tar.gz</phoenix.tar> <phoenix.folder>phoenix-4.7.0.2.6.4.0-91</phoenix.folder> + <spark.tar>http://dev.hortonworks.com.s3.amazonaws.com/HDP/centos7/3.x/BUILDS/3.0.0.0-439/tars/spark2/spark-2.1.0.3.0.0.0-439-bin-3.0.0.3.0.0.0-439.tgz</spark.tar> + <spark.folder>spark-2.1.0.3.0.0.0-439-bin-3.0.0.3.0.0.0-439</spark.folder> <resmonitor.install.dir> /usr/lib/python2.6/site-packages/resource_monitoring </resmonitor.install.dir> diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-config.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-config.xml index 2c6bbf7..9862f10 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-config.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-config.xml @@ -102,6 +102,10 @@ performParanoidChecks: false # Path to Level DB directory dbDirPath: {{ams_ad_data_dir}} + + spark: + mode: {{admanager_spark_op_mode}} + masterHostPort: {{admanager_spark_hostport}} </value> <value-attributes> <type>content</type> diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-env.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-env.xml index a79796b..91073ee 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-env.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-env.xml @@ -83,10 +83,10 @@ export AMS_AD_HEAPSIZE={{ams_admanager_heapsize}} # Anomaly Detection Manager data dir - export AMS_AD_DATA_DIR={{ams_admanager_data_dir}} + export AMS_AD_DATA_DIR={{ams_ad_data_dir}} # Anomaly Detection Manager options - export AMS_AD_OPTS=" + export AMS_AD_OPTS=$AMS_AD_OPTS {% if security_enabled %} export AMS_AD_OPTS="$AMS_AD_OPTS -Djava.security.auth.login.config={{ams_ad_jaas_config_file}}" {% endif %} @@ -95,6 +95,8 @@ export AMS_AD_GC_OPTS="-XX:+UseConcMarkSweepGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{ams_ad_log_dir}}/admanager-gc.log-`date +'%Y%m%d%H%M'`" export AMS_AD_OPTS="$AMS_AD_OPTS $AMS_AD_GC_OPTS" + # Anomaly Detection Manager data dir + export AMS_AD_STANDALONE_SPARK_ENABLED={{ams_ad_standalone_spark_enabled}} </value> <value-attributes> <type>content</type> diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-log4j.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-log4j.xml index b1f821e..ad28dcd 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-log4j.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-log4j.xml @@ -63,7 +63,7 @@ # # Define some default values that can be overridden by system properties - ams.ad.log.dir=. + ams.ad.log.dir={{ams_ad_log_dir}} ams.ad.log.file=ambari-metrics-admanager.log # Root logger option diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-spark-env.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-spark-env.xml new file mode 100644 index 0000000..3c2fb89 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-spark-env.xml @@ -0,0 +1,129 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_adding_forbidden="true"> + <property> + <name>spark_daemon_memory</name> + <value>512</value> + <description>Memory for Master, Worker and history server (default: 1G)</description> + <value-attributes> + <type>int</type> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>spark_master_port</name> + <value>6190</value> + <description>Start the master on this port</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>spark_master_webui_port</name> + <value>6180</value> + <description>Port for the master web UI</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>spark_worker_cores</name> + <value>4</value> + <description>Total number of cores to allow Spark applications to use on the machine</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>spark_worker_memory</name> + <value>2048</value> + <description>Total amount of memory to allow Spark applications to use on the machine</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>spark_worker_webui_port</name> + <value>6181</value> + <description>Port for the worker web UI</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <description>This is the jinja template for spark-env.sh file</description> + <value> + #!/usr/bin/env bash + + # This file is sourced when running various Spark programs. + # Copy it as spark-env.sh and edit that to configure Spark for your site. + + # Generic options for the daemons used in the standalone deploy mode + + export SPARK_MASTER_HOST={{hostname}} + export SPARK_MASTER_PORT={{spark_master_port}} + export SPARK_MASTER_WEBUI_PORT={{spark_master_webui_port}} + export SPARK_WORKER_CORES={{spark_worker_cores}} + export SPARK_WORKER_MEMORY={{spark_worker_memory}}m + export SPARK_WORKER_WEBUI_PORT={{spark_worker_webui_port}} + export SPARK_WORKER_DIR={{ams_ad_log_dir}} + + export SPARK_MASTER_OPTS=$SPARK_MASTER_OPTS + export SPARK_WORKER_OPTS=$SPARK_WORKER_OPTS + + export SPARK_MASTER_PORT={{spark_master_port}} + # Alternate conf dir. (Default: ${SPARK_HOME}/conf) + export SPARK_CONF_DIR={{ams_ad_conf_dir}} + + # Where log files are stored.(Default:${SPARK_HOME}/logs) + export SPARK_LOG_DIR={{ams_ad_log_dir}} + + # Where the pid file is stored. (Default: /tmp) + export SPARK_PID_DIR={{ams_ad_pid_dir}} + + #Memory for Master, Worker and history server (default: 1024MB) + export SPARK_DAEMON_MEMORY={{spark_daemon_memory}}m + + # A string representing this instance of spark.(Default: $USER) + SPARK_IDENT_STRING=$USER + + # The scheduling priority for daemons. (Default: 0) + SPARK_NICENESS=0 + + # Options read in YARN client mode + #SPARK_EXECUTOR_INSTANCES="2" #Number of workers to start (Default: 2) + #SPARK_EXECUTOR_CORES="1" #Number of cores for the workers (Default: 1). + #SPARK_EXECUTOR_MEMORY="1G" #Memory per Worker (e.g. 1000M, 2G) (Default: 1G) + #SPARK_DRIVER_MEMORY="512M" #Memory for Master (e.g. 1000M, 2G) (Default: 512 Mb) + #SPARK_YARN_APP_NAME="spark" #The name of your application (Default: Spark) + #SPARK_YARN_QUEUE="default" #The hadoop queue to use for allocation requests (Default: default) + #SPARK_YARN_DIST_FILES="" #Comma separated list of files to be distributed with the job. + #SPARK_YARN_DIST_ARCHIVES="" #Comma separated list of archives to be distributed with the job. + + #export HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}} + #export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-{{hadoop_conf_dir}}} + + # The java implementation to use. + export JAVA_HOME={{java_home}} + + #HDP Version + export HDP_VERSION=3.0.0 + + </value> + <value-attributes> + <type>content</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> +</configuration> diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml index 41e278d..bcf6268 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml @@ -138,6 +138,7 @@ <config-type>ams-admanager-config</config-type> <config-type>ams-admanager-env</config-type> <config-type>ams-admanager-log4j</config-type> + <config-type>ams-admanager-spark-env</config-type> </configuration-dependencies> <logs> <log> diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py index 7ab0547..4c6951a 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py @@ -555,14 +555,26 @@ def ams(name=None, action=None): if (params.ams_ad_log4j_props != None): File(os.path.join(params.ams_ad_conf_dir, "log4j.properties"), owner=params.ams_user, - content=params.ams_ad_log4j_props + content=InlineTemplate(params.ams_ad_log4j_props) ) + File(format("{ams_ad_conf_dir}/ams-admanager-spark-env.sh"), + owner=params.ams_user, + group=params.user_group, + content=InlineTemplate(params.ams_ad_spark_env_sh_template) + ) + if action != 'stop': for dir in ams_ad_directories: Execute(('chown', '-R', params.ams_user, dir), sudo=True ) + Execute(('chmod', '-R', '755', format("{ams_admanager_lib_dir}/spark/bin")), + sudo = True, + ) + Execute(('chmod', '-R', '755', format("{ams_admanager_lib_dir}/spark/sbin")), + sudo = True, + ) def is_spnego_enabled(params): if 'core-site' in params.config['configurations'] \ diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams_admanager.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams_admanager.py index 96c4454..33c8832 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams_admanager.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams_admanager.py @@ -45,7 +45,7 @@ class AmsADManager(Script): Execute(start_cmd, user=params.ams_user ) - pidfile = format("{ams_ad_pid_dir}/admanager.pid") + pidfile = format("{ams_ad_pid_dir}/ambari-metrics-admanager.pid") if not sudo.path_exists(pidfile): Logger.warning("Pid file doesn't exist after starting of the component.") else: @@ -57,7 +57,7 @@ class AmsADManager(Script): env.set_params(params) self.configure(env, action = 'stop') Execute((format("{ams_admanager_script}"), 'stop'), - sudo=True + user=params.ams_user ) def status(self, env): diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py index 40d3db6..dd2a686 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py @@ -154,6 +154,7 @@ def get_ambari_version(): pass return ambari_version +hostname = config['agentLevelParams']['hostname'] ams_collector_log_dir = config['configurations']['ams-env']['metrics_collector_log_dir'] ams_collector_conf_dir = "/etc/ambari-metrics-collector/conf" @@ -199,6 +200,26 @@ ams_admanager_config_template = config['configurations']['ams-admanager-config'] ams_admanager_script = "/usr/sbin/ambari-metrics-admanager" ams_admanager_port = config['configurations']['ams-admanager-config']['ambari.metrics.admanager.application.port'] ams_admanager_heapsize = config['configurations']['ams-admanager-env']['ams_admanager_heapsize'] +ams_admanager_lib_dir = "/usr/lib/ambari-metrics-anomaly-detection" +ams_admanager_jar = format("{ams_admanager_lib_dir}/ambari-metrics-anomaly-detection-service-*.jar") +ams_ad_log_max_backup_size = default('configurations/ams-admanager-log4j/ams_ad_log_max_backup_size',80) +ams_ad_log_number_of_backup_files = default('configurations/ams-admanager-log4j/ams_ad_log_number_of_backup_files',60) + +admanager_spark_op_mode = config['configurations']['ams-admanager-config']['ambari.metrics.admanager.spark.operation.mode'] +ams_ad_spark_env_sh_template = config['configurations']['ams-admanager-spark-env']['content'] +spark_master_port = default("/configurations/ams-admanager-spark-env/spark_master_port", 6190) +spark_master_webui_port = default("/configurations/ams-admanager-spark-env/spark_master_webui_port", 6180) +spark_worker_cores = default("/configurations/ams-admanager-spark-env/spark_worker_cores", 4) +spark_worker_memory = default("/configurations/ams-admanager-spark-env/spark_worker_memory", 2048) +spark_worker_webui_port = default("/configurations/ams-admanager-spark-env/spark_worker_webui_port", 6181) +spark_daemon_memory = default("/configurations/ams-admanager-spark-env/spark_daemon_memory", 1024) + +if admanager_spark_op_mode == 'spark-on-yarn': + admanager_spark_hostport = hostname + ":" + spark_master_port #TODO : Fix for spark on yarn mode. + ams_ad_standalone_spark_enabled = False +else: + admanager_spark_hostport = hostname + ":" + spark_master_port + ams_ad_standalone_spark_enabled = True if (('ams-admanager-log4j' in config['configurations']) and ('content' in config['configurations']['ams-admanager-log4j'])): ams_ad_log4j_props = config['configurations']['ams-admanager-log4j']['content'] @@ -289,7 +310,6 @@ else: hbase_heapsize = master_heapsize max_open_files_limit = default("/configurations/ams-hbase-env/max_open_files_limit", "32768") -hostname = config['agentLevelParams']['hostname'] cluster_zookeeper_quorum_hosts = ",".join(config['clusterHostInfo']['zookeeper_server_hosts']) if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']: diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status_params.py index 3373592..347c290 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status_params.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status_params.py @@ -37,7 +37,7 @@ ams_ad_pid_dir = config['configurations']['ams-admanager-env']['ams_ad_pid_dir'] monitor_pid_file = format("{ams_monitor_pid_dir}/ambari-metrics-monitor.pid") grafana_pid_file = format("{ams_grafana_pid_dir}/grafana-server.pid") -ams_ad_pid_file = format("{ams_ad_pid_dir}/admanager.pid") +ams_ad_pid_file = format("{ams_ad_pid_dir}/ambari-metrics-admanager.pid") security_enabled = config['configurations']['cluster-env']['security_enabled'] ams_hbase_conf_dir = format("{hbase_conf_dir}") -- To stop receiving notification emails like this one, please contact [email protected].
