Repository: ambari Updated Branches: refs/heads/branch-3.0-ams ba9be8028 -> cb0341e07
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb0341e0/ambari-metrics/ambari-metrics-assembly/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-assembly/pom.xml b/ambari-metrics/ambari-metrics-assembly/pom.xml index d9875ce..0c7b786 100644 --- a/ambari-metrics/ambari-metrics-assembly/pom.xml +++ b/ambari-metrics/ambari-metrics-assembly/pom.xml @@ -42,6 +42,7 @@ <storm-sink-legacy.dir>${project.basedir}/../ambari-metrics-storm-sink-legacy</storm-sink-legacy.dir> <flume-sink.dir>${project.basedir}/../ambari-metrics-flume-sink</flume-sink.dir> <kafka-sink.dir>${project.basedir}/../ambari-metrics-kafka-sink</kafka-sink.dir> + <anomaly-detection.dir>${project.basedir}/../ambari-metrics-anomaly-detection-service</anomaly-detection.dir> <python.ver>python >= 2.6</python.ver> <python.devel>python-devel</python.devel> <deb.publisher>Apache</deb.publisher> @@ -56,6 +57,7 @@ <storm.sink.legacy.jar>ambari-metrics-storm-sink-legacy-with-common-${project.version}.jar</storm.sink.legacy.jar> <flume.sink.jar>ambari-metrics-flume-sink-with-common-${project.version}.jar</flume.sink.jar> <kafka.sink.jar>ambari-metrics-kafka-sink-with-common-${project.version}.jar</kafka.sink.jar> + <anomaly.detection.jar>ambari-metrics-anomaly-detection-service-${project.version}.jar</anomaly.detection.jar> </properties> <build> @@ -139,6 +141,22 @@ </configuration> </execution> <execution> + <id>anomaly-detection</id> + <phase>prepare-package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <attach>false</attach> + <finalName>ambari-metrics-anomaly-detection-${project.version}</finalName> + <appendAssemblyId>false</appendAssemblyId> + <descriptors> + <descriptor>${assemblydescriptor.anomaly-detection}</descriptor> + </descriptors> + <tarLongFileMode>gnu</tarLongFileMode> + </configuration> + </execution> + <execution> <id>hadoop-sink</id> <phase>prepare-package</phase> <goals> @@ -637,6 +655,81 @@ </configuration> </execution> + <!--ambari-metrics-anomaly-detection--> + <execution> + <id>ambari-metrics-anomaly-detection</id> + <phase>package</phase> + <goals> + <goal>rpm</goal> + </goals> + <configuration> + <name>ambari-metrics-anomaly-detection</name> + <copyright>2012, Apache Software Foundation</copyright> + <group>Development</group> + <description>Maven Recipe: RPM Package.</description> + <autoRequires>false</autoRequires> + + + <defaultFilemode>644</defaultFilemode> + <defaultDirmode>755</defaultDirmode> + <defaultUsername>root</defaultUsername> + <defaultGroupname>root</defaultGroupname> + + <postinstallScriptlet> + <scriptFile>${project.build.directory}/resources/rpm/anomaly-detection/postinstall.sh</scriptFile> + <fileEncoding>utf-8</fileEncoding> + </postinstallScriptlet> + + <mappings> + <mapping> + <!--jars--> + <directory>/usr/lib/ambari-metrics-anomaly-detection/</directory> + <sources> + <source> + <location> + ${anomaly-detection.dir}/target/ambari-metrics-anomaly-detection-service-${project.version}.jar + </location> + </source> + </sources> + </mapping> + <mapping> + <directory>/usr/lib/ambari-metrics-anomaly-detection/spark</directory> + <sources> + <source> + <location> + ${anomaly-detection.dir}/target/embedded/spark + </location> + </source> + </sources> + </mapping> + <mapping> + <directory>/usr/sbin</directory> + <filemode>755</filemode> + <username>root</username> + <groupname>root</groupname> + <directoryIncluded>false</directoryIncluded> + <sources> + <source> + <location>${anomaly-detection.dir}/conf/unix/ambari-metrics-admanager</location> + <filter>false</filter> + </source> + </sources> + </mapping> + <mapping> + <directory>/etc/ambari-metrics-anomaly-detection/conf</directory> + <configuration>true</configuration> + <sources> + <source> + <location>${anomaly-detection.dir}/conf/unix/config.yaml</location> + </source> + <source> + <location>${anomaly-detection.dir}/conf/unix/log4j.properties</location> + </source> + </sources> + </mapping> + </mappings> + </configuration> + </execution> </executions> </plugin> @@ -756,10 +849,13 @@ <path>/etc/ambari-metrics-collector/conf</path> <path>/etc/ambari-metrics-grafana/conf</path> <path>/etc/ams-hbase/conf</path> + <path>/etc/ambari-metrics-anomaly-detection/conf</path> <path>/var/run/ams-hbase</path> <path>/var/run/ambari-metrics-grafana</path> <path>/var/log/ambari-metrics-grafana</path> <path>/var/lib/ambari-metrics-collector</path> + <path>/usr/lib/ambari-metrics-anomaly-detection</path> + <path>/var/lib/ambari-metrics-anomaly-detection</path> <path>/var/lib/ambari-metrics-monitor/lib</path> <path>/var/lib/ambari-metrics-grafana</path> <path>/usr/lib/ambari-metrics-hadoop-sink</path> @@ -978,6 +1074,49 @@ </mapper> </data> + <!-- Anomaly Detection --> + <data> + <src>${anomaly-detection.dir}/target/${anomaly.detection.jar}</src> + <type>file</type> + <mapper> + <type>perm</type> + <dirmode>644</dirmode> + <prefix>/usr/lib/ambari-metrics-anomaly-detection</prefix> + </mapper> + </data> + <data> + <type>link</type> + <linkName>/usr/lib/ambari-metrics-anomaly-detection/ambari-metrics-anomaly-detection-service.jar</linkName> + <linkTarget>/usr/lib/ambari-metrics-anomaly-detection/${anomaly.detection.jar}</linkTarget> + <symlink>true</symlink> + </data> + <data> + <src>${anomaly-detection.dir}/target/embedded/spark</src> + <type>directory</type> + <mapper> + <type>perm</type> + <prefix>/usr/lib/ambari-metrics-anomaly-detection/spark</prefix> + <filemode>644</filemode> + </mapper> + </data> + <data> + <src>${anomaly-detection.dir}/conf/unix/config.yaml</src> + <type>file</type> + <mapper> + <type>perm</type> + <filemode>755</filemode> + <prefix>/etc/ambari-metrics-anomaly-detection/conf</prefix> + </mapper> + </data> + <data> + <src>${anomaly-detection.dir}/conf/unix/log4j.properties</src> + <type>file</type> + <mapper> + <type>perm</type> + <filemode>755</filemode> + <prefix>/etc/ambari-metrics-anomaly-detection/conf</prefix> + </mapper> + </data> <!-- hadoop sink --> <data> @@ -1074,6 +1213,8 @@ <assemblydescriptor.monitor>src/main/assembly/monitor.xml</assemblydescriptor.monitor> <assemblydescriptor.sink>src/main/assembly/sink.xml</assemblydescriptor.sink> <assemblydescriptor.grafana>src/main/assembly/grafana.xml</assemblydescriptor.grafana> + <assemblydescriptor.anomaly-detection>src/main/assembly/anomaly-detection.xml</assemblydescriptor.anomaly-detection> + <packagingFormat>jar</packagingFormat> </properties> <build> @@ -1353,6 +1494,13 @@ <artifactId>ambari-metrics-host-aggregator</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.apache.ambari</groupId> + <artifactId>ambari-metrics-anomaly-detection-service</artifactId> + <version>${project.version}</version> + <type>pom</type> + <optional>true</optional> + </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/ambari/blob/cb0341e0/ambari-metrics/ambari-metrics-assembly/src/main/assembly/anomaly-detection.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-assembly/src/main/assembly/anomaly-detection.xml b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/anomaly-detection.xml new file mode 100644 index 0000000..b05aaf3 --- /dev/null +++ b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/anomaly-detection.xml @@ -0,0 +1,60 @@ +<?xml version="1.0"?> + +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1 http://maven.apache.org/xsd/assembly-1.1.1.xsd";> + <id>anomaly-detection</id> + <formats> + <format>dir</format> + <format>tar.gz</format> + </formats> + + <fileSets> + <fileSet> + <directory>${anomaly-detection.dir}/target/embedded/spark</directory> + <outputDirectory>anomaly-detection/spark</outputDirectory> + </fileSet> + <fileSet> + <directory>${anomaly-detection.dir}/conf/unix</directory> + <outputDirectory>anomaly-detection/bin</outputDirectory> + <includes> + <include>ambari-metrics-admanager</include> + </includes> + </fileSet> + <fileSet> + <directory>${anomaly-detection.dir}/conf/unix</directory> + <outputDirectory>anomaly-detection/conf</outputDirectory> + <includes> + <include>config.yaml</include> + <include>log4j.properties</include> + </includes> + </fileSet> + </fileSets> + + <files> + <file> + <fileMode>644</fileMode> + <source>${anomaly-detection.dir}/target/ambari-metrics-anomaly-detection-service-${project.version}.jar + </source> + <outputDirectory>anomaly-detection</outputDirectory> + </file> + </files> +</assembly> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/cb0341e0/ambari-metrics/ambari-metrics-assembly/src/main/package/rpm/anomaly-detection/postinstall.sh ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-assembly/src/main/package/rpm/anomaly-detection/postinstall.sh b/ambari-metrics/ambari-metrics-assembly/src/main/package/rpm/anomaly-detection/postinstall.sh new file mode 100644 index 0000000..399c439 --- /dev/null +++ b/ambari-metrics/ambari-metrics-assembly/src/main/package/rpm/anomaly-detection/postinstall.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + + +ANOMALY_DETECTION_LINK_NAME="/usr/lib/ambari-metrics-anomaly-detection/ambari-metrics-anomaly-detection-service.jar" +ANOMALY_DETECTION_JAR="/usr/lib/ambari-metrics-anomaly-detection/${anomaly.detection.jar}" + +JARS=(${ANOMALY_DETECTION_JAR}) +LINKS=(${ANOMALY_DETECTION_LINK_NAME}) + +for index in ${!LINKS[*]} +do + rm -f ${LINKS[$index]} ; ln -s ${JARS[$index]} ${LINKS[$index]} +done http://git-wip-us.apache.org/repos/asf/ambari/blob/cb0341e0/ambari-metrics/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/pom.xml b/ambari-metrics/pom.xml index 4848084..ff81060 100644 --- a/ambari-metrics/pom.xml +++ b/ambari-metrics/pom.xml @@ -32,9 +32,9 @@ <module>ambari-metrics-timelineservice</module> <module>ambari-metrics-host-monitoring</module> <module>ambari-metrics-grafana</module> - <module>ambari-metrics-assembly</module> <module>ambari-metrics-host-aggregator</module> <module>ambari-metrics-anomaly-detection-service</module> + <module>ambari-metrics-assembly</module> </modules> <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> @@ -53,6 +53,8 @@ <grafana.tar>https://grafanarel.s3.amazonaws.com/builds/grafana-2.6.0.linux-x64.tar.gz</grafana.tar> <phoenix.tar>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.6.0.3/tars/phoenix-4.7.0.2.6.0.3-8.tar.gz</phoenix.tar> <phoenix.folder>phoenix-4.7.0.2.6.0.3-8</phoenix.folder> + <spark.tar>http://dev.hortonworks.com.s3.amazonaws.com/HDP/centos7/3.x/BUILDS/3.0.0.0-439/tars/spark2/spark-2.1.0.3.0.0.0-439-bin-3.0.0.3.0.0.0-439.tgz</spark.tar> + <spark.folder>spark-2.1.0.3.0.0.0-439-bin-3.0.0.3.0.0.0-439</spark.folder> <resmonitor.install.dir> /usr/lib/python2.6/site-packages/resource_monitoring </resmonitor.install.dir> http://git-wip-us.apache.org/repos/asf/ambari/blob/cb0341e0/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-config.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-config.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-config.xml index 2c6bbf7..9862f10 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-config.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-config.xml @@ -102,6 +102,10 @@ performParanoidChecks: false # Path to Level DB directory dbDirPath: {{ams_ad_data_dir}} + + spark: + mode: {{admanager_spark_op_mode}} + masterHostPort: {{admanager_spark_hostport}} </value> <value-attributes> <type>content</type> http://git-wip-us.apache.org/repos/asf/ambari/blob/cb0341e0/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-env.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-env.xml index a79796b..91073ee 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-env.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-env.xml @@ -83,10 +83,10 @@ export AMS_AD_HEAPSIZE={{ams_admanager_heapsize}} # Anomaly Detection Manager data dir - export AMS_AD_DATA_DIR={{ams_admanager_data_dir}} + export AMS_AD_DATA_DIR={{ams_ad_data_dir}} # Anomaly Detection Manager options - export AMS_AD_OPTS=" + export AMS_AD_OPTS=$AMS_AD_OPTS {% if security_enabled %} export AMS_AD_OPTS="$AMS_AD_OPTS -Djava.security.auth.login.config={{ams_ad_jaas_config_file}}" {% endif %} @@ -95,6 +95,8 @@ export AMS_AD_GC_OPTS="-XX:+UseConcMarkSweepGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{ams_ad_log_dir}}/admanager-gc.log-`date +'%Y%m%d%H%M'`" export AMS_AD_OPTS="$AMS_AD_OPTS $AMS_AD_GC_OPTS" + # Anomaly Detection Manager data dir + export AMS_AD_STANDALONE_SPARK_ENABLED={{ams_ad_standalone_spark_enabled}} </value> <value-attributes> <type>content</type> http://git-wip-us.apache.org/repos/asf/ambari/blob/cb0341e0/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-log4j.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-log4j.xml index b1f821e..ad28dcd 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-log4j.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-log4j.xml @@ -63,7 +63,7 @@ # # Define some default values that can be overridden by system properties - ams.ad.log.dir=. + ams.ad.log.dir={{ams_ad_log_dir}} ams.ad.log.file=ambari-metrics-admanager.log # Root logger option http://git-wip-us.apache.org/repos/asf/ambari/blob/cb0341e0/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-spark-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-spark-env.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-spark-env.xml new file mode 100644 index 0000000..3c2fb89 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-spark-env.xml @@ -0,0 +1,129 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_adding_forbidden="true"> + <property> + <name>spark_daemon_memory</name> + <value>512</value> + <description>Memory for Master, Worker and history server (default: 1G)</description> + <value-attributes> + <type>int</type> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>spark_master_port</name> + <value>6190</value> + <description>Start the master on this port</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>spark_master_webui_port</name> + <value>6180</value> + <description>Port for the master web UI</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>spark_worker_cores</name> + <value>4</value> + <description>Total number of cores to allow Spark applications to use on the machine</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>spark_worker_memory</name> + <value>2048</value> + <description>Total amount of memory to allow Spark applications to use on the machine</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>spark_worker_webui_port</name> + <value>6181</value> + <description>Port for the worker web UI</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <description>This is the jinja template for spark-env.sh file</description> + <value> + #!/usr/bin/env bash + + # This file is sourced when running various Spark programs. + # Copy it as spark-env.sh and edit that to configure Spark for your site. + + # Generic options for the daemons used in the standalone deploy mode + + export SPARK_MASTER_HOST={{hostname}} + export SPARK_MASTER_PORT={{spark_master_port}} + export SPARK_MASTER_WEBUI_PORT={{spark_master_webui_port}} + export SPARK_WORKER_CORES={{spark_worker_cores}} + export SPARK_WORKER_MEMORY={{spark_worker_memory}}m + export SPARK_WORKER_WEBUI_PORT={{spark_worker_webui_port}} + export SPARK_WORKER_DIR={{ams_ad_log_dir}} + + export SPARK_MASTER_OPTS=$SPARK_MASTER_OPTS + export SPARK_WORKER_OPTS=$SPARK_WORKER_OPTS + + export SPARK_MASTER_PORT={{spark_master_port}} + # Alternate conf dir. (Default: ${SPARK_HOME}/conf) + export SPARK_CONF_DIR={{ams_ad_conf_dir}} + + # Where log files are stored.(Default:${SPARK_HOME}/logs) + export SPARK_LOG_DIR={{ams_ad_log_dir}} + + # Where the pid file is stored. (Default: /tmp) + export SPARK_PID_DIR={{ams_ad_pid_dir}} + + #Memory for Master, Worker and history server (default: 1024MB) + export SPARK_DAEMON_MEMORY={{spark_daemon_memory}}m + + # A string representing this instance of spark.(Default: $USER) + SPARK_IDENT_STRING=$USER + + # The scheduling priority for daemons. (Default: 0) + SPARK_NICENESS=0 + + # Options read in YARN client mode + #SPARK_EXECUTOR_INSTANCES="2" #Number of workers to start (Default: 2) + #SPARK_EXECUTOR_CORES="1" #Number of cores for the workers (Default: 1). + #SPARK_EXECUTOR_MEMORY="1G" #Memory per Worker (e.g. 1000M, 2G) (Default: 1G) + #SPARK_DRIVER_MEMORY="512M" #Memory for Master (e.g. 1000M, 2G) (Default: 512 Mb) + #SPARK_YARN_APP_NAME="spark" #The name of your application (Default: Spark) + #SPARK_YARN_QUEUE="default" #The hadoop queue to use for allocation requests (Default: default) + #SPARK_YARN_DIST_FILES="" #Comma separated list of files to be distributed with the job. + #SPARK_YARN_DIST_ARCHIVES="" #Comma separated list of archives to be distributed with the job. + + #export HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}} + #export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-{{hadoop_conf_dir}}} + + # The java implementation to use. + export JAVA_HOME={{java_home}} + + #HDP Version + export HDP_VERSION=3.0.0 + + </value> + <value-attributes> + <type>content</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/cb0341e0/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml index a80340c..9238103 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml @@ -138,6 +138,7 @@ <config-type>ams-admanager-config</config-type> <config-type>ams-admanager-env</config-type> <config-type>ams-admanager-log4j</config-type> + <config-type>ams-admanager-spark-env</config-type> </configuration-dependencies> <logs> <log> http://git-wip-us.apache.org/repos/asf/ambari/blob/cb0341e0/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py index b22a870..2204eeb 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py @@ -550,14 +550,26 @@ def ams(name=None, action=None): if (params.ams_ad_log4j_props != None): File(os.path.join(params.ams_ad_conf_dir, "log4j.properties"), owner=params.ams_user, - content=params.ams_ad_log4j_props + content=InlineTemplate(params.ams_ad_log4j_props) ) + File(format("{ams_ad_conf_dir}/ams-admanager-spark-env.sh"), + owner=params.ams_user, + group=params.user_group, + content=InlineTemplate(params.ams_ad_spark_env_sh_template) + ) + if action != 'stop': for dir in ams_ad_directories: Execute(('chown', '-R', params.ams_user, dir), sudo=True ) + Execute(('chmod', '-R', '755', format("{ams_admanager_lib_dir}/spark/bin")), + sudo = True, + ) + Execute(('chmod', '-R', '755', format("{ams_admanager_lib_dir}/spark/sbin")), + sudo = True, + ) def is_spnego_enabled(params): if 'core-site' in params.config['configurations'] \ http://git-wip-us.apache.org/repos/asf/ambari/blob/cb0341e0/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams_admanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams_admanager.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams_admanager.py index 96c4454..33c8832 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams_admanager.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams_admanager.py @@ -45,7 +45,7 @@ class AmsADManager(Script): Execute(start_cmd, user=params.ams_user ) - pidfile = format("{ams_ad_pid_dir}/admanager.pid") + pidfile = format("{ams_ad_pid_dir}/ambari-metrics-admanager.pid") if not sudo.path_exists(pidfile): Logger.warning("Pid file doesn't exist after starting of the component.") else: @@ -57,7 +57,7 @@ class AmsADManager(Script): env.set_params(params) self.configure(env, action = 'stop') Execute((format("{ams_admanager_script}"), 'stop'), - sudo=True + user=params.ams_user ) def status(self, env): http://git-wip-us.apache.org/repos/asf/ambari/blob/cb0341e0/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py index b7004b8..a18b8f6 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py @@ -154,6 +154,7 @@ def get_ambari_version(): pass return ambari_version +hostname = config["hostname"] ams_collector_log_dir = config['configurations']['ams-env']['metrics_collector_log_dir'] ams_collector_conf_dir = "/etc/ambari-metrics-collector/conf" @@ -199,6 +200,26 @@ ams_admanager_config_template = config['configurations']['ams-admanager-config'] ams_admanager_script = "/usr/sbin/ambari-metrics-admanager" ams_admanager_port = config['configurations']['ams-admanager-config']['ambari.metrics.admanager.application.port'] ams_admanager_heapsize = config['configurations']['ams-admanager-env']['ams_admanager_heapsize'] +ams_admanager_lib_dir = "/usr/lib/ambari-metrics-anomaly-detection" +ams_admanager_jar = format("{ams_admanager_lib_dir}/ambari-metrics-anomaly-detection-service-*.jar") +ams_ad_log_max_backup_size = default('configurations/ams-admanager-log4j/ams_ad_log_max_backup_size',80) +ams_ad_log_number_of_backup_files = default('configurations/ams-admanager-log4j/ams_ad_log_number_of_backup_files',60) + +admanager_spark_op_mode = config['configurations']['ams-admanager-config']['ambari.metrics.admanager.spark.operation.mode'] +ams_ad_spark_env_sh_template = config['configurations']['ams-admanager-spark-env']['content'] +spark_master_port = default("/configurations/ams-admanager-spark-env/spark_master_port", 6190) +spark_master_webui_port = default("/configurations/ams-admanager-spark-env/spark_master_webui_port", 6180) +spark_worker_cores = default("/configurations/ams-admanager-spark-env/spark_worker_cores", 4) +spark_worker_memory = default("/configurations/ams-admanager-spark-env/spark_worker_memory", 2048) +spark_worker_webui_port = default("/configurations/ams-admanager-spark-env/spark_worker_webui_port", 6181) +spark_daemon_memory = default("/configurations/ams-admanager-spark-env/spark_daemon_memory", 1024) + +if admanager_spark_op_mode == 'spark-on-yarn': + admanager_spark_hostport = hostname + ":" + spark_master_port #TODO : Fix for spark on yarn mode. + ams_ad_standalone_spark_enabled = False +else: + admanager_spark_hostport = hostname + ":" + spark_master_port + ams_ad_standalone_spark_enabled = True if (('ams-admanager-log4j' in config['configurations']) and ('content' in config['configurations']['ams-admanager-log4j'])): ams_ad_log4j_props = config['configurations']['ams-admanager-log4j']['content'] @@ -289,7 +310,6 @@ else: hbase_heapsize = master_heapsize max_open_files_limit = default("/configurations/ams-hbase-env/max_open_files_limit", "32768") -hostname = config["hostname"] cluster_zookeeper_quorum_hosts = ",".join(config['clusterHostInfo']['zookeeper_hosts']) if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']: http://git-wip-us.apache.org/repos/asf/ambari/blob/cb0341e0/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status_params.py index c01b368..334403c 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status_params.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status_params.py @@ -37,7 +37,7 @@ ams_ad_pid_dir = config['configurations']['ams-admanager-env']['ams_ad_pid_dir'] monitor_pid_file = format("{ams_monitor_pid_dir}/ambari-metrics-monitor.pid") grafana_pid_file = format("{ams_grafana_pid_dir}/grafana-server.pid") -ams_ad_pid_file = format("{ams_ad_pid_dir}/admanager.pid") +ams_ad_pid_file = format("{ams_ad_pid_dir}/ambari-metrics-admanager.pid") security_enabled = config['configurations']['cluster-env']['security_enabled'] ams_hbase_conf_dir = format("{hbase_conf_dir}")
