Repository: ambari Updated Branches: refs/heads/branch-2.1 b74b2dc05 -> 15c0ce192
AMBARI-13513. Add cmd option config to spark thrift server (Judy Nash via alejandro) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/15c0ce19 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/15c0ce19 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/15c0ce19 Branch: refs/heads/branch-2.1 Commit: 15c0ce1929fc30f55552836ad02f174b1ee8ba6e Parents: b74b2dc Author: Alejandro Fernandez <[email protected]> Authored: Mon Oct 26 13:56:14 2015 -0700 Committer: Alejandro Fernandez <[email protected]> Committed: Mon Oct 26 13:56:14 2015 -0700 ---------------------------------------------------------------------- .../SPARK/1.2.0.2.2/configuration/spark-env.xml | 6 ++++++ .../common-services/SPARK/1.2.0.2.2/package/scripts/params.py | 3 +++ .../SPARK/1.2.0.2.2/package/scripts/spark_service.py | 2 +- .../test/python/stacks/2.3/SPARK/test_spark_thrift_server.py | 2 +- .../src/test/python/stacks/2.3/configs/spark_default.json | 5 +++-- 5 files changed, 14 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/15c0ce19/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-env.xml b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-env.xml index 79e3b52..25fcc6b 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-env.xml +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-env.xml @@ -100,4 +100,10 @@ fi </value> </property> + <property> + <name>spark_thrift_cmd_opts</name> + <description>additional spark thrift server commandline options</description> + <value> + </value> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/15c0ce19/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py index be81f59..50b241f 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py @@ -161,9 +161,12 @@ if security_enabled: # thrift server support - available on HDP 2.3 or higher spark_thrift_sparkconf = None +spark_thrift_cmd_opts_properties = '' if version and compare_versions(format_hdp_stack_version(version), '2.3.2.0') >= 0 \ and 'spark-thrift-sparkconf' in config['configurations']: spark_thrift_sparkconf = config['configurations']['spark-thrift-sparkconf'] + spark_thrift_cmd_opts_properties = config['configurations']['spark-env']['spark_thrift_cmd_opts'] + if is_hive_installed: spark_hive_properties.update(config['configurations']['spark-hive-site-override']) http://git-wip-us.apache.org/repos/asf/ambari/blob/15c0ce19/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py index 68a395b..b274c61 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py @@ -51,7 +51,7 @@ def spark_service(name, action): elif name == 'sparkthriftserver': thriftserver_no_op_test = format( 'ls {spark_thrift_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_thrift_server_pid_file}` >/dev/null 2>&1') - Execute(format('{spark_thrift_server_start} --properties-file {spark_thrift_server_conf_file}'), + Execute(format('{spark_thrift_server_start} --properties-file {spark_thrift_server_conf_file} {spark_thrift_cmd_opts_properties}'), user=params.spark_user, environment={'JAVA_HOME': params.java_home}, not_if=thriftserver_no_op_test http://git-wip-us.apache.org/repos/asf/ambari/blob/15c0ce19/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py b/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py index f5f12f2..d6ac468 100644 --- a/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py +++ b/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py @@ -53,7 +53,7 @@ class TestSparkThriftServer(RMFTestCase): target = RMFTestCase.TARGET_COMMON_SERVICES ) self.assert_configure_default() - self.assertResourceCalled('Execute', '/usr/hdp/current/spark-client/sbin/start-thriftserver.sh --properties-file /usr/hdp/current/spark-client/conf/spark-thrift-sparkconf.conf', + self.assertResourceCalled('Execute', '/usr/hdp/current/spark-client/sbin/start-thriftserver.sh --properties-file /usr/hdp/current/spark-client/conf/spark-thrift-sparkconf.conf --driver-memory 1g', environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'}, not_if = 'ls /var/run/spark/spark-spark-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2-1.pid >/dev/null 2>&1 && ps -p `cat /var/run/spark/spark-spark-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2-1.pid` >/dev/null 2>&1', user = 'spark', http://git-wip-us.apache.org/repos/asf/ambari/blob/15c0ce19/ambari-server/src/test/python/stacks/2.3/configs/spark_default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/configs/spark_default.json b/ambari-server/src/test/python/stacks/2.3/configs/spark_default.json index 9f3fb90..84cd31f 100644 --- a/ambari-server/src/test/python/stacks/2.3/configs/spark_default.json +++ b/ambari-server/src/test/python/stacks/2.3/configs/spark_default.json @@ -21,7 +21,7 @@ "serviceName": "SLIDER", "role": "SLIDER", "commandParams": { - "version": "2.3.1.0-2067", + "version": "2.3.2.0-2067", "command_timeout": "300", "service_package_folder": "OOZIE", "script_type": "PYTHON", @@ -160,7 +160,8 @@ "spark_pid_dir": "/var/run/spark", "spark_log_dir": "/var/log/spark", "spark_group": "spark", - "spark_user": "spark" + "spark_user": "spark", + "spark_thrift_cmd_opts": "--driver-memory 1g" }, "spark-metrics-properties": { "content": "\n# syntax: [instance].sink|source.[name].[options]=[value]\n\n# This file configures Spark's internal metrics system. The metrics system is\n# divided into instances which correspond to internal components.\n# Each instance can be configured to report its metrics to one or more sinks.\n# Accepted values for [instance] are \"master\", \"worker\", \"executor\", \"driver\",\n# and \"applications\". A wild card \"*\" can be used as an instance name, in\n# which case all instances will inherit the supplied property.\n#\n# Within an instance, a \"source\" specifies a particular set of grouped metrics.\n# there are two kinds of sources:\n# 1. Spark internal sources, like MasterSource, WorkerSource, etc, which will\n# collect a Spark component's internal state. Each instance is paired with a\n# Spark source that is added automatically.\n# 2. Common sources, like JvmSource, which will collect low level state.\n# These can be added through configuration options and ar e then loaded\n# using reflection.\n#\n# A \"sink\" specifies where metrics are delivered to. Each instance can be\n# assigned one or more sinks.\n#\n# The sink|source field specifies whether the property relates to a sink or\n# source.\n#\n# The [name] field specifies the name of source or sink.\n#\n# The [options] field is the specific property of this source or sink. The\n# source or sink is responsible for parsing this property.\n#\n# Notes:\n# 1. To add a new sink, set the \"class\" option to a fully qualified class\n# name (see examples below).\n# 2. Some sinks involve a polling period. The minimum allowed polling period\n# is 1 second.\n# 3. Wild card properties can be overridden by more specific properties.\n# For example, master.sink.console.period takes precedence over\n# *.sink.console.period.\n# 4. A metrics specific configuration\n# \"spark.metrics.conf=${SPARK_HOME}/conf/metrics.properties\" should be\n# added to Java properties using -Dspark.metrics.conf=xxx if you wa nt to\n# customize metrics system. You can also put the file in ${SPARK_HOME}/conf\n# and it will be loaded automatically.\n# 5. MetricsServlet is added by default as a sink in master, worker and client\n# driver, you can send http request \"/metrics/json\" to get a snapshot of all the\n# registered metrics in json format. For master, requests \"/metrics/master/json\" and\n# \"/metrics/applications/json\" can be sent seperately to get metrics snapshot of\n# instance master and applications. MetricsServlet may not be configured by self.\n#\n\n## List of available sinks and their properties.\n\n# org.apache.spark.metrics.sink.ConsoleSink\n# Name: Default: Description:\n# period 10 Poll period\n# unit seconds Units of poll period\n\n# org.apache.spark.metrics.sink.CSVSink\n# Name: Default: Description:\n# period 10 Poll period\n# unit seconds Units of poll period\n# directory /tmp Where to store CSV files\n\n# org.apache.spark.metrics.sink.GangliaSink\n# Name: Default: Description:\n# host NONE Hostname or multicast group of Ganglia server\n# port NONE Port of Ganglia server(s)\n# period 10 Poll period\n# unit seconds Units of poll period\n# ttl 1 TTL of messages sent by Ganglia\n# mode multicast Ganglia network mode ('unicast' or 'multicast')\n\n# org.apache.spark.metrics.sink.JmxSink\n\n# org.apache.spark.metrics.sink.MetricsServlet\n# Name: Default: Description:\n# path VARIES* Path prefix from the web server root\n# sample false Whether to show entire set of samples for histograms ('false' or 'true')\n#\n# * Default path is /metrics/json for all instances except the master. The master has two paths:\n# /metrics/aplications/json # App information\n# /metrics/master/json # Master information\n\n# org.apache.spark.metrics.sink.GraphiteSink\n# Name: Default: Description:\n# host NONE Hostname of Graphite server\n# port NONE Port of Graphite server\n# period 10 Poll period\n# unit seconds Units of poll period\n# prefix EMPTY STRING Prefix to prepend to metric name \n\n## Examples\n# Enable JmxSink for all instances by class name\n#*.sink.jmx.class=org.apache.spark.metrics.sink.JmxSink\n\n# Enable ConsoleSink for all instances by class name\n#*.sink.console.class=org.apache.spark.metrics.sink.ConsoleSink\n\n# Polling period for ConsoleSink\n#*.sink.console.period=10\n\n#*.sink.console.unit=seconds\n\n# Master instance overlap polling period\n#master.sink.console.period=15\n\n#master.sink.console.unit=seconds\n\n# Enable CsvSink for all instances\n#*.sink.csv.class=org.apache.spark.metrics.sink.CsvSink\n\n# Polling period for CsvSink\n#*.sink.csv.period=1\n\n#*.sink.csv.unit=minutes\n\n# Polling directory for CsvSink\n#*.sink.csv.directory=/tmp/\n\n# Worker instance overlap polling period\n#worker.sink.csv.period=10\n\n#worker.sink.csv.unit=minutes\n\n# Enable jvm source for instance master, worker, driver and executor\n#master.source.jvm.class=org.apache.spark.metrics.source.JvmSource\n\n#worker.source.jvm.class=org.apache.spark.metrics.source .JvmSource\n\n#driver.source.jvm.class=org.apache.spark.metrics.source.JvmSource\n\n#executor.source.jvm.class=org.apache.spark.metrics.source.JvmSource"
