Repository: ambari Updated Branches: refs/heads/trunk 60b54592d -> e195171f3
AMBARI-15699: Remove HDP specific hardcodings from common-services config properties (jluniya) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e195171f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e195171f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e195171f Branch: refs/heads/trunk Commit: e195171f39bb455d3536c35c90bf145b9a16d0f6 Parents: 60b5459 Author: Jayush Luniya <[email protected]> Authored: Mon Apr 4 18:03:23 2016 -0700 Committer: Jayush Luniya <[email protected]> Committed: Mon Apr 4 18:03:23 2016 -0700 ---------------------------------------------------------------------- .../1.6.1.2.2.0/configuration/accumulo-env.xml | 2 +- .../1.6.1.2.2.0/configuration/accumulo-site.xml | 14 +-- .../ranger-knox-plugin-properties.xml | 2 +- .../OOZIE/4.0.0.2.0/configuration/oozie-env.xml | 4 - .../OOZIE/4.2.0.2.3/configuration/oozie-env.xml | 3 - .../configuration/ranger-kms-policymgr-ssl.xml | 4 +- .../0.6.0.2.5/configuration/zeppelin-env.xml | 2 +- .../services/OOZIE/configuration/oozie-env.xml | 103 ++++++++++++++++ .../services/OOZIE/configuration/oozie-env.xml | 111 ++++++++++++++++++ .../ZEPPELIN/configuration/zeppelin-env.xml | 117 +++++++++++++++++++ 10 files changed, 343 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/e195171f/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/configuration/accumulo-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/configuration/accumulo-env.xml b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/configuration/accumulo-env.xml index 325886f..e525523 100644 --- a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/configuration/accumulo-env.xml +++ b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/configuration/accumulo-env.xml @@ -84,7 +84,7 @@ </property> <property> <name>accumulo_instance_name</name> - <value>hdp-accumulo-instance</value> + <value>{{stack_name}}-accumulo-instance</value> <display-name>Instance Name</display-name> <description>Accumulo Instance Name.</description> <value-attributes> http://git-wip-us.apache.org/repos/asf/ambari/blob/e195171f/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/configuration/accumulo-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/configuration/accumulo-site.xml b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/configuration/accumulo-site.xml index c2e0200..dcad802 100644 --- a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/configuration/accumulo-site.xml +++ b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/configuration/accumulo-site.xml @@ -176,13 +176,13 @@ $ACCUMULO_HOME/lib/accumulo-proxy.jar, $ACCUMULO_HOME/lib/[^.].*.jar, $ZOOKEEPER_HOME/zookeeper[^.].*.jar, $HADOOP_CONF_DIR, -/usr/hdp/current/hadoop-client/[^.].*.jar, -/usr/hdp/current/hadoop-client/lib/(?!slf4j)[^.].*.jar, -/usr/hdp/current/hadoop-hdfs-client/[^.].*.jar, -/usr/hdp/current/hadoop-mapreduce-client/[^.].*.jar, -/usr/hdp/current/hadoop-yarn-client/[^.].*.jar, -/usr/hdp/current/hadoop-yarn-client/lib/jersey.*.jar, -/usr/hdp/current/hive-client/lib/hive-accumulo-handler.jar, +{{stack_root}}/current/hadoop-client/[^.].*.jar, +{{stack_root}}/current/hadoop-client/lib/(?!slf4j)[^.].*.jar, +{{stack_root}}/current/hadoop-hdfs-client/[^.].*.jar, +{{stack_root}}/current/hadoop-mapreduce-client/[^.].*.jar, +{{stack_root}}/current/hadoop-yarn-client/[^.].*.jar, +{{stack_root}}/current/hadoop-yarn-client/lib/jersey.*.jar, +{{stack_root}}/current/hive-client/lib/hive-accumulo-handler.jar, </value> <description>List of jars and directories that will be added to Accumulo's classpath. Ordering matters, as the entries will be http://git-wip-us.apache.org/repos/asf/ambari/blob/e195171f/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/configuration/ranger-knox-plugin-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/configuration/ranger-knox-plugin-properties.xml b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/configuration/ranger-knox-plugin-properties.xml index eef1daf..f3dfd1d 100644 --- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/configuration/ranger-knox-plugin-properties.xml +++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/configuration/ranger-knox-plugin-properties.xml @@ -73,7 +73,7 @@ <property> <name>KNOX_HOME</name> - <value>/usr/hdp/current/knox-server</value> + <value>{{stack_root}}/current/knox-server</value> <display-name>Knox Home</display-name> <description>Knox home folder</description> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/e195171f/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-env.xml b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-env.xml index 6b3bb6c..8f0cdfa 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-env.xml +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-env.xml @@ -187,10 +187,6 @@ export OOZIE_ADMIN_PORT={{oozie_server_admin_port}} # # export OOZIE_BASE_URL="http://${OOZIE_HTTP_HOSTNAME}:${OOZIE_HTTP_PORT}/oozie"; export JAVA_LIBRARY_PATH={{hadoop_lib_home}}/native/Linux-amd64-64 - -# Set Hadoop-related properties -export HADOOP_OPTS="-Dhdp.version=${HDP_VERSION} ${HADOOP_OPTS}" - </value> <value-attributes> <type>content</type> http://git-wip-us.apache.org/repos/asf/ambari/blob/e195171f/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.2.3/configuration/oozie-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.2.3/configuration/oozie-env.xml b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.2.3/configuration/oozie-env.xml index 3cda54a..aa860f4 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.2.3/configuration/oozie-env.xml +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.2.3/configuration/oozie-env.xml @@ -99,9 +99,6 @@ export OOZIE_CLIENT_OPTS="${OOZIE_CLIENT_OPTS} -Doozie.connection.retry.count=5 export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:{{jdbc_libs_dir}}" export JAVA_LIBRARY_PATH="$JAVA_LIBRARY_PATH:{{jdbc_libs_dir}}" {% endif %} - -# Set Hadoop-related properties -export HADOOP_OPTS="-Dhdp.version=${HDP_VERSION} ${HADOOP_OPTS}" </value> <value-attributes> <type>content</type> http://git-wip-us.apache.org/repos/asf/ambari/blob/e195171f/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-policymgr-ssl.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-policymgr-ssl.xml b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-policymgr-ssl.xml index f6b4cbc..ed89260 100644 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-policymgr-ssl.xml +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-policymgr-ssl.xml @@ -22,7 +22,7 @@ <property> <name>xasecure.policymgr.clientssl.keystore</name> - <value>/usr/hdp/current/ranger-kms/conf/ranger-plugin-keystore.jks</value> + <value>{{stack_root}}/current/ranger-kms/conf/ranger-plugin-keystore.jks</value> <description>Java Keystore files</description> </property> @@ -38,7 +38,7 @@ <property> <name>xasecure.policymgr.clientssl.truststore</name> - <value>/usr/hdp/current/ranger-kms/conf/ranger-plugin-truststore.jks</value> + <value>{{stack_root}}/current/ranger-kms/conf/ranger-plugin-truststore.jks</value> <description>java truststore file</description> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/e195171f/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml index ee0c48f..3126cda 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml @@ -65,7 +65,7 @@ export ZEPPELIN_PID_DIR={{zeppelin_pid_dir}} export JAVA_HOME={{java64_home}} # Additional jvm options. for example, export ZEPPELIN_JAVA_OPTS="-Dspark.executor.memory=8g -Dspark.cores.max=16" -export ZEPPELIN_JAVA_OPTS="-Dhdp.version={{full_stack_version}} -Dspark.executor.memory={{executor_mem}} -Dspark.executor.instances={{executor_instances}} -Dspark.yarn.queue={{spark_queue}}" +export ZEPPELIN_JAVA_OPTS="-Dspark.executor.memory={{executor_mem}} -Dspark.executor.instances={{executor_instances}} -Dspark.yarn.queue={{spark_queue}}" # Zeppelin jvm mem options Default -Xmx1024m -XX:MaxPermSize=512m http://git-wip-us.apache.org/repos/asf/ambari/blob/e195171f/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/configuration/oozie-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/configuration/oozie-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/configuration/oozie-env.xml new file mode 100644 index 0000000..1b6e557 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/configuration/oozie-env.xml @@ -0,0 +1,103 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration supports_adding_forbidden="true"> + + <!-- oozie-env.sh --> + <property> + <name>content</name> + <description>This is the jinja template for oozie-env.sh file</description> + <value> +#!/bin/bash + +if [ -d "/usr/lib/bigtop-tomcat" ]; then + export OOZIE_CONFIG=${OOZIE_CONFIG:-/etc/oozie/conf} + export CATALINA_BASE=${CATALINA_BASE:-{{oozie_server_dir}}} + export CATALINA_TMPDIR=${CATALINA_TMPDIR:-{{oozie_tmp_dir}}} + export OOZIE_CATALINA_HOME=/usr/lib/bigtop-tomcat +fi + +#Set JAVA HOME +export JAVA_HOME={{java_home}} + +export JRE_HOME=${JAVA_HOME} + +# Set Oozie specific environment variables here. + +# Settings for the Embedded Tomcat that runs Oozie +# Java System properties for Oozie should be specified in this variable +# +{% if java_version < 8 %} +export CATALINA_OPTS="$CATALINA_OPTS -Xmx{{oozie_heapsize}} -XX:MaxPermSize={{oozie_permsize}}" +{% else %} +export CATALINA_OPTS="$CATALINA_OPTS -Xmx{{oozie_heapsize}}" +{% endif %} +# Oozie configuration file to load from Oozie configuration directory +# +# export OOZIE_CONFIG_FILE=oozie-site.xml + +# Oozie logs directory +# +export OOZIE_LOG={{oozie_log_dir}} + +# Oozie pid directory +# +export CATALINA_PID={{pid_file}} + +#Location of the data for oozie +export OOZIE_DATA={{oozie_data_dir}} + +# Oozie Log4J configuration file to load from Oozie configuration directory +# +# export OOZIE_LOG4J_FILE=oozie-log4j.properties + +# Reload interval of the Log4J configuration file, in seconds +# +# export OOZIE_LOG4J_RELOAD=10 + +# The port Oozie server runs +# +export OOZIE_HTTP_PORT={{oozie_server_port}} + +# The admin port Oozie server runs +# +export OOZIE_ADMIN_PORT={{oozie_server_admin_port}} + +# The host name Oozie server runs on +# +# export OOZIE_HTTP_HOSTNAME=`hostname -f` + +# The base URL for callback URLs to Oozie +# +# export OOZIE_BASE_URL="http://${OOZIE_HTTP_HOSTNAME}:${OOZIE_HTTP_PORT}/oozie"; +export JAVA_LIBRARY_PATH={{hadoop_lib_home}}/native/Linux-amd64-64 + +# Set Hadoop-related properties +export HADOOP_OPTS="-Dhdp.version=${HDP_VERSION} ${HADOOP_OPTS}" + + </value> + <value-attributes> + <type>content</type> + </value-attributes> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/e195171f/ambari-server/src/main/resources/stacks/HDP/2.3/services/OOZIE/configuration/oozie-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/OOZIE/configuration/oozie-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/OOZIE/configuration/oozie-env.xml new file mode 100644 index 0000000..3cda54a --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/OOZIE/configuration/oozie-env.xml @@ -0,0 +1,111 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration supports_adding_forbidden="true"> + + <!-- oozie-env.sh --> + <property> + <name>content</name> + <description>This is the jinja template for oozie-env.sh file</description> + <value> +#!/bin/bash + +if [ -d "/usr/lib/bigtop-tomcat" ]; then + export OOZIE_CONFIG=${OOZIE_CONFIG:-{{conf_dir}}} + export CATALINA_BASE=${CATALINA_BASE:-{{oozie_server_dir}}} + export CATALINA_TMPDIR=${CATALINA_TMPDIR:-/var/tmp/oozie} + export OOZIE_CATALINA_HOME=/usr/lib/bigtop-tomcat +fi + +#Set JAVA HOME +export JAVA_HOME={{java_home}} + +export JRE_HOME=${JAVA_HOME} + +# Set Oozie specific environment variables here. + +# Settings for the Embedded Tomcat that runs Oozie +# Java System properties for Oozie should be specified in this variable +# +{% if java_version < 8 %} +export CATALINA_OPTS="$CATALINA_OPTS -Xmx{{oozie_heapsize}} -XX:MaxPermSize={{oozie_permsize}}" +{% else %} +export CATALINA_OPTS="$CATALINA_OPTS -Xmx{{oozie_heapsize}}" +{% endif %} +# Oozie configuration file to load from Oozie configuration directory +# +# export OOZIE_CONFIG_FILE=oozie-site.xml + +# Oozie logs directory +# +export OOZIE_LOG={{oozie_log_dir}} + +# Oozie pid directory +# +export CATALINA_PID={{pid_file}} + +#Location of the data for oozie +export OOZIE_DATA={{oozie_data_dir}} + +# Oozie Log4J configuration file to load from Oozie configuration directory +# +# export OOZIE_LOG4J_FILE=oozie-log4j.properties + +# Reload interval of the Log4J configuration file, in seconds +# +# export OOZIE_LOG4J_RELOAD=10 + +# The port Oozie server runs +# +export OOZIE_HTTP_PORT={{oozie_server_port}} + +# The admin port Oozie server runs +# +export OOZIE_ADMIN_PORT={{oozie_server_admin_port}} + +# The host name Oozie server runs on +# +# export OOZIE_HTTP_HOSTNAME=`hostname -f` + +# The base URL for callback URLs to Oozie +# +# export OOZIE_BASE_URL="http://${OOZIE_HTTP_HOSTNAME}:${OOZIE_HTTP_PORT}/oozie"; +export JAVA_LIBRARY_PATH={{hadoop_lib_home}}/native/Linux-amd64-64 + +# At least 1 minute of retry time to account for server downtime during +# upgrade/downgrade +export OOZIE_CLIENT_OPTS="${OOZIE_CLIENT_OPTS} -Doozie.connection.retry.count=5 " + +{% if sqla_db_used or lib_dir_available %} +export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:{{jdbc_libs_dir}}" +export JAVA_LIBRARY_PATH="$JAVA_LIBRARY_PATH:{{jdbc_libs_dir}}" +{% endif %} + +# Set Hadoop-related properties +export HADOOP_OPTS="-Dhdp.version=${HDP_VERSION} ${HADOOP_OPTS}" + </value> + <value-attributes> + <type>content</type> + </value-attributes> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/e195171f/ambari-server/src/main/resources/stacks/HDP/2.5/services/ZEPPELIN/configuration/zeppelin-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ZEPPELIN/configuration/zeppelin-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ZEPPELIN/configuration/zeppelin-env.xml new file mode 100644 index 0000000..2cbc38d --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ZEPPELIN/configuration/zeppelin-env.xml @@ -0,0 +1,117 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<configuration> + + <property> + <name>content</name> + <description>This is the jinja template for zeppelin-env.sh file</description> + <value> +# Spark master url. eg. spark://master_addr:7077. Leave empty if you want to use local mode +export MASTER=yarn-client +export SPARK_YARN_JAR={{spark_jar}} + + +# Where log files are stored. PWD by default. +export ZEPPELIN_LOG_DIR={{zeppelin_log_dir}} + +# The pid files are stored. /tmp by default. +export ZEPPELIN_PID_DIR={{zeppelin_pid_dir}} + + +export JAVA_HOME={{java64_home}} + +# Additional jvm options. for example, export ZEPPELIN_JAVA_OPTS="-Dspark.executor.memory=8g -Dspark.cores.max=16" +export ZEPPELIN_JAVA_OPTS="-Dhdp.version={{full_stack_version}} -Dspark.executor.memory={{executor_mem}} -Dspark.executor.instances={{executor_instances}} -Dspark.yarn.queue={{spark_queue}}" + + +# Zeppelin jvm mem options Default -Xmx1024m -XX:MaxPermSize=512m +# export ZEPPELIN_MEM + +# zeppelin interpreter process jvm mem options. Defualt = ZEPPELIN_MEM +# export ZEPPELIN_INTP_MEM + +# zeppelin interpreter process jvm options. Default = ZEPPELIN_JAVA_OPTS +# export ZEPPELIN_INTP_JAVA_OPTS + +# Where notebook saved +# export ZEPPELIN_NOTEBOOK_DIR + +# Id of notebook to be displayed in homescreen. ex) 2A94M5J1Z +# export ZEPPELIN_NOTEBOOK_HOMESCREEN + +# hide homescreen notebook from list when this value set to "true". default "false" +# export ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE + +# Bucket where notebook saved +# export ZEPPELIN_NOTEBOOK_S3_BUCKET + +# User in bucket where notebook saved. For example bucket/user/notebook/2A94M5J1Z/note.json +# export ZEPPELIN_NOTEBOOK_S3_USER + +# A string representing this instance of zeppelin. $USER by default +# export ZEPPELIN_IDENT_STRING + +# The scheduling priority for daemons. Defaults to 0. +# export ZEPPELIN_NICENESS + + +#### Spark interpreter configuration #### + +## Use provided spark installation ## +## defining SPARK_HOME makes Zeppelin run spark interpreter process using spark-submit +## +# (required) When it is defined, load it instead of Zeppelin embedded Spark libraries +export SPARK_HOME={{spark_home}} + +# (optional) extra options to pass to spark submit. eg) "--driver-memory 512M --executor-memory 1G". +# export SPARK_SUBMIT_OPTIONS + +## Use embedded spark binaries ## +## without SPARK_HOME defined, Zeppelin still able to run spark interpreter process using embedded spark binaries. +## however, it is not encouraged when you can define SPARK_HOME +## +# Options read in YARN client mode +# yarn-site.xml is located in configuration directory in HADOOP_CONF_DIR. +export HADOOP_CONF_DIR=/etc/hadoop/conf + +# Pyspark (supported with Spark 1.2.1 and above) +# To configure pyspark, you need to set spark distribution's path to 'spark.home' property in Interpreter setting screen in Zeppelin GUI +# path to the python command. must be the same path on the driver(Zeppelin) and all workers. +# export PYSPARK_PYTHON + +export PYTHONPATH="${SPARK_HOME}/python:${SPARK_HOME}/python/lib/py4j-0.8.2.1-src.zip" +export SPARK_YARN_USER_ENV="PYTHONPATH=${PYTHONPATH}" + +## Spark interpreter options ## +## +# Use HiveContext instead of SQLContext if set true. true by default. +# export ZEPPELIN_SPARK_USEHIVECONTEXT + +# Execute multiple SQL concurrently if set true. false by default. +# export ZEPPELIN_SPARK_CONCURRENTSQL + +# Max number of SparkSQL result to display. 1000 by default. +# export ZEPPELIN_SPARK_MAXRESULT + + </value> + </property> + +</configuration>
