AMBARI-22228 - Remove Remote Tea Runtime library dependency from Storm binaries (Arun Mahadevan via jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8852f33b Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8852f33b Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8852f33b Branch: refs/heads/feature-branch-AMBARI-21307 Commit: 8852f33b90d4983dd9927184f0c20153eaf3b065 Parents: 3b5cbed Author: Jonathan Hurley <[email protected]> Authored: Wed Oct 18 16:54:16 2017 -0400 Committer: Jonathan Hurley <[email protected]> Committed: Wed Oct 18 17:27:11 2017 -0400 ---------------------------------------------------------------------- .../resources/stacks/HDP/2.3/upgrades/config-upgrade.xml | 5 +++++ .../stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml | 3 +++ .../resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml | 1 + .../resources/stacks/HDP/2.4/upgrades/config-upgrade.xml | 4 ++++ .../stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml | 3 +++ .../resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml | 1 + .../resources/stacks/HDP/2.5/upgrades/config-upgrade.xml | 10 ++++++++++ .../stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 3 +++ .../resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 1 + .../HDP/2.6/services/STORM/configuration/storm-site.xml | 6 +++--- .../resources/stacks/HDP/2.6/upgrades/config-upgrade.xml | 4 ++++ .../stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml | 4 ++++ .../resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml | 1 + 13 files changed, 43 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8852f33b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml index ff12150..c4a5b95 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml @@ -660,6 +660,11 @@ replace-with="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}"/> </definition> + <definition xsi:type="configure" id="storm_remove_jmxetric" summary="Removing jmxetric from childopts."> + <type>storm-site</type> + <regex-replace key="content" find=" -javaagent:.*JVM" replace-with=""/> + </definition> + </changes> </component> </service> http://git-wip-us.apache.org/repos/asf/ambari/blob/8852f33b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml index 4034f4b..907626d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml @@ -671,6 +671,9 @@ <summary>Update Storm log directory in storm worker log4j</summary> </task> </execute-stage> + <execute-stage service="STORM" component="NIMBUS" title="Removing jmxetric from childopts"> + <task xsi:type="configure" id="storm_remove_jmxetric"/> + </execute-stage> <!--ATLAS--> <execute-stage service="ATLAS" component="ATLAS_SERVER" title="Parameterizing Atlas Log4J Properties"> http://git-wip-us.apache.org/repos/asf/ambari/blob/8852f33b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml index 2e2c9c3..567e6e1 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml @@ -1130,6 +1130,7 @@ <task xsi:type="configure" id="storm_worker_log4j_parameterize" /> <task xsi:type="configure" id="storm_cluster_log4j_parameterize" /> <task xsi:type="configure" id="storm_worker_log4j_directory" /> + <task xsi:type="configure" id="storm_remove_jmxetric"/> </pre-upgrade> <pre-downgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/8852f33b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml index 5c1f33f..64121a4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml @@ -481,6 +481,10 @@ <replace key="content" find="${sys:storm.log.dir}/${sys:logfile.name}" replace-with="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}"/> </definition> + <definition xsi:type="configure" id="storm_remove_jmxetric" summary="Removing jmxetric from childopts."> + <type>storm-site</type> + <regex-replace key="content" find=" -javaagent:.*JVM" replace-with=""/> + </definition> </changes> </component> </service> http://git-wip-us.apache.org/repos/asf/ambari/blob/8852f33b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml index 4703709..faf5b76 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml @@ -591,6 +591,9 @@ <summary>Update Storm log directory in storm worker log4j</summary> </task> </execute-stage> + <execute-stage service="STORM" component="NIMBUS" title="Removing jmxetric from childopts"> + <task xsi:type="configure" id="storm_remove_jmxetric"/> + </execute-stage> <!-- KAFKA --> <execute-stage service="KAFKA" component="KAFKA_BROKER" title="Apply config changes for Kafka Broker"> http://git-wip-us.apache.org/repos/asf/ambari/blob/8852f33b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml index 35187c7..572a259 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml @@ -1092,6 +1092,7 @@ <task xsi:type="configure" id="storm_worker_log4j_parameterize" /> <task xsi:type="configure" id="storm_cluster_log4j_parameterize" /> <task xsi:type="configure" id="storm_worker_log4j_directory" /> + <task xsi:type="configure" id="storm_remove_jmxetric"/> </pre-upgrade> <pre-downgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/8852f33b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml index cf42e93..d138d60 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml @@ -63,6 +63,16 @@ <regex-replace key="content" find="A1" immediateFlush="false"
 fileName="\$\{sys:storm.log.dir}/\$\{sys:logfile.name}"
 filePattern="\$\{sys:storm.log.dir}/\$\{sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>\$\{pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)" replace-with="A1" immediateFlush="false"
 fileName="${sys:storm.log.dir}/${sys:logfile.name}"
 filePattern="${sys:storm.log.dir}/${sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>${pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_a1_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_a1_maxbackupindex}}"/> </definition> + <definition xsi:type="configure" id="storm_nimbus_autocred_config" summary="Update Storm's Nimbus AutoCred config"> + <type>storm-site</type> + <set key="nimbus.autocredential.plugins.classes" value="['org.apache.storm.hdfs.security.AutoHDFS', 'org.apache.storm.hbase.security.AutoHBase', 'org.apache.storm.hive.security.AutoHive']" if-type="streamline-common" if-key="authorizer.class.name" if-key-state="present"/> + <set key="nimbus.credential.renewers.classes" value="['org.apache.storm.hdfs.security.AutoHDFS', 'org.apache.storm.hbase.security.AutoHBase', 'org.apache.storm.hive.security.AutoHive']" if-type="streamline-common" if-key="authorizer.class.name" if-key-state="present"/> + <set key="nimbus.credential.renewers.freq.secs" value="82800" if-type="streamline-common" if-key="authorizer.class.name" if-key-state="present"/> + </definition> + <definition xsi:type="configure" id="storm_remove_jmxetric" summary="Removing jmxetric from childopts."> + <type>storm-site</type> + <regex-replace key="content" find=" -javaagent:.*JVM" replace-with=""/> + </definition> </changes> </component> </service> http://git-wip-us.apache.org/repos/asf/ambari/blob/8852f33b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml index cd69a9c..8012c90 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml @@ -526,6 +526,9 @@ <summary>Updating the Storm cluster Log4J properties to include parameterizations</summary> </task> </execute-stage> + <execute-stage service="STORM" component="NIMBUS" title="Removing jmxetric from childopts"> + <task xsi:type="configure" id="storm_remove_jmxetric"/> + </execute-stage> <!-- PIG --> <execute-stage service="PIG" component="PIG" title="Apply config changes for Pig"> http://git-wip-us.apache.org/repos/asf/ambari/blob/8852f33b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml index a6d3f29..7c43948 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml @@ -1050,6 +1050,7 @@ <pre-upgrade> <task xsi:type="configure" id="storm_worker_log4j_parameterize" /> <task xsi:type="configure" id="storm_cluster_log4j_parameterize" /> + <task xsi:type="configure" id="storm_remove_jmxetric"/> </pre-upgrade> <pre-downgrade/> <upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/8852f33b/ambari-server/src/main/resources/stacks/HDP/2.6/services/STORM/configuration/storm-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/STORM/configuration/storm-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/STORM/configuration/storm-site.xml index 902fdc9..74262b6 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/STORM/configuration/storm-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/STORM/configuration/storm-site.xml @@ -29,7 +29,7 @@ </property> <property> <name>nimbus.childopts</name> - <value>-Xmx1024m _JAAS_PLACEHOLDER -javaagent:/usr/hdp/current/storm-nimbus/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8649,wireformat31x=true,mode=multicast,config=/usr/hdp/current/storm-nimbus/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM</value> + <value>-Xmx1024m _JAAS_PLACEHOLDER</value> <description>This parameter is used by the storm-deploy project to configure the jvm options for the nimbus daemon.</description> <value-attributes> <overridable>false</overridable> @@ -38,7 +38,7 @@ </property> <property> <name>worker.childopts</name> - <value>-Xmx768m _JAAS_PLACEHOLDER -javaagent:/usr/hdp/current/storm-client/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8650,wireformat31x=true,mode=multicast,config=/usr/hdp/current/storm-client/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM</value> + <value>-Xmx768m _JAAS_PLACEHOLDER</value> <description>The jvm opts provided to workers launched by this supervisor. All \"%ID%\" substrings are replaced with an identifier for this worker.</description> <value-attributes> <type>multiLine</type> @@ -47,7 +47,7 @@ </property> <property> <name>supervisor.childopts</name> - <value>-Xmx256m _JAAS_PLACEHOLDER -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port={{jmxremote_port}} -javaagent:/usr/hdp/current/storm-supervisor/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8650,wireformat31x=true,mode=multicast,config=/usr/hdp/current/storm-supervisor/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM</value> + <value>-Xmx256m _JAAS_PLACEHOLDER -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port={{jmxremote_port}}</value> <description>This parameter is used by the storm-deploy project to configure the jvm options for the supervisor daemon.</description> <value-attributes> <overridable>false</overridable> http://git-wip-us.apache.org/repos/asf/ambari/blob/8852f33b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml index 91044d8..647bdeb 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml @@ -132,6 +132,10 @@ <set key="ranger.plugin.storm.ambari.cluster.name" value="{{cluster_name}}" if-type="ranger-storm-plugin-properties" if-key="ranger-storm-plugin-enabled" if-key-state="present"/> </definition> + <definition xsi:type="configure" id="storm_remove_jmxetric" summary="Removing jmxetric from childopts."> + <type>storm-site</type> + <regex-replace key="content" find=" -javaagent:.*JVM" replace-with=""/> + </definition> </changes> </component> </service> http://git-wip-us.apache.org/repos/asf/ambari/blob/8852f33b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml index 0b5e8c1..c9e90a9 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml @@ -333,6 +333,10 @@ <task xsi:type="configure" id="hdp_2_6_maint_ranger_storm_plugin_cluster_name"/> </execute-stage> + <execute-stage service="STORM" component="NIMBUS" title="Removing jmxetric from childopts"> + <task xsi:type="configure" id="storm_remove_jmxetric"/> + </execute-stage> + <!-- YARN --> <execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes for Ranger Yarn plugin"> <task xsi:type="configure" id="hdp_2_6_maint_ranger_yarn_plugin_cluster_name"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/8852f33b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml index 00597c6..176143c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml @@ -979,6 +979,7 @@ <component name="NIMBUS"> <pre-upgrade> <task xsi:type="configure" id="hdp_2_6_maint_ranger_storm_plugin_cluster_name"/> + <task xsi:type="configure" id="storm_remove_jmxetric"/> </pre-upgrade> <pre-downgrade/> <!-- no-op to prevent config changes on downgrade --> <upgrade>
