AMBARI-21460. Add new kafka client properties to the ambari managed atlas config (smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4ddbd624 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4ddbd624 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4ddbd624 Branch: refs/heads/branch-feature-AMBARI-21348 Commit: 4ddbd6246b5718f30912dec0f30939ac33b052dc Parents: 48bc763 Author: Sumit Mohanty <[email protected]> Authored: Fri Jul 14 15:42:52 2017 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Fri Jul 14 15:43:01 2017 -0700 ---------------------------------------------------------------------- .../4.0/stack-advisor/stack_advisor_25.py | 3 ++- .../stacks/HDP/2.5/upgrades/config-upgrade.xml | 4 ++++ .../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 6 +++++- .../stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 1 + .../ATLAS/configuration/application-properties.xml | 17 +++++++++++++++++ .../stacks/HDP/2.6/upgrades/config-upgrade.xml | 4 ++++ .../HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml | 4 ++++ .../stacks/HDP/2.6/upgrades/upgrade-2.6.xml | 1 + 8 files changed, 38 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddbd624/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_25.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_25.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_25.py index eb7d370..7e77382 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_25.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_25.py @@ -734,7 +734,8 @@ class HDP25StackAdvisor(HDP24StackAdvisor): self.checkAndStopLlapQueue(services, configurations, LLAP_QUEUE_NAME) putYarnSiteProperty = self.putProperty(configurations, "yarn-site", services) - stack_root = self.getStackRoot(services) + stack_root = self.getStackRoot(services + ) timeline_plugin_classes_values = [] timeline_plugin_classpath_values = [] http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddbd624/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml index 7fedc3a..6166318 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml @@ -123,6 +123,10 @@ <type>atlas-env</type> <replace key="content" find="-Xloggc:$ATLAS_LOG_DIRgc-worker.log" replace-with="-Xloggc:$ATLAS_LOG_DIR/gc-worker.log"/> </definition> + <definition xsi:type="configure" id="hdp_2_6_atlas_kafka_auto_commit_enable_property_delete" summary="Updating Atlas Kafka configurations."> + <type>application-properties</type> + <transfer operation="delete" delete-key="atlas.kafka.auto.commit.enable"/> + </definition> </changes> </component> <component name="SPARK_CLIENT"> http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddbd624/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml index 3cab083..ce10e8b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml @@ -453,7 +453,7 @@ <summary>Updating the Atlas Log4J properties to include parameterizations</summary> </task> </execute-stage> - + <execute-stage service="ATLAS" component="ATLAS_SERVER" title="Updating Atlas TLS Exclude Protocols"> <task xsi:type="configure" id="hdp_2_5_4_0_atlas_exclude_tls_protocol"> <summary>Updating Atlas TLS Exclude Protocols to exclude TLS v1.2</summary> @@ -472,6 +472,10 @@ </task> </execute-stage> + <execute-stage service="ATLAS" component="ATLAS_SERVER" title="Updating Atlas Kafka configurations."> + <task xsi:type="configure" id="hdp_2_6_atlas_kafka_auto_commit_enable_property_delete"/> + </execute-stage> + <!--KAFKA--> <execute-stage service="KAFKA" component="KAFKA_BROKER" title="Parameterizing Kafka Log4J Properties"> <task xsi:type="configure" id="kafka_log4j_parameterize"> http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddbd624/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml index 22c9a8d..840b17d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml @@ -1101,6 +1101,7 @@ <task xsi:type="configure" id="hdp_2_5_4_0_atlas_exclude_tls_protocol"/> <task xsi:type="configure" id="increase_atlas_zookeeper_timeouts"/> <task xsi:type="configure" id="atlas_env_gc_worker"/> + <task xsi:type="configure" id="hdp_2_6_atlas_kafka_auto_commit_enable_property_delete"/> </pre-upgrade> <pre-downgrade /> <upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddbd624/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/configuration/application-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/configuration/application-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/configuration/application-properties.xml index 91de1b0..c271dc3 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/configuration/application-properties.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/configuration/application-properties.xml @@ -92,4 +92,21 @@ </value-attributes> <on-ambari-upgrade add="true"/> </property> + <property> + <name>atlas.kafka.session.timeout.ms</name> + <value>30000</value> + <description>New Kafka consumer API</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>atlas.kafka.enable.auto.commit</name> + <value>false</value> + <description>New Kafka consumer API</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>atlas.kafka.auto.commit.enable</name> + <deleted>true</deleted> + <on-ambari-upgrade add="false"/> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddbd624/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml index 6dd2129..c2c1532 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml @@ -171,6 +171,10 @@ <type>atlas-env</type> <replace key="content" find="-Xloggc:$ATLAS_LOG_DIRgc-worker.log" replace-with="-Xloggc:$ATLAS_LOG_DIR/gc-worker.log"/> </definition> + <definition xsi:type="configure" id="hdp_2_6_atlas_kafka_auto_commit_enable_property_delete" summary="Updating Atlas Kafka configurations."> + <type>application-properties</type> + <transfer operation="delete" delete-key="atlas.kafka.auto.commit.enable"/> + </definition> </changes> </component> </service> http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddbd624/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml index e262971..df609cd 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml @@ -358,6 +358,10 @@ <task xsi:type="configure" id="atlas_env_gc_worker"/> </execute-stage> + <execute-stage service="ATLAS" component="ATLAS_SERVER" title="Updating Atlas Kafka configurations."> + <task xsi:type="configure" id="hdp_2_6_atlas_kafka_auto_commit_enable_property_delete"/> + </execute-stage> + <!-- KMS --> <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Apply config changes for Ranger Kms plugin"> <task xsi:type="configure" id="hdp_2_6_maint_ranger_kms_plugin_cluster_name"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddbd624/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml index 6b01ce9..b376fa7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml @@ -1034,6 +1034,7 @@ <pre-upgrade> <task xsi:type="configure" id="hdp_2_6_maint_ranger_atlas_plugin_cluster_name"/> <task xsi:type="configure" id="atlas_env_gc_worker"/> + <task xsi:type="configure" id="hdp_2_6_atlas_kafka_auto_commit_enable_property_delete"/> </pre-upgrade> <pre-downgrade/> <!-- no-op to prevent config changes on downgrade --> <upgrade>
