AMBARI-22162. Move out the druid configurations from hive-site to hive-interactive-site. (Slim Bouguerra via Swapan Shridhar).
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7e885a39 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7e885a39 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7e885a39 Branch: refs/heads/feature-branch-AMBARI-21307 Commit: 7e885a390888a5cddc4cf448af3cfd225f59cf3d Parents: 03273bd Author: Swapan Shridhar <sshrid...@hortonworks.com> Authored: Wed Oct 11 00:57:01 2017 -0700 Committer: Swapan Shridhar <sshrid...@hortonworks.com> Committed: Wed Oct 11 00:57:13 2017 -0700 ---------------------------------------------------------------------- .../HIVE/0.12.0.2.0/configuration/hive-site.xml | 217 ------------------ .../configuration/hive-interactive-site.xml | 225 +++++++++++++++++++ .../stacks/HDP/2.6/services/stack_advisor.py | 13 +- .../stacks/HDP/2.6/upgrades/config-upgrade.xml | 21 +- .../stacks/2.6/common/test_stack_advisor.py | 17 +- 5 files changed, 250 insertions(+), 243 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/7e885a39/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml index d66cf4c..69d1c69 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml @@ -451,223 +451,6 @@ limitations under the License. <on-ambari-upgrade add="false"/> </property> - <!-- Druid related properties --> - <property> - <name>hive.druid.broker.address.default</name> - <value>localhost:8082</value> - <description>Host name of druid router if any or broker</description> - <on-ambari-upgrade add="false"/> - <depends-on> - <property> - <type>druid-router</type> - <name>druid.port</name> - </property> - </depends-on> - </property> - - <property> - <name>hive.druid.metadata.uri</name> - <value>jdbc:mysql://localhost:3355/druid</value> - <description>URI to connect to the database (for example jdbc:mysql://hostname:port/DBName)</description> - <on-ambari-upgrade add="false"/> - <depends-on> - <property> - <type>druid-common</type> - <name>druid.metadata.storage.connector.connectURI</name> - </property> - </depends-on> - </property> - - <property> - <name>hive.druid.coordinator.address.default</name> - <value>localhost:8082</value> - <description>Host name of druid router if any or broker</description> - <on-ambari-upgrade add="false"/> - <depends-on> - <property> - <type>druid-coordinator</type> - <name>druid.port</name> - </property> - </depends-on> - </property> - - <property> - <name>hive.druid.metadata.password</name> - <value>{{druid_metadata_password}}</value> - <property-type>PASSWORD</property-type> - <display-name>Druid Metadata Password</display-name> - <description>Druid meta data storage password</description> - <value-attributes> - <type>password</type> - <empty-value-valid>true</empty-value-valid> - </value-attributes> - <on-ambari-upgrade add="false"/> - </property> - - <property> - <name>hive.druid.metadata.username</name> - <value>druid</value> - <description>Username used to connect to druid metadata storage</description> - <on-ambari-upgrade add="false"/> - <depends-on> - <property> - <type>druid-common</type> - <name>druid.metadata.storage.connector.user</name> - </property> - </depends-on> - </property> - - <property> - <name>hive.druid.indexer.segments.granularity</name> - <display-name>Default Granularity for the Druid segments</display-name> - <value-attributes> - <type>value-list</type> - <entries> - <entry> - <value>YEAR</value> - </entry> - <entry> - <value>MONTH</value> - </entry> - <entry> - <value>WEEK</value> - </entry> - <entry> - <value>DAY</value> - </entry> - <entry> - <value>HOUR</value> - </entry> - <entry> - <value>MINUTE</value> - </entry> - <entry> - <value>SECOND</value> - </entry> - </entries> - </value-attributes> - <value>MINUTE</value> - <description>Default Granularity for the segments created by the Druid storage handler, this can be overridden per table using table property druid.segment.granularity </description> - <on-ambari-upgrade add="false"/> - </property> - <property> - <name>hive.druid.indexer.partition.size.max</name> - <value>5000000</value> - <description>Maximum number of records per segment partition</description> - <on-ambari-upgrade add="false"/> - </property> - <property> - <name>hive.druid.indexer.memory.rownum.max</name> - <value>75000</value> - <description>Maximum number of records in memory while storing data in Druid</description> - <on-ambari-upgrade add="false"/> - </property> - - <property> - <name>hive.druid.select.distribute</name> - <value>true</value> - <description>If it is set to true, we distribute the execution of Druid Select queries</description> - <on-ambari-upgrade add="false"/> - <value-attributes> - <type>boolean</type> - </value-attributes> - </property> - - <property> - <name>hive.druid.basePersistDirectory</name> - <value></value> - <description> - Local temporary directory used to persist intermediate indexing state, - if empty (recommended) will default to JVM system property java.io.tmpdir. - </description> - <value-attributes> - <empty-value-valid>true</empty-value-valid> - </value-attributes> - <on-ambari-upgrade add="false"/> - </property> - - <property> - <name>hive.druid.storage.storageDirectory</name> - <value>{{druid_storage_dir}}</value> - <description> - Druid deep storage location for segments. - </description> - <on-ambari-upgrade add="false"/> - </property> - <property> - <name>hive.druid.metadata.db.type</name> - <display-name>Druid metadata storage type </display-name> - <value-attributes> - <overridable>false</overridable> - <type>value-list</type> - <entries> - <entry> - <value>mysql</value> - <label>MYSQL</label> - </entry> - <entry> - <value>postgresql</value> - <label>POSTGRESQL</label> - </entry> - </entries> - </value-attributes> - <value>mysql</value> - <depends-on> - <property> - <type>druid-common</type> - <name>druid.metadata.storage.type</name> - </property> - </depends-on> - <description>Druid metadata storage type</description> - <on-ambari-upgrade add="false"/> - </property> - - <property> - <name>hive.druid.passiveWaitTimeMs</name> - <value>30000</value> - <description> - Wait time in ms default to 30 seconds. - </description> - <on-ambari-upgrade add="false"/> - </property> - - <property> - <name>hive.druid.working.directory</name> - <value>/tmp/druid-indexing</value> - <description> - Default hdfs working directory used to store some intermediate metadata. - </description> - <on-ambari-upgrade add="false"/> - </property> - - <property> - <name>hive.druid.maxTries</name> - <value>5</value> - <description> - Maximum number of http call retries before giving up. - </description> - <on-ambari-upgrade add="false"/> - </property> - - <property> - <name>hive.druid.bitmap.type</name> - <display-name>Druid metadata storage type </display-name> - <value-attributes> - <type>value-list</type> - <entries> - <entry> - <value>roaring</value> - </entry> - <entry> - <value>concise</value> - </entry> - </entries> - </value-attributes> - <value>roaring</value> - <description>Druid Coding algorithm use to encode the bitmaps</description> - <on-ambari-upgrade add="false"/> - </property> - <!-- This property is removed in HDP 2.5 and higher. --> <property> <name>atlas.rest.address</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/7e885a39/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml index aae2efa..64cef3e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml @@ -124,4 +124,229 @@ limitations under the License. <on-ambari-upgrade add="false"/> </property> + <!-- Druid related properties --> + <property> + <name>hive.druid.broker.address.default</name> + <value>localhost:8082</value> + <description>Host name of druid router if any or broker</description> + <on-ambari-upgrade add="false"/> + <depends-on> + <property> + <type>druid-router</type> + <name>druid.port</name> + </property> + </depends-on> + </property> + + <property> + <name>hive.druid.metadata.uri</name> + <value>jdbc:mysql://localhost:3355/druid</value> + <description>URI to connect to the database (for example jdbc:mysql://hostname:port/DBName)</description> + <on-ambari-upgrade add="false"/> + <depends-on> + <property> + <type>druid-common</type> + <name>druid.metadata.storage.connector.connectURI</name> + </property> + </depends-on> + </property> + + <property> + <name>hive.druid.coordinator.address.default</name> + <value>localhost:8082</value> + <description>Host name of druid router if any or broker</description> + <on-ambari-upgrade add="false"/> + <depends-on> + <property> + <type>druid-coordinator</type> + <name>druid.port</name> + </property> + </depends-on> + </property> + + <property> + <name>hive.druid.metadata.password</name> + <value>{{druid_metadata_password}}</value> + <property-type>PASSWORD</property-type> + <display-name>Druid Metadata Password</display-name> + <description>Druid meta data storage password</description> + <value-attributes> + <type>password</type> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + + <property> + <name>hive.druid.metadata.username</name> + <value>druid</value> + <description>Username used to connect to druid metadata storage</description> + <on-ambari-upgrade add="false"/> + <depends-on> + <property> + <type>druid-common</type> + <name>druid.metadata.storage.connector.user</name> + </property> + </depends-on> + </property> + + <property> + <name>hive.druid.indexer.segments.granularity</name> + <display-name>Default Granularity for the Druid segments</display-name> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>YEAR</value> + </entry> + <entry> + <value>MONTH</value> + </entry> + <entry> + <value>WEEK</value> + </entry> + <entry> + <value>DAY</value> + </entry> + <entry> + <value>HOUR</value> + </entry> + <entry> + <value>MINUTE</value> + </entry> + <entry> + <value>SECOND</value> + </entry> + </entries> + </value-attributes> + <value>DAY</value> + <description>Default Granularity for the segments created by the Druid storage handler, this can be overridden per table using table property druid.segment.granularity </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive.druid.indexer.partition.size.max</name> + <value>1000000</value> + <description>Maximum number of records per segment partition</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive.druid.indexer.memory.rownum.max</name> + <value>75000</value> + <description>Maximum number of records in memory while storing data in Druid</description> + <on-ambari-upgrade add="false"/> + </property> + + <property> + <name>hive.druid.select.distribute</name> + <value>true</value> + <description>If it is set to true, we distribute the execution of Druid Select queries</description> + <on-ambari-upgrade add="false"/> + <value-attributes> + <type>boolean</type> + </value-attributes> + </property> + + <property> + <name>hive.druid.basePersistDirectory</name> + <value></value> + <description> + Local temporary directory used to persist intermediate indexing state, + if empty (recommended) will default to JVM system property java.io.tmpdir. + </description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + + <property> + <name>hive.druid.storage.storageDirectory</name> + <value>{{druid_storage_dir}}</value> + <description> + Druid deep storage location for segments. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive.druid.metadata.db.type</name> + <display-name>Druid metadata storage type </display-name> + <value-attributes> + <overridable>false</overridable> + <type>value-list</type> + <entries> + <entry> + <value>mysql</value> + <label>MYSQL</label> + </entry> + <entry> + <value>postgresql</value> + <label>POSTGRESQL</label> + </entry> + </entries> + </value-attributes> + <value>mysql</value> + <depends-on> + <property> + <type>druid-common</type> + <name>druid.metadata.storage.type</name> + </property> + </depends-on> + <description>Druid metadata storage type</description> + <on-ambari-upgrade add="false"/> + </property> + + <property> + <name>hive.druid.passiveWaitTimeMs</name> + <value>30000</value> + <description> + Wait time in ms default to 30 seconds. + </description> + <on-ambari-upgrade add="false"/> + </property> + + <property> + <name>hive.druid.working.directory</name> + <value>/tmp/druid-indexing</value> + <description> + Default hdfs working directory used to store some intermediate metadata. + </description> + <on-ambari-upgrade add="false"/> + </property> + + <property> + <name>hive.druid.maxTries</name> + <value>5</value> + <description> + Maximum number of http call retries before giving up. + </description> + <on-ambari-upgrade add="false"/> + </property> + + <property> + <name>hive.druid.bitmap.type</name> + <display-name>Druid metadata storage type </display-name> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>roaring</value> + </entry> + <entry> + <value>concise</value> + </entry> + </entries> + </value-attributes> + <value>roaring</value> + <description>Druid Coding algorithm use to encode the bitmaps</description> + <on-ambari-upgrade add="false"/> + </property> + + <property> + <name>hive.druid.http.read.timeout</name> + <value>PT10M</value> + <description> + Maximum number of http call retries before giving up. + </description> + <on-ambari-upgrade add="false"/> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/7e885a39/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py index 0d2925e..38af525 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py @@ -605,7 +605,7 @@ class HDP26StackAdvisor(HDP25StackAdvisor): # druid is not in list of services to be installed servicesList = [service["StackServices"]["service_name"] for service in services["services"]] if 'DRUID' in servicesList: - putHiveSiteProperty = self.putProperty(configurations, "hive-site", services) + putHiveInteractiveSiteProperty = self.putProperty(configurations, "hive-interactive-site", services) if 'druid-coordinator' in services['configurations']: component_hosts = self.getHostsWithComponent("DRUID", 'DRUID_COORDINATOR', services, hosts) if component_hosts is not None and len(component_hosts) > 0: @@ -621,7 +621,6 @@ class HDP26StackAdvisor(HDP25StackAdvisor): if component_hosts is not None and len(component_hosts) > 0: # pick the first host = component_hosts[0] - print host druid_broker_host_port = str(host['Hosts']['host_name']) + ":" + str( services['configurations']['druid-router']['properties']['druid.port']) elif 'druid-broker' in services['configurations']: @@ -642,11 +641,11 @@ class HDP26StackAdvisor(HDP25StackAdvisor): else: druid_metadata_user = "" - putHiveSiteProperty('hive.druid.broker.address.default', druid_broker_host_port) - putHiveSiteProperty('hive.druid.coordinator.address.default', druid_coordinator_host_port) - putHiveSiteProperty('hive.druid.metadata.uri', druid_metadata_uri) - putHiveSiteProperty('hive.druid.metadata.username', druid_metadata_user) - putHiveSiteProperty('hive.druid.metadata.db.type', druid_metadata_type) + putHiveInteractiveSiteProperty('hive.druid.broker.address.default', druid_broker_host_port) + putHiveInteractiveSiteProperty('hive.druid.coordinator.address.default', druid_coordinator_host_port) + putHiveInteractiveSiteProperty('hive.druid.metadata.uri', druid_metadata_uri) + putHiveInteractiveSiteProperty('hive.druid.metadata.username', druid_metadata_user) + putHiveInteractiveSiteProperty('hive.druid.metadata.db.type', druid_metadata_type) def recommendHBASEConfigurations(self, configurations, clusterData, services, hosts): http://git-wip-us.apache.org/repos/asf/ambari/blob/7e885a39/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml index 2b4c656..fd7e438 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml @@ -75,11 +75,19 @@ <set key ="atlas.jaas.ticketBased-KafkaClient.option.useTicketCache" value="true" if-type="cluster-env" if-key="security_enabled" if-value="true"/> </definition> + </changes> + </component> + <component name="HIVE_SERVER_INTERACTIVE"> + <changes> + <definition xsi:type="configure" id="llap_update_tez_shuffle_ssl_enable" summary="Update additional LLAP-Tez settings"> + <type>tez-interactive-site</type> + <set key="tez.runtime.shuffle.ssl.enable" value="false"/> + </definition> <definition xsi:type="configure" id="hdp_2_6_maint_druid_config_for_hive_hook" summary="Updating druid hive related properties"> <type>hive-site</type> <set key="hive.druid.metadata.password" value="{{druid_metadata_password}}" if-type="druid-common"/> - <set key="hive.druid.indexer.segments.granularity" value="MINUTE" if-type="druid-common"/> - <set key="hive.druid.indexer.partition.size.max" value="5000000" if-type="druid-common"/> + <set key="hive.druid.indexer.segments.granularity" value="DAY" if-type="druid-common"/> + <set key="hive.druid.indexer.partition.size.max" value="1000000" if-type="druid-common"/> <set key="hive.druid.indexer.memory.rownum.max" value="75000" if-type="druid-common"/> <set key="hive.druid.select.distribute" value="true" if-type="druid-common"/> <set key="hive.druid.basePersistDirectory" value="" if-type="druid-common"/> @@ -87,15 +95,6 @@ <set key="hive.druid.passiveWaitTimeMs" value="30000" if-type="druid-common"/> <set key="hive.druid.working.directory" value="/tmp/druid-indexing" if-type="druid-common"/> <set key="hive.druid.bitmap.type" value="roaring" if-type="druid-common"/> - - </definition> - </changes> - </component> - <component name="HIVE_SERVER_INTERACTIVE"> - <changes> - <definition xsi:type="configure" id="llap_update_tez_shuffle_ssl_enable" summary="Update additional LLAP-Tez settings"> - <type>tez-interactive-site</type> - <set key="tez.runtime.shuffle.ssl.enable" value="false"/> </definition> </changes> </component> http://git-wip-us.apache.org/repos/asf/ambari/blob/7e885a39/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py index ec44b3d..1c937b7 100644 --- a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py @@ -1835,12 +1835,7 @@ class TestHDP26StackAdvisor(TestCase): 'hive.security.metastore.authorization.manager': 'org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider', 'hive.exec.dynamic.partition.mode': 'strict', 'hive.optimize.sort.dynamic.partition': 'false', - 'hive.server2.enable.doAs': 'false', - 'hive.druid.broker.address.default': 'c6401.ambari.apache.org:8082', - 'hive.druid.coordinator.address.default': 'c6401.ambari.apache.org:8081', - 'hive.druid.metadata.db.type': 'mysql', - 'hive.druid.metadata.uri': 'jdbc:mysql://c6401.ambari.apache.org:3306/druid?createDatabaseIfNotExist=true', - 'hive.druid.metadata.username': 'druid', + 'hive.server2.enable.doAs': 'false' }, 'property_attributes': { 'hive.tez.container.size': { @@ -1885,7 +1880,13 @@ class TestHDP26StackAdvisor(TestCase): } }, 'hive-interactive-site': { - 'properties': {} + 'properties': { + 'hive.druid.broker.address.default': 'c6401.ambari.apache.org:8082', + 'hive.druid.coordinator.address.default': 'c6401.ambari.apache.org:8081', + 'hive.druid.metadata.db.type': 'mysql', + 'hive.druid.metadata.uri': 'jdbc:mysql://c6401.ambari.apache.org:3306/druid?createDatabaseIfNotExist=true', + 'hive.druid.metadata.username': 'druid' + } }, 'yarn-site': { 'properties': { @@ -1941,7 +1942,7 @@ class TestHDP26StackAdvisor(TestCase): services['configurations']['druid-router'] = {} services['configurations']['druid-router']['properties'] = {} services['configurations']['druid-router']['properties']['druid.port'] = 8083 - expected['hive-site']['properties']['hive.druid.broker.address.default'] = 'c6401.ambari.apache.org:8083' + expected['hive-interactive-site']['properties']['hive.druid.broker.address.default'] = 'c6401.ambari.apache.org:8083' recommendedConfigurations = {} self.stackAdvisor.recommendHIVEConfigurations(recommendedConfigurations, clusterData, services, hosts)
