AMBARI-22188. Make hive server create directories related to replication (echekanskiy)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4a47e792 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4a47e792 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4a47e792 Branch: refs/heads/feature-branch-AMBARI-21307 Commit: 4a47e792ae6e4fe4fb67ca80c96c5e2054f7a9cd Parents: 7341567 Author: Eugene Chekanskiy <echekans...@hortonworks.com> Authored: Wed Oct 11 19:38:41 2017 +0300 Committer: Eugene Chekanskiy <echekans...@hortonworks.com> Committed: Wed Oct 11 19:38:41 2017 +0300 ---------------------------------------------------------------------- .../HIVE/0.12.0.2.0/configuration/hive-site.xml | 47 +++++++++++++++++++ .../HIVE/0.12.0.2.0/package/scripts/hive.py | 34 +++++++++++++- .../0.12.0.2.0/package/scripts/params_linux.py | 4 ++ .../HIVE/2.1.0.3.0/configuration/hive-site.xml | 48 ++++++++++++++++++++ .../HIVE/2.1.0.3.0/package/scripts/hive.py | 32 +++++++++++++ .../2.1.0.3.0/package/scripts/params_linux.py | 4 ++ 6 files changed, 168 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/4a47e792/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml index 69d1c69..762530b 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml @@ -471,4 +471,51 @@ limitations under the License. </depends-on> <on-ambari-upgrade add="false"/> </property> + + <!-- Hive replication properties --> + <property> + <name>hive.metastore.dml.events</name> + <value/> + <description>If true, the metastore will be asked to fire events for DML operations</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive.repl.cm.enabled</name> + <value/> + <description>Turn on ChangeManager, so delete files will go to cmrootdir.</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive.metastore.transactional.event.listeners</name> + <value/> + <description>A comma separated list of Java classes that implement the org.apache.hadoop.hive.metastore.MetaStoreEventListener interface. Both the metastore event and corresponding listener method will be invoked in the same JDO transaction.</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive.repl.cmrootdir</name> + <value/> + <description>Root dir for ChangeManager, used for deleted files.</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive.repl.rootdir</name> + <value/> + <description>HDFS root dir for all replication dumps.</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/4a47e792/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py index 8e176b6..c4b34a5 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py @@ -245,7 +245,22 @@ def setup_hiveserver2(): owner=params.hive_user, group=params.hdfs_user, mode=0777) # Hive expects this dir to be writeable by everyone as it is used as a temp dir - + + if params.hive_repl_cmrootdir is not None: + params.HdfsResource(params.hive_repl_cmrootdir, + type = "directory", + action = "create_on_execute", + owner = params.hive_user, + group=params.user_group, + mode = 01777) + if params.hive_repl_rootdir is not None: + params.HdfsResource(params.hive_repl_rootdir, + type = "directory", + action = "create_on_execute", + owner = params.hive_user, + group=params.user_group, + mode = 0700) + params.HdfsResource(None, action="execute") def setup_non_client(): @@ -310,6 +325,23 @@ def setup_metastore(): create_parents = True, mode=0777) + if params.hive_repl_cmrootdir is not None: + params.HdfsResource(params.hive_repl_cmrootdir, + type = "directory", + action = "create_on_execute", + owner = params.hive_user, + group=params.user_group, + mode = 01777) + if params.hive_repl_rootdir is not None: + params.HdfsResource(params.hive_repl_rootdir, + type = "directory", + action = "create_on_execute", + owner = params.hive_user, + group=params.user_group, + mode = 0700) + if params.hive_repl_cmrootdir is not None or params.hive_repl_rootdir is not None: + params.HdfsResource(None, action="execute") + def create_metastore_schema(): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/4a47e792/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index 1d68ef4..16e1a71 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -871,3 +871,7 @@ if 'druid-common' in config['configurations'] \ manage_hive_fsroot = default('/configurations/cluster-env/manage_hive_fsroot', True) + +# replication directories +hive_repl_cmrootdir = default('/configurations/hive-site/hive.repl.cmrootdir', None) +hive_repl_rootdir = default('/configurations/hive-site/hive.repl.rootdir', None) http://git-wip-us.apache.org/repos/asf/ambari/blob/4a47e792/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-site.xml b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-site.xml index edbbc9d..3c153b6 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-site.xml +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-site.xml @@ -2001,4 +2001,52 @@ limitations under the License. </description> <on-ambari-upgrade add="false"/> </property> + + <!-- Hive replication properties --> + <property> + <name>hive.metastore.dml.events</name> + <value/> + <description>If true, the metastore will be asked to fire events for DML operations</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive.repl.cm.enabled</name> + <value/> + <description>Turn on ChangeManager, so delete files will go to cmrootdir.</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive.metastore.transactional.event.listeners</name> + <value/> + <description>A comma separated list of Java classes that implement the org.apache.hadoop.hive.metastore.MetaStoreEventListener interface. Both the metastore event and corresponding listener method will be invoked in the same JDO transaction.</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive.repl.cmrootdir</name> + <value/> + <description>Root dir for ChangeManager, used for deleted files.</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive.repl.rootdir</name> + <value/> + <description>HDFS root dir for all replication dumps.</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/4a47e792/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py index e7e92ec..22ff9fd 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py @@ -245,6 +245,21 @@ def setup_hiveserver2(): group=params.hdfs_user, mode=0777) # Hive expects this dir to be writeable by everyone as it is used as a temp dir + if params.hive_repl_cmrootdir is not None: + params.HdfsResource(params.hive_repl_cmrootdir, + type = "directory", + action = "create_on_execute", + owner = params.hive_user, + group=params.user_group, + mode = 01777) + if params.hive_repl_rootdir is not None: + params.HdfsResource(params.hive_repl_rootdir, + type = "directory", + action = "create_on_execute", + owner = params.hive_user, + group=params.user_group, + mode = 0700) + params.HdfsResource(None, action="execute") def setup_non_client(): @@ -300,6 +315,23 @@ def setup_metastore(): content=StaticFile('startMetastore.sh') ) + if params.hive_repl_cmrootdir is not None: + params.HdfsResource(params.hive_repl_cmrootdir, + type = "directory", + action = "create_on_execute", + owner = params.hive_user, + group=params.user_group, + mode = 01777) + if params.hive_repl_rootdir is not None: + params.HdfsResource(params.hive_repl_rootdir, + type = "directory", + action = "create_on_execute", + owner = params.hive_user, + group=params.user_group, + mode = 0700) + if params.hive_repl_cmrootdir is not None or params.hive_repl_rootdir is not None: + params.HdfsResource(None, action="execute") + def create_metastore_schema(): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/4a47e792/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py index f6b676b..fb2c84a 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py @@ -848,3 +848,7 @@ ranger_hive_metastore_lookup = default('/configurations/ranger-hive-plugin-prope if security_enabled: hive_metastore_principal_with_host = hive_metastore_principal.replace('_HOST', hostname.lower()) + +# replication directories +hive_repl_cmrootdir = default('/configurations/hive-site/hive.repl.cmrootdir', None) +hive_repl_rootdir = default('/configurations/hive-site/hive.repl.rootdir', None)
