Repository: ambari Updated Branches: refs/heads/trunk f592dbe39 -> 18e9d2d31
AMBARI-11298. Ambari set up for Falcon Hive DR. (mpapirkovskyy) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/18e9d2d3 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/18e9d2d3 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/18e9d2d3 Branch: refs/heads/trunk Commit: 18e9d2d319d418296058a2f757be9503e781a849 Parents: f592dbe Author: Myroslav Papirkovskyy <[email protected]> Authored: Thu May 21 17:55:10 2015 +0300 Committer: Myroslav Papirkovskyy <[email protected]> Committed: Thu May 21 17:57:27 2015 +0300 ---------------------------------------------------------------------- .../0.5.0.2.1/configuration/falcon-env.xml | 7 ++++- .../FALCON/0.5.0.2.1/package/scripts/falcon.py | 11 ++++++++ .../0.5.0.2.1/package/scripts/params_linux.py | 5 ++++ .../FALCON/configuration/falcon-env.xml | 27 ++++++++++++++++++++ .../stacks/2.1/FALCON/test_falcon_server.py | 17 ++++++++++++ .../test/python/stacks/2.1/configs/default.json | 3 ++- .../test/python/stacks/2.1/configs/secured.json | 3 ++- .../stacks/2.2/configs/falcon-upgrade.json | 3 ++- 8 files changed, 72 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/18e9d2d3/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml index 363e4f3..75178d2 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml @@ -111,5 +111,10 @@ export FALCON_DATA_DIR={{falcon_embeddedmq_data}} # Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir. #export FALCON_EXPANDED_WEBAPP_DIR= </value> - </property> + </property> + <property> + <name>supports_hive_dr</name> + <value>false</value> + <description>Enable support of Falcon Hive DR</description> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/18e9d2d3/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py index 86318f3..639071e 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py @@ -120,6 +120,17 @@ def falcon(type, action = None): owner=params.falcon_user, recursive=True ) + if params.supports_hive_dr: + params.HdfsResource(params.dfs_data_mirroring_dir, + type="directory", + action="create_on_execute", + owner=params.hdfs_user, + group=params.user_group, + recursive_chown = True, + recursive_chmod = True, + mode=0770, + source=params.local_data_mirroring_dir + ) params.HdfsResource(None, action="execute") Directory(params.falcon_local_dir, owner=params.falcon_user, http://git-wip-us.apache.org/repos/asf/ambari/blob/18e9d2d3/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py index 9663d11..031f5ab 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py @@ -101,6 +101,11 @@ hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) + +supports_hive_dr = config['configurations']['falcon-env']['supports_hive_dr'] +local_data_mirroring_dir = "/usr/hdp/current/falcon-server/data-mirroring" +dfs_data_mirroring_dir = "/apps/data-mirroring" + import functools #create partial functions with common arguments for every HdfsResource call #to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code http://git-wip-us.apache.org/repos/asf/ambari/blob/18e9d2d3/ambari-server/src/main/resources/stacks/HDP/2.3/services/FALCON/configuration/falcon-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/FALCON/configuration/falcon-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/FALCON/configuration/falcon-env.xml new file mode 100644 index 0000000..449f8fd --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/FALCON/configuration/falcon-env.xml @@ -0,0 +1,27 @@ +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <property> + <name>supports_hive_dr</name> + <value>true</value> + <description>Enable support of Falcon Hive DR</description> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/18e9d2d3/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py index 8fa7cd2..7fc5bce 100644 --- a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py +++ b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py @@ -138,6 +138,23 @@ class TestFalconServer(RMFTestCase): owner = 'falcon', recursive = True, ) + self.assertResourceCalled('HdfsResource', '/apps/data-mirroring', + security_enabled = False, + hadoop_bin_dir = '/usr/bin', + keytab = UnknownConfigurationMock(), + kinit_path_local = '/usr/bin/kinit', + user = 'hdfs', + owner = 'hdfs', + group='hadoop', + hadoop_conf_dir = '/etc/hadoop/conf', + type = 'directory', + recursive_chown = True, + recursive_chmod = True, + action = ['create_on_execute'], + mode = 0770, + source='/usr/hdp/current/falcon-server/data-mirroring' + ) + self.assertResourceCalled('HdfsResource', None, security_enabled = False, hadoop_bin_dir = '/usr/bin', http://git-wip-us.apache.org/repos/asf/ambari/blob/18e9d2d3/ambari-server/src/test/python/stacks/2.1/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/configs/default.json b/ambari-server/src/test/python/stacks/2.1/configs/default.json index 41a7c23..f878aad 100644 --- a/ambari-server/src/test/python/stacks/2.1/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.1/configs/default.json @@ -666,7 +666,8 @@ "content": "\n# The java implementation to use. If JAVA_HOME is not found we expect java and jar to be in path\nexport JAVA_HOME={{java_home}}\n\n# any additional java opts you want to set. This will apply to both client and server operations\n#export FALCON_OPTS=\n\n# any additional java opts that you want to set for client only\n#export FALCON_CLIENT_OPTS=\n\n# java heap size we want to set for the client. Default is 1024MB\n#export FALCON_CLIENT_HEAP=\n\n# any additional opts you want to set for prisim service.\n#export FALCON_PRISM_OPTS=\n\n# java heap size we want to set for the prisim service. Default is 1024MB\n#export FALCON_PRISM_HEAP=\n\n# any additional opts you want to set for falcon service.\nexport FALCON_SERVER_OPTS=\"-Dfalcon.embeddedmq={{falcon_embeddedmq_enabled}} -Dfalcon.emeddedmq.port={{falcon_emeddedmq_port}}\"\n\n# java heap size we want to set for the falcon server. Default is 1024MB\n#export FALCON_SERVER_HEAP=\n\n# What is is considered as falco n home dir. Default is the base locaion of the installed software\n#export FALCON_HOME_DIR=\n\n# Where log files are stored. Defatult is logs directory under the base install location\nexport FALCON_LOG_DIR={{falcon_log_dir}}\n\n# Where pid files are stored. Defatult is logs directory under the base install location\nexport FALCON_PID_DIR={{falcon_pid_dir}}\n\n# where the falcon active mq data is stored. Defatult is logs/data directory under the base install location\nexport FALCON_DATA_DIR={{falcon_embeddedmq_data}}\n\n# Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir.\n#export FALCON_EXPANDED_WEBAPP_DIR=", "falcon.embeddedmq.data": "/hadoop/falcon/embeddedmq/data", "falcon.embeddedmq": "true", - "falcon_store_uri": "file:///hadoop/falcon/store" + "falcon_store_uri": "file:///hadoop/falcon/store", + "supports_hive_dr": "true" }, "oozie-env": { "oozie_derby_database": "Derby", http://git-wip-us.apache.org/repos/asf/ambari/blob/18e9d2d3/ambari-server/src/test/python/stacks/2.1/configs/secured.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/configs/secured.json b/ambari-server/src/test/python/stacks/2.1/configs/secured.json index be22980..193012e 100644 --- a/ambari-server/src/test/python/stacks/2.1/configs/secured.json +++ b/ambari-server/src/test/python/stacks/2.1/configs/secured.json @@ -658,7 +658,8 @@ "content": "\n# The java implementation to use. If JAVA_HOME is not found we expect java and jar to be in path\nexport JAVA_HOME={{java_home}}\n\n# any additional java opts you want to set. This will apply to both client and server operations\n#export FALCON_OPTS=\n\n# any additional java opts that you want to set for client only\n#export FALCON_CLIENT_OPTS=\n\n# java heap size we want to set for the client. Default is 1024MB\n#export FALCON_CLIENT_HEAP=\n\n# any additional opts you want to set for prisim service.\n#export FALCON_PRISM_OPTS=\n\n# java heap size we want to set for the prisim service. Default is 1024MB\n#export FALCON_PRISM_HEAP=\n\n# any additional opts you want to set for falcon service.\nexport FALCON_SERVER_OPTS=\"-Dfalcon.embeddedmq={{falcon_embeddedmq_enabled}} -Dfalcon.emeddedmq.port={{falcon_emeddedmq_port}}\"\n\n# java heap size we want to set for the falcon server. Default is 1024MB\n#export FALCON_SERVER_HEAP=\n\n# What is is considered as falco n home dir. Default is the base locaion of the installed software\n#export FALCON_HOME_DIR=\n\n# Where log files are stored. Defatult is logs directory under the base install location\nexport FALCON_LOG_DIR={{falcon_log_dir}}\n\n# Where pid files are stored. Defatult is logs directory under the base install location\nexport FALCON_PID_DIR={{falcon_pid_dir}}\n\n# where the falcon active mq data is stored. Defatult is logs/data directory under the base install location\nexport FALCON_DATA_DIR={{falcon_embeddedmq_data}}\n\n# Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir.\n#export FALCON_EXPANDED_WEBAPP_DIR=", "falcon.embeddedmq.data": "/hadoop/falcon/embeddedmq/data", "falcon.embeddedmq": "true", - "falcon_store_uri": "file:///hadoop/falcon/store" + "falcon_store_uri": "file:///hadoop/falcon/store", + "supports_hive_dr": "true" }, "oozie-env": { "oozie_derby_database": "Derby", http://git-wip-us.apache.org/repos/asf/ambari/blob/18e9d2d3/ambari-server/src/test/python/stacks/2.2/configs/falcon-upgrade.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/configs/falcon-upgrade.json b/ambari-server/src/test/python/stacks/2.2/configs/falcon-upgrade.json index c54a4e6..dce5cae 100644 --- a/ambari-server/src/test/python/stacks/2.2/configs/falcon-upgrade.json +++ b/ambari-server/src/test/python/stacks/2.2/configs/falcon-upgrade.json @@ -176,7 +176,8 @@ "content": "\n# The java implementation to use. If JAVA_HOME is not found we expect java and jar to be in path\nexport JAVA_HOME={{java_home}}\n\n# any additional java opts you want to set. This will apply to both client and server operations\n#export FALCON_OPTS=\n\n# any additional java opts that you want to set for client only\n#export FALCON_CLIENT_OPTS=\n\n# java heap size we want to set for the client. Default is 1024MB\n#export FALCON_CLIENT_HEAP=\n\n# any additional opts you want to set for prisim service.\n#export FALCON_PRISM_OPTS=\n\n# java heap size we want to set for the prisim service. Default is 1024MB\n#export FALCON_PRISM_HEAP=\n\n# any additional opts you want to set for falcon service.\nexport FALCON_SERVER_OPTS=\"-Dfalcon.embeddedmq={{falcon_embeddedmq_enabled}} -Dfalcon.emeddedmq.port={{falcon_emeddedmq_port}}\"\n\n# java heap size we want to set for the falcon server. Default is 1024MB\n#export FALCON_SERVER_HEAP=\n\n# What is is considered as falco n home dir. Default is the base location of the installed software\n#export FALCON_HOME_DIR=\n\n# Where log files are stored. Defatult is logs directory under the base install location\nexport FALCON_LOG_DIR={{falcon_log_dir}}\n\n# Where pid files are stored. Defatult is logs directory under the base install location\nexport FALCON_PID_DIR={{falcon_pid_dir}}\n\n# where the falcon active mq data is stored. Defatult is logs/data directory under the base install location\nexport FALCON_DATA_DIR={{falcon_embeddedmq_data}}\n\n# Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir.\n#export FALCON_EXPANDED_WEBAPP_DIR=", "falcon.embeddedmq.data": "/hadoop/falcon/embeddedmq/data", "falcon.embeddedmq": "true", - "falcon_store_uri": "file:///hadoop/falcon/store" + "falcon_store_uri": "file:///hadoop/falcon/store", + "supports_hive_dr": "true" }, "core-site": { "proxyuser_group": "users",
