Repository: ambari Updated Branches: refs/heads/branch-2.5 212ba9136 -> 4780973a5
AMBARI-19328. Fix permissions for Druid config files and data directories (Nishant Bangarwa via smohanty) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4780973a Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4780973a Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4780973a Branch: refs/heads/branch-2.5 Commit: 4780973a5627723a22e735b32c2c76cf009a9119 Parents: 212ba91 Author: Sumit Mohanty <[email protected]> Authored: Fri Jan 6 13:25:21 2017 -0800 Committer: Sumit Mohanty <[email protected]> Committed: Fri Jan 6 13:25:21 2017 -0800 ---------------------------------------------------------------------- .../0.9.2/configuration/druid-historical.xml | 4 +- .../0.9.2/configuration/druid-middlemanager.xml | 2 +- .../DRUID/0.9.2/package/scripts/druid.py | 36 ++++++++-- .../DRUID/0.9.2/package/scripts/params.py | 2 + .../test/python/stacks/2.6/DRUID/test_druid.py | 74 +++++++++++++------- .../test/python/stacks/2.6/configs/default.json | 6 +- 6 files changed, 87 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/4780973a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml index 0545e4c..9b65404 100644 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml @@ -61,7 +61,7 @@ </property> <property> <name>druid.segmentCache.locations</name> - <value>[{"path":"/apps/druid/segmentCache","maxSize"\:300000000000}]</value> + <value>[{"path":"/apps/druid/segmentCache","maxSize":300000000000}]</value> <description>Segments assigned to a Historical node are first stored on the local file system (in a disk cache) and then served by the Historical node. These locations define where that local cache resides. </description> @@ -78,7 +78,7 @@ </property> <property> <name>druid.segmentCache.infoDir</name> - <value>/apps/druid/segmentCache</value> + <value>/apps/druid/segmentCache/info_dir</value> <description>Historical nodes keep track of the segments they are serving so that when the process is restarted they can reload the same segments without waiting for the Coordinator to reassign. This path defines where this metadata is kept. Directory will be created if needed. http://git-wip-us.apache.org/repos/asf/ambari/blob/4780973a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-middlemanager.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-middlemanager.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-middlemanager.xml index 967438e..921d16d 100644 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-middlemanager.xml +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-middlemanager.xml @@ -55,7 +55,7 @@ </property> <property> <name>druid.indexer.task.baseTaskDir</name> - <value>/tmp/persistent/tasks</value> + <value>/apps/druid/tasks</value> <description> Base temporary working directory for druid tasks. </description> http://git-wip-us.apache.org/repos/asf/ambari/blob/4780973a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py index 6ddedac..4a4f1c5 100644 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py @@ -16,13 +16,13 @@ See the License for the specific language governing permissions and limitations under the License. """ +import json import os from resource_management.libraries.resources.properties_file import PropertiesFile from resource_management.core.resources.system import Directory, Execute, File from resource_management.core.source import DownloadSource from resource_management.core.source import InlineTemplate from resource_management.libraries.functions import format -from resource_management.libraries.resources import XmlConfig from resource_management.libraries.functions.show_logs import show_logs from resource_management.core.logger import Logger @@ -34,7 +34,8 @@ def druid(upgrade_type=None, nodeType=None): # Environment Variables File(format("{params.druid_conf_dir}/druid-env.sh"), owner=params.druid_user, - content=InlineTemplate(params.druid_env_sh_template) + content=InlineTemplate(params.druid_env_sh_template), + mode = 0700 ) # common config @@ -60,6 +61,7 @@ def druid(upgrade_type=None, nodeType=None): properties=druid_common_config, owner=params.druid_user, group=params.user_group, + mode = 0600 ) Logger.info("Created common.runtime.properties") @@ -92,6 +94,7 @@ def druid(upgrade_type=None, nodeType=None): properties=node_config, owner=params.druid_user, group=params.user_group, + mode = 0600 ) Logger.info(format("Created druid-{node_type_lowercase} runtime.properties")) @@ -177,17 +180,38 @@ def create_hadoop_directory(hadoop_dir): def ensure_base_directories(): import params Directory( - [params.druid_log_dir, params.druid_pid_dir, params.druid_common_conf_dir, params.druid_coordinator_conf_dir, - params.druid_broker_conf_dir, params.druid_middlemanager_conf_dir, params.druid_historical_conf_dir, - params.druid_overlord_conf_dir, params.druid_router_conf_dir, params.druid_segment_infoDir], + [params.druid_log_dir, params.druid_pid_dir], mode=0755, - cd_access='a', owner=params.druid_user, group=params.user_group, create_parents=True, recursive_ownership=True, ) + Directory( + [params.druid_conf_dir, params.druid_common_conf_dir, params.druid_coordinator_conf_dir, + params.druid_broker_conf_dir, params.druid_middlemanager_conf_dir, params.druid_historical_conf_dir, + params.druid_overlord_conf_dir, params.druid_router_conf_dir, params.druid_segment_infoDir, + params.druid_tasks_dir], + mode=0700, + owner=params.druid_user, + group=params.user_group, + create_parents=True, + recursive_ownership=True, + ) + + segment_cache_locations = json.loads(params.druid_segment_cache_locations) + for segment_cache_location in segment_cache_locations: + Directory( + segment_cache_location["path"], + mode=0700, + owner=params.druid_user, + group=params.user_group, + create_parents=True, + recursive_ownership=True, + ) + + def get_daemon_cmd(params=None, node_type=None, command=None): return format('source {params.druid_conf_dir}/druid-env.sh ; {params.druid_home}/bin/node.sh {node_type} {command}') http://git-wip-us.apache.org/repos/asf/ambari/blob/4780973a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py index 8292634..c83b16e 100644 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py @@ -67,6 +67,8 @@ druid_router_conf_dir = druid_conf_dir + "/router" druid_extensions_dir = druid_home + "/extensions" druid_hadoop_dependencies_dir = druid_home + "/hadoop-dependencies" druid_segment_infoDir = config['configurations']['druid-historical']['druid.segmentCache.infoDir'] +druid_segment_cache_locations = config['configurations']['druid-historical']['druid.segmentCache.locations'] +druid_tasks_dir = config['configurations']['druid-middlemanager']['druid.indexer.task.baseTaskDir'] druid_user = config['configurations']['druid-env']['druid_user'] druid_log_dir = config['configurations']['druid-env']['druid_log_dir'] druid_classpath = config['configurations']['druid-env']['druid_classpath'] http://git-wip-us.apache.org/repos/asf/ambari/blob/4780973a/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py b/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py index 95cbb94..104dce6 100644 --- a/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py +++ b/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py @@ -321,7 +321,6 @@ class TestDruid(RMFTestCase): self.assertResourceCalled('Directory', '/var/log/druid', mode=0755, - cd_access='a', owner='druid', group='hadoop', create_parents=True, @@ -330,7 +329,14 @@ class TestDruid(RMFTestCase): self.assertResourceCalled('Directory', '/var/run/druid', mode=0755, - cd_access='a', + owner='druid', + group='hadoop', + create_parents=True, + recursive_ownership=True + ) + + self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf'), + mode=0700, owner='druid', group='hadoop', create_parents=True, @@ -338,8 +344,7 @@ class TestDruid(RMFTestCase): ) self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/_common'), - mode=0755, - cd_access='a', + mode=0700, owner='druid', group='hadoop', create_parents=True, @@ -347,8 +352,7 @@ class TestDruid(RMFTestCase): ) self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/coordinator'), - mode=0755, - cd_access='a', + mode=0700, owner='druid', group='hadoop', create_parents=True, @@ -356,8 +360,7 @@ class TestDruid(RMFTestCase): ) self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/broker'), - mode=0755, - cd_access='a', + mode=0700, owner='druid', group='hadoop', create_parents=True, @@ -365,8 +368,7 @@ class TestDruid(RMFTestCase): ) self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/middleManager'), - mode=0755, - cd_access='a', + mode=0700, owner='druid', group='hadoop', create_parents=True, @@ -374,8 +376,7 @@ class TestDruid(RMFTestCase): ) self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/historical'), - mode=0755, - cd_access='a', + mode=0700, owner='druid', group='hadoop', create_parents=True, @@ -383,8 +384,7 @@ class TestDruid(RMFTestCase): ) self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/overlord'), - mode=0755, - cd_access='a', + mode=0700, owner='druid', group='hadoop', create_parents=True, @@ -392,8 +392,23 @@ class TestDruid(RMFTestCase): ) self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/router'), - mode=0755, - cd_access='a', + mode=0700, + owner='druid', + group='hadoop', + create_parents=True, + recursive_ownership=True + ) + + self.assertResourceCalled('Directory', '/apps/druid/segmentCache/info_dir', + mode=0700, + owner='druid', + group='hadoop', + create_parents=True, + recursive_ownership=True + ) + + self.assertResourceCalled('Directory', '/apps/druid/tasks', + mode=0700, owner='druid', group='hadoop', create_parents=True, @@ -401,8 +416,7 @@ class TestDruid(RMFTestCase): ) self.assertResourceCalled('Directory', '/apps/druid/segmentCache', - mode=0755, - cd_access='a', + mode=0700, owner='druid', group='hadoop', create_parents=True, @@ -411,7 +425,8 @@ class TestDruid(RMFTestCase): self.assertResourceCalled('File', format('/usr/hdp/current/{role}/conf/druid-env.sh'), owner = 'druid', - content = InlineTemplate(self.getConfig()['configurations']['druid-env']['content']) + content = InlineTemplate(self.getConfig()['configurations']['druid-env']['content']), + mode = 0700 ) druid_common_config = mutable_config_dict(self.getConfig()['configurations']['druid-common']) druid_common_config['druid.host'] = 'c6401.ambari.apache.org' @@ -424,7 +439,8 @@ class TestDruid(RMFTestCase): dir=format("/usr/hdp/current/{role}/conf/_common"), properties=druid_common_config, owner='druid', - group='hadoop' + group='hadoop', + mode = 0600 ) self.assertResourceCalled('File', format('/usr/hdp/current/{role}/conf/_common/druid-log4j.xml'), @@ -445,7 +461,8 @@ class TestDruid(RMFTestCase): dir=format('/usr/hdp/current/{role}/conf/coordinator'), properties=self.getConfig()['configurations']['druid-coordinator'], owner='druid', - group='hadoop' + group='hadoop', + mode = 0600 ) self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/coordinator/jvm.config"), @@ -463,7 +480,8 @@ class TestDruid(RMFTestCase): dir=format('/usr/hdp/current/{role}/conf/overlord'), properties=self.getConfig()['configurations']['druid-overlord'], owner='druid', - group='hadoop' + group='hadoop', + mode = 0600 ) self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/overlord/jvm.config"), @@ -481,7 +499,8 @@ class TestDruid(RMFTestCase): dir=format('/usr/hdp/current/{role}/conf/historical'), properties=self.getConfig()['configurations']['druid-historical'], owner='druid', - group='hadoop' + group='hadoop', + mode = 0600 ) self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/historical/jvm.config"), @@ -500,7 +519,8 @@ class TestDruid(RMFTestCase): dir=format('/usr/hdp/current/{role}/conf/broker'), properties=self.getConfig()['configurations']['druid-broker'], owner='druid', - group='hadoop' + group='hadoop', + mode = 0600 ) self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/broker/jvm.config"), @@ -519,7 +539,8 @@ class TestDruid(RMFTestCase): dir=format('/usr/hdp/current/{role}/conf/middleManager'), properties=self.getConfig()['configurations']['druid-middlemanager'], owner='druid', - group='hadoop' + group='hadoop', + mode = 0600 ) self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/middleManager/jvm.config"), @@ -537,7 +558,8 @@ class TestDruid(RMFTestCase): dir=format('/usr/hdp/current/{role}/conf/router'), properties=self.getConfig()['configurations']['druid-router'], owner='druid', - group='hadoop' + group='hadoop', + mode = 0600 ) self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/router/jvm.config"), http://git-wip-us.apache.org/repos/asf/ambari/blob/4780973a/ambari-server/src/test/python/stacks/2.6/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.6/configs/default.json b/ambari-server/src/test/python/stacks/2.6/configs/default.json index c3f1d55..fa12757 100644 --- a/ambari-server/src/test/python/stacks/2.6/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.6/configs/default.json @@ -405,7 +405,8 @@ "druid.extensions.loadList": "[\"mysql-metadata-storage\", \"druid-datasketches\"]" }, "druid-historical" : { - "druid.segmentCache.infoDir" : "/apps/druid/segmentCache" + "druid.segmentCache.infoDir" : "/apps/druid/segmentCache/info_dir", + "druid.segmentCache.locations" :"[{\"path\":\"/apps/druid/segmentCache\",\"maxSize\":300000000000}]" }, "druid-coordinator" : { "druid.service" : "druid/coordinator" @@ -418,7 +419,8 @@ }, "druid-middlemanager" : { "druid.service" : "druid/middlemanager", - "druid.indexer.task.hadoopWorkingPath" : "/tmp/druid-indexing" + "druid.indexer.task.hadoopWorkingPath" : "/tmp/druid-indexing", + "druid.indexer.task.baseTaskDir" : "/apps/druid/tasks" }, "druid-router" : { "druid.service" : "druid/router"
