Repository: ambari Updated Branches: refs/heads/trunk fc586a87a -> 0ed40ae0f
AMBARI-19328. Fix permissions for Druid config files and data directories (Nishant Bangarwa via smohanty) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0ed40ae0 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0ed40ae0 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0ed40ae0 Branch: refs/heads/trunk Commit: 0ed40ae0f818cc083363d33c493ba13bb5ef2a05 Parents: fc586a8 Author: Sumit Mohanty <[email protected]> Authored: Fri Jan 6 13:25:21 2017 -0800 Committer: Sumit Mohanty <[email protected]> Committed: Fri Jan 6 14:02:08 2017 -0800 ---------------------------------------------------------------------- .../0.9.2/configuration/druid-historical.xml | 4 +- .../0.9.2/configuration/druid-middlemanager.xml | 2 +- .../DRUID/0.9.2/package/scripts/druid.py | 36 ++++++++-- .../DRUID/0.9.2/package/scripts/params.py | 2 + .../test/python/stacks/2.6/DRUID/test_druid.py | 74 +++++++++++++------- .../test/python/stacks/2.6/configs/default.json | 6 +- 6 files changed, 87 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/0ed40ae0/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml index 0545e4c..9b65404 100644 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml @@ -61,7 +61,7 @@ </property> <property> <name>druid.segmentCache.locations</name> - <value>[{"path":"/apps/druid/segmentCache","maxSize"\:300000000000}]</value> + <value>[{"path":"/apps/druid/segmentCache","maxSize":300000000000}]</value> <description>Segments assigned to a Historical node are first stored on the local file system (in a disk cache) and then served by the Historical node. These locations define where that local cache resides. </description> @@ -78,7 +78,7 @@ </property> <property> <name>druid.segmentCache.infoDir</name> - <value>/apps/druid/segmentCache</value> + <value>/apps/druid/segmentCache/info_dir</value> <description>Historical nodes keep track of the segments they are serving so that when the process is restarted they can reload the same segments without waiting for the Coordinator to reassign. This path defines where this metadata is kept. Directory will be created if needed. http://git-wip-us.apache.org/repos/asf/ambari/blob/0ed40ae0/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-middlemanager.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-middlemanager.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-middlemanager.xml index 967438e..921d16d 100644 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-middlemanager.xml +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-middlemanager.xml @@ -55,7 +55,7 @@ </property> <property> <name>druid.indexer.task.baseTaskDir</name> - <value>/tmp/persistent/tasks</value> + <value>/apps/druid/tasks</value> <description> Base temporary working directory for druid tasks. </description> http://git-wip-us.apache.org/repos/asf/ambari/blob/0ed40ae0/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py index 6ddedac..4a4f1c5 100644 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py @@ -16,13 +16,13 @@ See the License for the specific language governing permissions and limitations under the License. """ +import json import os from resource_management.libraries.resources.properties_file import PropertiesFile from resource_management.core.resources.system import Directory, Execute, File from resource_management.core.source import DownloadSource from resource_management.core.source import InlineTemplate from resource_management.libraries.functions import format -from resource_management.libraries.resources import XmlConfig from resource_management.libraries.functions.show_logs import show_logs from resource_management.core.logger import Logger @@ -34,7 +34,8 @@ def druid(upgrade_type=None, nodeType=None): # Environment Variables File(format("{params.druid_conf_dir}/druid-env.sh"), owner=params.druid_user, - content=InlineTemplate(params.druid_env_sh_template) + content=InlineTemplate(params.druid_env_sh_template), + mode = 0700 ) # common config @@ -60,6 +61,7 @@ def druid(upgrade_type=None, nodeType=None): properties=druid_common_config, owner=params.druid_user, group=params.user_group, + mode = 0600 ) Logger.info("Created common.runtime.properties") @@ -92,6 +94,7 @@ def druid(upgrade_type=None, nodeType=None): properties=node_config, owner=params.druid_user, group=params.user_group, + mode = 0600 ) Logger.info(format("Created druid-{node_type_lowercase} runtime.properties")) @@ -177,17 +180,38 @@ def create_hadoop_directory(hadoop_dir): def ensure_base_directories(): import params Directory( - [params.druid_log_dir, params.druid_pid_dir, params.druid_common_conf_dir, params.druid_coordinator_conf_dir, - params.druid_broker_conf_dir, params.druid_middlemanager_conf_dir, params.druid_historical_conf_dir, - params.druid_overlord_conf_dir, params.druid_router_conf_dir, params.druid_segment_infoDir], + [params.druid_log_dir, params.druid_pid_dir], mode=0755, - cd_access='a', owner=params.druid_user, group=params.user_group, create_parents=True, recursive_ownership=True, ) + Directory( + [params.druid_conf_dir, params.druid_common_conf_dir, params.druid_coordinator_conf_dir, + params.druid_broker_conf_dir, params.druid_middlemanager_conf_dir, params.druid_historical_conf_dir, + params.druid_overlord_conf_dir, params.druid_router_conf_dir, params.druid_segment_infoDir, + params.druid_tasks_dir], + mode=0700, + owner=params.druid_user, + group=params.user_group, + create_parents=True, + recursive_ownership=True, + ) + + segment_cache_locations = json.loads(params.druid_segment_cache_locations) + for segment_cache_location in segment_cache_locations: + Directory( + segment_cache_location["path"], + mode=0700, + owner=params.druid_user, + group=params.user_group, + create_parents=True, + recursive_ownership=True, + ) + + def get_daemon_cmd(params=None, node_type=None, command=None): return format('source {params.druid_conf_dir}/druid-env.sh ; {params.druid_home}/bin/node.sh {node_type} {command}') http://git-wip-us.apache.org/repos/asf/ambari/blob/0ed40ae0/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py index 8292634..c83b16e 100644 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py @@ -67,6 +67,8 @@ druid_router_conf_dir = druid_conf_dir + "/router" druid_extensions_dir = druid_home + "/extensions" druid_hadoop_dependencies_dir = druid_home + "/hadoop-dependencies" druid_segment_infoDir = config['configurations']['druid-historical']['druid.segmentCache.infoDir'] +druid_segment_cache_locations = config['configurations']['druid-historical']['druid.segmentCache.locations'] +druid_tasks_dir = config['configurations']['druid-middlemanager']['druid.indexer.task.baseTaskDir'] druid_user = config['configurations']['druid-env']['druid_user'] druid_log_dir = config['configurations']['druid-env']['druid_log_dir'] druid_classpath = config['configurations']['druid-env']['druid_classpath'] http://git-wip-us.apache.org/repos/asf/ambari/blob/0ed40ae0/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py b/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py index 95cbb94..104dce6 100644 --- a/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py +++ b/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py @@ -321,7 +321,6 @@ class TestDruid(RMFTestCase): self.assertResourceCalled('Directory', '/var/log/druid', mode=0755, - cd_access='a', owner='druid', group='hadoop', create_parents=True, @@ -330,7 +329,14 @@ class TestDruid(RMFTestCase): self.assertResourceCalled('Directory', '/var/run/druid', mode=0755, - cd_access='a', + owner='druid', + group='hadoop', + create_parents=True, + recursive_ownership=True + ) + + self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf'), + mode=0700, owner='druid', group='hadoop', create_parents=True, @@ -338,8 +344,7 @@ class TestDruid(RMFTestCase): ) self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/_common'), - mode=0755, - cd_access='a', + mode=0700, owner='druid', group='hadoop', create_parents=True, @@ -347,8 +352,7 @@ class TestDruid(RMFTestCase): ) self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/coordinator'), - mode=0755, - cd_access='a', + mode=0700, owner='druid', group='hadoop', create_parents=True, @@ -356,8 +360,7 @@ class TestDruid(RMFTestCase): ) self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/broker'), - mode=0755, - cd_access='a', + mode=0700, owner='druid', group='hadoop', create_parents=True, @@ -365,8 +368,7 @@ class TestDruid(RMFTestCase): ) self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/middleManager'), - mode=0755, - cd_access='a', + mode=0700, owner='druid', group='hadoop', create_parents=True, @@ -374,8 +376,7 @@ class TestDruid(RMFTestCase): ) self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/historical'), - mode=0755, - cd_access='a', + mode=0700, owner='druid', group='hadoop', create_parents=True, @@ -383,8 +384,7 @@ class TestDruid(RMFTestCase): ) self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/overlord'), - mode=0755, - cd_access='a', + mode=0700, owner='druid', group='hadoop', create_parents=True, @@ -392,8 +392,23 @@ class TestDruid(RMFTestCase): ) self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/router'), - mode=0755, - cd_access='a', + mode=0700, + owner='druid', + group='hadoop', + create_parents=True, + recursive_ownership=True + ) + + self.assertResourceCalled('Directory', '/apps/druid/segmentCache/info_dir', + mode=0700, + owner='druid', + group='hadoop', + create_parents=True, + recursive_ownership=True + ) + + self.assertResourceCalled('Directory', '/apps/druid/tasks', + mode=0700, owner='druid', group='hadoop', create_parents=True, @@ -401,8 +416,7 @@ class TestDruid(RMFTestCase): ) self.assertResourceCalled('Directory', '/apps/druid/segmentCache', - mode=0755, - cd_access='a', + mode=0700, owner='druid', group='hadoop', create_parents=True, @@ -411,7 +425,8 @@ class TestDruid(RMFTestCase): self.assertResourceCalled('File', format('/usr/hdp/current/{role}/conf/druid-env.sh'), owner = 'druid', - content = InlineTemplate(self.getConfig()['configurations']['druid-env']['content']) + content = InlineTemplate(self.getConfig()['configurations']['druid-env']['content']), + mode = 0700 ) druid_common_config = mutable_config_dict(self.getConfig()['configurations']['druid-common']) druid_common_config['druid.host'] = 'c6401.ambari.apache.org' @@ -424,7 +439,8 @@ class TestDruid(RMFTestCase): dir=format("/usr/hdp/current/{role}/conf/_common"), properties=druid_common_config, owner='druid', - group='hadoop' + group='hadoop', + mode = 0600 ) self.assertResourceCalled('File', format('/usr/hdp/current/{role}/conf/_common/druid-log4j.xml'), @@ -445,7 +461,8 @@ class TestDruid(RMFTestCase): dir=format('/usr/hdp/current/{role}/conf/coordinator'), properties=self.getConfig()['configurations']['druid-coordinator'], owner='druid', - group='hadoop' + group='hadoop', + mode = 0600 ) self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/coordinator/jvm.config"), @@ -463,7 +480,8 @@ class TestDruid(RMFTestCase): dir=format('/usr/hdp/current/{role}/conf/overlord'), properties=self.getConfig()['configurations']['druid-overlord'], owner='druid', - group='hadoop' + group='hadoop', + mode = 0600 ) self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/overlord/jvm.config"), @@ -481,7 +499,8 @@ class TestDruid(RMFTestCase): dir=format('/usr/hdp/current/{role}/conf/historical'), properties=self.getConfig()['configurations']['druid-historical'], owner='druid', - group='hadoop' + group='hadoop', + mode = 0600 ) self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/historical/jvm.config"), @@ -500,7 +519,8 @@ class TestDruid(RMFTestCase): dir=format('/usr/hdp/current/{role}/conf/broker'), properties=self.getConfig()['configurations']['druid-broker'], owner='druid', - group='hadoop' + group='hadoop', + mode = 0600 ) self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/broker/jvm.config"), @@ -519,7 +539,8 @@ class TestDruid(RMFTestCase): dir=format('/usr/hdp/current/{role}/conf/middleManager'), properties=self.getConfig()['configurations']['druid-middlemanager'], owner='druid', - group='hadoop' + group='hadoop', + mode = 0600 ) self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/middleManager/jvm.config"), @@ -537,7 +558,8 @@ class TestDruid(RMFTestCase): dir=format('/usr/hdp/current/{role}/conf/router'), properties=self.getConfig()['configurations']['druid-router'], owner='druid', - group='hadoop' + group='hadoop', + mode = 0600 ) self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/router/jvm.config"), http://git-wip-us.apache.org/repos/asf/ambari/blob/0ed40ae0/ambari-server/src/test/python/stacks/2.6/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.6/configs/default.json b/ambari-server/src/test/python/stacks/2.6/configs/default.json index c3f1d55..fa12757 100644 --- a/ambari-server/src/test/python/stacks/2.6/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.6/configs/default.json @@ -405,7 +405,8 @@ "druid.extensions.loadList": "[\"mysql-metadata-storage\", \"druid-datasketches\"]" }, "druid-historical" : { - "druid.segmentCache.infoDir" : "/apps/druid/segmentCache" + "druid.segmentCache.infoDir" : "/apps/druid/segmentCache/info_dir", + "druid.segmentCache.locations" :"[{\"path\":\"/apps/druid/segmentCache\",\"maxSize\":300000000000}]" }, "druid-coordinator" : { "druid.service" : "druid/coordinator" @@ -418,7 +419,8 @@ }, "druid-middlemanager" : { "druid.service" : "druid/middlemanager", - "druid.indexer.task.hadoopWorkingPath" : "/tmp/druid-indexing" + "druid.indexer.task.hadoopWorkingPath" : "/tmp/druid-indexing", + "druid.indexer.task.baseTaskDir" : "/apps/druid/tasks" }, "druid-router" : { "druid.service" : "druid/router"
