This is an automated email from the ASF dual-hosted git repository. aonishuk pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push: new 3ea0ad1 AMBARI-23943. Hiveserver2 fails to start on viewFS enabled cluster: {hive_server2_zookeeper_namespace} is not ready yet (aonishuk) 3ea0ad1 is described below commit 3ea0ad173855ff8a2b37c588261f5ac91e0bf2f2 Author: Andrew Onishuk <aonis...@hortonworks.com> AuthorDate: Fri May 25 10:35:50 2018 +0300 AMBARI-23943. Hiveserver2 fails to start on viewFS enabled cluster: {hive_server2_zookeeper_namespace} is not ready yet (aonishuk) --- .../dummy_files/alert_definitions.json | 4 +- .../libraries/providers/hdfs_resource.py | 44 +++++++-- .../common-services/HDFS/2.1.0.2.0/metainfo.xml | 1 + .../4.0.0.2.0/package/scripts/oozie_service.py | 2 +- .../SPARK/1.2.1/package/scripts/livy_server.py | 2 +- .../SPARK2/2.0.0/package/scripts/livy2_server.py | 2 +- .../2.1.0.2.0/package/scripts/resourcemanager.py | 2 +- .../before-START/files/fast-hdfs-resource.jar | Bin 28296600 -> 19286899 bytes .../before-START/scripts/shared_initialization.py | 2 +- .../2.0.6/hooks/before-START/test_before_start.py | 16 +++ .../apache/ambari/fast_hdfs_resource/Resource.java | 9 ++ .../apache/ambari/fast_hdfs_resource/Runner.java | 110 ++++++++++++++------- 12 files changed, 147 insertions(+), 47 deletions(-) diff --git a/ambari-agent/src/test/python/ambari_agent/dummy_files/alert_definitions.json b/ambari-agent/src/test/python/ambari_agent/dummy_files/alert_definitions.json index d9a82a7..341017c 100644 --- a/ambari-agent/src/test/python/ambari_agent/dummy_files/alert_definitions.json +++ b/ambari-agent/src/test/python/ambari_agent/dummy_files/alert_definitions.json @@ -7,9 +7,9 @@ { "name": "namenode_process", "service": "HDFS", - "enabled": true, - "interval": 6, "component": "NAMENODE", + "interval": 6, + "enabled": true, "label": "NameNode process", "source": { "reporting": { diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py index 23bfbc5..a7b43c7 100644 --- a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py +++ b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py @@ -74,6 +74,32 @@ class HdfsResourceJar: while execute does all the expensive creating/deleting work executing the jar with the json as parameter. """ def action_delayed(self, action_name, main_resource): + dfs_type = main_resource.resource.dfs_type + + if main_resource.resource.nameservices is None: # all nameservices + nameservices = namenode_ha_utils.get_nameservices(main_resource.resource.hdfs_site) + else: + nameservices = main_resource.resource.nameservices + + # non-federated cluster + if not nameservices: + self.action_delayed_for_nameservice(None, action_name, main_resource) + else: + for nameservice in nameservices: + try: + if not dfs_type: + raise Fail("<serviceType> for fileSystem service should be set in metainfo.xml") + nameservice = dfs_type.lower() + "://" + nameservice + + self.action_delayed_for_nameservice(nameservice, action_name, main_resource) + except namenode_ha_utils.NoActiveNamenodeException as ex: + # one of ns can be down (during initial start forexample) no need to worry for federated cluster + if len(nameservices) > 1: + Logger.exception("Cannot run HdfsResource for nameservice {0}. Due to no active namenode present".format(nameservice)) + else: + raise + + def action_delayed_for_nameservice(self, nameservice, action_name, main_resource): resource = {} env = Environment.get_instance() if not 'hdfs_files' in env.config: @@ -90,6 +116,8 @@ class HdfsResourceJar: elif getattr(main_resource.resource, field_name): resource[json_field_name] = getattr(main_resource.resource, field_name) + resource['nameservice'] = nameservice + # Add resource to create env.config['hdfs_files'].append(resource) @@ -159,9 +187,9 @@ class WebHDFSUtil: self.logoutput = logoutput @staticmethod - def is_webhdfs_available(is_webhdfs_enabled, default_fs): + def is_webhdfs_available(is_webhdfs_enabled, dfs_type): # only hdfs seems to support webHDFS - return (is_webhdfs_enabled and default_fs.startswith("hdfs")) + return (is_webhdfs_enabled and dfs_type == 'HDFS') def run_command(self, *args, **kwargs): """ @@ -562,11 +590,17 @@ class HdfsResourceWebHDFS: class HdfsResourceProvider(Provider): def __init__(self, resource): super(HdfsResourceProvider,self).__init__(resource) + + self.assert_parameter_is_set('dfs_type') self.fsType = getattr(resource, 'dfs_type') + self.ignored_resources_list = HdfsResourceProvider.get_ignored_resources_list(self.resource.hdfs_resource_ignore_file) - if self.fsType != 'HCFS': + + if self.fsType == 'HDFS': self.assert_parameter_is_set('hdfs_site') self.webhdfs_enabled = self.resource.hdfs_site['dfs.webhdfs.enabled'] + else: + self.webhdfs_enabled = False @staticmethod def parse_path(path): @@ -629,9 +663,7 @@ class HdfsResourceProvider(Provider): self.get_hdfs_resource_executor().action_execute(self) def get_hdfs_resource_executor(self): - if self.fsType == 'HCFS': - return HdfsResourceJar() - elif WebHDFSUtil.is_webhdfs_available(self.webhdfs_enabled, self.resource.default_fs): + if WebHDFSUtil.is_webhdfs_available(self.webhdfs_enabled, self.fsType): return HdfsResourceWebHDFS() else: return HdfsResourceJar() diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml index 19b378b..f26eee9 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml @@ -21,6 +21,7 @@ <service> <name>HDFS</name> <displayName>HDFS</displayName> + <serviceType>HDFS</serviceType> <!-- This tag is used only for main fileSystem service. It sets filesystem schema for ambari --> <comment>Apache Hadoop Distributed File System</comment> <version>2.1.0.2.0</version> diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py index 29d6f4b..612bb29 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py @@ -139,7 +139,7 @@ def oozie_service(action = 'start', upgrade_type=None): params.HdfsResource(None, action="execute") hdfs_share_dir_exists = True # skip time-expensive hadoop fs -ls check - elif WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.default_fs): + elif WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.dfs_type): # check with webhdfs is much faster than executing hadoop fs -ls. util = WebHDFSUtil(params.hdfs_site, nameservice, params.oozie_user, params.security_enabled) list_status = util.run_command(params.hdfs_share_dir, 'GETFILESTATUS', method='GET', ignore_status_codes=['404'], assertable_result=False) diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/livy_server.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/livy_server.py index 7c858c5..3e45774 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/livy_server.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/livy_server.py @@ -114,7 +114,7 @@ class LivyServer(Script): nameservices = namenode_ha_utils.get_nameservices(params.hdfs_site) nameservice = None if not nameservices else nameservices[-1] - if WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.default_fs): + if WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.dfs_type): # check with webhdfs is much faster than executing hdfs dfs -test util = WebHDFSUtil(params.hdfs_site, nameservice, params.hdfs_user, params.security_enabled) list_status = util.run_command(dir_path, 'GETFILESTATUS', method='GET', ignore_status_codes=['404'], assertable_result=False) diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py index 492fd67..27b1d25 100644 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py @@ -113,7 +113,7 @@ class LivyServer(Script): nameservices = namenode_ha_utils.get_nameservices(params.hdfs_site) nameservice = None if not nameservices else nameservices[-1] - if WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.default_fs): + if WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.dfs_type): # check with webhdfs is much faster than executing hdfs dfs -test util = WebHDFSUtil(params.hdfs_site, nameservice, params.hdfs_user, params.security_enabled) list_status = util.run_command(dir_path, 'GETFILESTATUS', method='GET', ignore_status_codes=['404'], assertable_result=False) diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py index 601ced8..99ad69f 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py @@ -230,7 +230,7 @@ class ResourcemanagerDefault(Resourcemanager): nameservices = namenode_ha_utils.get_nameservices(params.hdfs_site) nameservice = None if not nameservices else nameservices[-1] - if WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.default_fs): + if WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.dfs_type): # check with webhdfs is much faster than executing hdfs dfs -test util = WebHDFSUtil(params.hdfs_site, nameservice, params.hdfs_user, params.security_enabled) list_status = util.run_command(dir_path, 'GETFILESTATUS', method='GET', ignore_status_codes=['404'], assertable_result=False) diff --git a/ambari-server/src/main/resources/stack-hooks/before-START/files/fast-hdfs-resource.jar b/ambari-server/src/main/resources/stack-hooks/before-START/files/fast-hdfs-resource.jar index 6c993bf..b8f633f 100644 Binary files a/ambari-server/src/main/resources/stack-hooks/before-START/files/fast-hdfs-resource.jar and b/ambari-server/src/main/resources/stack-hooks/before-START/files/fast-hdfs-resource.jar differ diff --git a/ambari-server/src/main/resources/stack-hooks/before-START/scripts/shared_initialization.py b/ambari-server/src/main/resources/stack-hooks/before-START/scripts/shared_initialization.py index 541de9c..c26265a 100644 --- a/ambari-server/src/main/resources/stack-hooks/before-START/scripts/shared_initialization.py +++ b/ambari-server/src/main/resources/stack-hooks/before-START/scripts/shared_initialization.py @@ -65,7 +65,7 @@ def setup_hadoop(): # if WebHDFS is not enabled we need this jar to create hadoop folders and copy tarballs to HDFS. if params.sysprep_skip_copy_fast_jar_hdfs: print "Skipping copying of fast-hdfs-resource.jar as host is sys prepped" - elif params.dfs_type == 'HCFS' or not WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.default_fs): + elif params.dfs_type == 'HCFS' or not WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.dfs_type): # for source-code of jar goto contrib/fast-hdfs-resource File(format("{ambari_libs_dir}/fast-hdfs-resource.jar"), mode=0644, diff --git a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py index 8e20d17..6329ee4 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py +++ b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py @@ -60,6 +60,10 @@ class TestHookBeforeStart(RMFTestCase): create_parents = True, cd_access = 'a', ) + self.assertResourceCalled('File', '/var/lib/ambari-agent/lib/fast-hdfs-resource.jar', + content = StaticFile('fast-hdfs-resource.jar'), + mode = 0644, + ) self.assertResourceCalled('File', '/etc/hadoop/conf/commons-logging.properties', content = Template('commons-logging.properties.j2'), owner = 'hdfs', @@ -137,6 +141,10 @@ class TestHookBeforeStart(RMFTestCase): create_parents = True, cd_access = 'a', ) + self.assertResourceCalled('File', '/var/lib/ambari-agent/lib/fast-hdfs-resource.jar', + content = StaticFile('fast-hdfs-resource.jar'), + mode = 0644, + ) self.assertResourceCalled('File', '/etc/hadoop/conf/commons-logging.properties', content = Template('commons-logging.properties.j2'), owner = 'root', @@ -219,6 +227,10 @@ class TestHookBeforeStart(RMFTestCase): create_parents = True, cd_access = 'a', ) + self.assertResourceCalled('File', '/var/lib/ambari-agent/lib/fast-hdfs-resource.jar', + content = StaticFile('fast-hdfs-resource.jar'), + mode = 0644, + ) self.assertResourceCalled('File', '/etc/hadoop/conf/commons-logging.properties', content = Template('commons-logging.properties.j2'), owner = 'hdfs', @@ -303,6 +315,10 @@ class TestHookBeforeStart(RMFTestCase): create_parents = True, cd_access = 'a', ) + self.assertResourceCalled('File', '/var/lib/ambari-agent/lib/fast-hdfs-resource.jar', + content = StaticFile('fast-hdfs-resource.jar'), + mode = 0644, + ) self.assertResourceCalled('File', '/etc/hadoop/conf/commons-logging.properties', content = Template('commons-logging.properties.j2'), owner = 'hdfs', diff --git a/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Resource.java b/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Resource.java index 9cbfab2..5c7cbda 100644 --- a/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Resource.java +++ b/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Resource.java @@ -44,6 +44,7 @@ public class Resource { private String owner; private String group; private String mode; + private String nameservice; private boolean recursiveChown; private boolean recursiveChmod; private boolean changePermissionforParents; @@ -105,6 +106,14 @@ public class Resource { this.mode = mode; } + public String getNameservice() { + return nameservice; + } + + public void setNameservice(String nameservice) { + this.nameservice = nameservice; + } + public boolean isRecursiveChown() { return recursiveChown; } diff --git a/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Runner.java b/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Runner.java index 98119b0..9cf0a73 100644 --- a/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Runner.java +++ b/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Runner.java @@ -22,6 +22,10 @@ import java.io.FileReader; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -52,52 +56,88 @@ public class Runner { Gson gson = new Gson(); Resource[] resources = null; - FileSystem dfs = null; + Map<String, FileSystem> fileSystemNameToInstance = new HashMap<String, FileSystem>(); + Map<String, List<Resource>> fileSystemToResource = new HashMap<String, List<Resource>>(); - try { - Configuration conf = new Configuration(); - dfs = FileSystem.get(conf); + try { // 3 - Load data from JSON resources = (Resource[]) gson.fromJson(new FileReader(jsonFilePath), Resource[].class); - - // 4 - Connect to HDFS - System.out.println("Using filesystem uri: " + FileSystem.getDefaultUri(conf).toString()); - dfs.initialize(FileSystem.getDefaultUri(conf), conf); + Configuration conf = new Configuration(); + FileSystem dfs = null; + + // Creating connections for (Resource resource : resources) { - System.out.println("Creating: " + resource); + String nameservice = resource.getNameservice(); + + if(!fileSystemNameToInstance.containsKey(nameservice)) { + URI fileSystemUrl; + if(nameservice == null) { + fileSystemUrl = FileSystem.getDefaultUri(conf); + } else { + fileSystemUrl = new URI(nameservice); + } - Resource.checkResourceParameters(resource, dfs); + dfs = FileSystem.get(fileSystemUrl, conf); - Path pathHadoop = null; + // 4 - Connect to DFS + System.out.println("Initializing filesystem uri: " + fileSystemUrl); + dfs.initialize(fileSystemUrl, conf); - if (resource.getAction().equals("download")) { - pathHadoop = new Path(resource.getSource()); + fileSystemNameToInstance.put(nameservice, dfs); } - else { - String path = resource.getTarget(); - pathHadoop = new Path(path); - if (!resource.isManageIfExists() && dfs.exists(pathHadoop)) { - System.out.println( - String.format("Skipping the operation for not managed DFS directory %s since immutable_paths contains it.", path) - ); - continue; - } + + if(!fileSystemToResource.containsKey(nameservice)) { + fileSystemToResource.put(nameservice, new ArrayList<Resource>()); } + fileSystemToResource.get(nameservice).add(resource); + } - if (resource.getAction().equals("create")) { - // 5 - Create - Resource.createResource(resource, dfs, pathHadoop); - Resource.setMode(resource, dfs, pathHadoop); - Resource.setOwner(resource, dfs, pathHadoop); - } else if (resource.getAction().equals("delete")) { - // 6 - Delete - dfs.delete(pathHadoop, true); - } else if (resource.getAction().equals("download")) { - // 7 - Download - dfs.copyToLocalFile(pathHadoop, new Path(resource.getTarget())); + //for (Resource resource : resources) { + for (Map.Entry<String, List<Resource>> entry : fileSystemToResource.entrySet()) { + String nameservice = entry.getKey(); + List<Resource> resourcesNameservice = entry.getValue(); + + for(Resource resource: resourcesNameservice) { + if (nameservice != null) { + System.out.println("Creating: " + resource + " in " + nameservice); + } else { + System.out.println("Creating: " + resource + " in default filesystem"); + } + + dfs = fileSystemNameToInstance.get(nameservice); + + Resource.checkResourceParameters(resource, dfs); + + Path pathHadoop = null; + + if (resource.getAction().equals("download")) { + pathHadoop = new Path(resource.getSource()); + } else { + String path = resource.getTarget(); + pathHadoop = new Path(path); + if (!resource.isManageIfExists() && dfs.exists(pathHadoop)) { + System.out.println( + String.format("Skipping the operation for not managed DFS directory %s since immutable_paths contains it.", path) + ); + continue; + } + } + + if (resource.getAction().equals("create")) { + // 5 - Create + Resource.createResource(resource, dfs, pathHadoop); + Resource.setMode(resource, dfs, pathHadoop); + Resource.setOwner(resource, dfs, pathHadoop); + } else if (resource.getAction().equals("delete")) { + // 6 - Delete + dfs.delete(pathHadoop, true); + } else if (resource.getAction().equals("download")) { + // 7 - Download + dfs.copyToLocalFile(pathHadoop, new Path(resource.getTarget())); + } } } } @@ -106,7 +146,9 @@ public class Runner { e.printStackTrace(); } finally { - dfs.close(); + for(FileSystem dfs:fileSystemNameToInstance.values()) { + dfs.close(); + } } System.out.println("All resources created."); -- To stop receiving notification emails like this one, please contact aonis...@apache.org.