This is an automated email from the ASF dual-hosted git repository.
aonishuk pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new f231e83 AMBARI-24839. Ambari is trying to create hbase.rootdir using
s3 url (#2609)
f231e83 is described below
commit f231e836afbd05d4ffd961226d1f6e5996b43816
Author: aonishuk <aonis...@hortonworks.com>
AuthorDate: Thu Nov 15 11:48:30 2018 +0200
AMBARI-24839. Ambari is trying to create hbase.rootdir using s3 url (#2609)
* AMBARI-24839. Ambari is trying to create hbase.rootdir using s3 url
(aonishuk)
* AMBARI-24839. Ambari is trying to create hbase.rootdir using s3 url
(aonishuk)
---
.../dummy_files/alert_definitions.json | 32 ++++++++---------
.../libraries/providers/hdfs_resource.py | 36 +++++++++++++++----
.../before-START/files/fast-hdfs-resource.jar | Bin 19286899 -> 16202231
bytes
.../stack-hooks/before-START/scripts/params.py | 3 ++
.../before-START/scripts/shared_initialization.py | 27 +++++++-------
.../stack-hooks/before-START/test_before_start.py | 32 ++++++++---------
.../apache/ambari/fast_hdfs_resource/Runner.java | 40 +++++++++++++++------
7 files changed, 108 insertions(+), 62 deletions(-)
diff --git
a/ambari-agent/src/test/python/ambari_agent/dummy_files/alert_definitions.json
b/ambari-agent/src/test/python/ambari_agent/dummy_files/alert_definitions.json
index d9a82a7..5962ac6 100644
---
a/ambari-agent/src/test/python/ambari_agent/dummy_files/alert_definitions.json
+++
b/ambari-agent/src/test/python/ambari_agent/dummy_files/alert_definitions.json
@@ -1,33 +1,33 @@
{
"0": {
- "clusterName": "c1",
- "hash": "12341234134412341243124",
- "hostName": "c6401.ambari.apache.org",
+ "clusterName": "c1",
+ "hash": "12341234134412341243124",
+ "hostName": "c6401.ambari.apache.org",
"alertDefinitions": [
{
- "name": "namenode_process",
- "service": "HDFS",
- "enabled": true,
- "interval": 6,
- "component": "NAMENODE",
- "label": "NameNode process",
+ "name": "namenode_process",
+ "service": "HDFS",
+ "component": "NAMENODE",
+ "interval": 6,
+ "enabled": true,
+ "label": "NameNode process",
"source": {
"reporting": {
"critical": {
"text": "Could not load process info: {0} on host {1}:{2}"
- },
+ },
"ok": {
"text": "TCP OK - {0:.4f} response time on port {1}"
}
- },
- "type": "PORT",
- "uri": "{{hdfs-site/dfs.namenode.http-address}}",
+ },
+ "type": "PORT",
+ "uri": "{{hdfs-site/dfs.namenode.http-address}}",
"default_port": 50070
- },
- "scope": "HOST",
+ },
+ "scope": "HOST",
"uuid": "3f82ae27-fa6a-465b-b77d-67963ac55d2f"
}
- ],
+ ],
"configurations": {
"hdfs-site": {
"dfs.namenode.http-address": "c6401.ambari.apache.org:50070"
diff --git
a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
index 4a22c39..b22e9b2 100644
---
a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
+++
b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
@@ -35,6 +35,7 @@ from resource_management.core.logger import Logger
from resource_management.core.providers import Provider
from resource_management.core.resources.system import Execute
from resource_management.core.resources.system import File
+from resource_management.libraries.functions.is_empty import is_empty
from resource_management.libraries.functions import format
from resource_management.libraries.functions import namenode_ha_utils
from resource_management.libraries.functions.get_user_call_output import
get_user_call_output
@@ -270,6 +271,8 @@ class WebHDFSUtil:
if file_to_put:
cmd += ["--data-binary", "@"+file_to_put, "-H", "Content-Type:
application/octet-stream"]
+ else:
+ cmd += ["-d", "", "-H", "Content-Length: 0"]
if self.security_enabled:
cmd += ["--negotiate", "-u", ":"]
@@ -612,8 +615,16 @@ class HdfsResourceProvider(Provider):
def __init__(self, resource):
super(HdfsResourceProvider,self).__init__(resource)
+ self.has_core_configs = not is_empty(getattr(resource, 'default_fs'))
+
+ if not self.has_core_configs:
+ self.webhdfs_enabled = False
+ self.fsType = None
+ return
+
self.assert_parameter_is_set('dfs_type')
self.fsType = getattr(resource, 'dfs_type')
+ self.can_use_webhdfs = True
self.ignored_resources_list =
HdfsResourceProvider.get_ignored_resources_list(self.resource.hdfs_resource_ignore_file)
@@ -658,14 +669,20 @@ class HdfsResourceProvider(Provider):
return hdfs_resources_to_ignore
def action_delayed(self, action_name):
+ if not self.has_core_configs:
+ Logger.info("Cannot find core-site or core-site/fs.defaultFs. Assuming
usage of external filesystem for services. Ambari will not manage the
directories.")
+ return
+
self.assert_parameter_is_set('type')
-
+
path_protocol = urlparse(self.resource.target).scheme.lower()
default_fs_protocol = urlparse(self.resource.default_fs).scheme.lower()
+ # for protocols which are different that defaultFs webhdfs will not be
able to create directories
+ # so for them fast-hdfs-resource.jar should be used
if path_protocol and default_fs_protocol != "viewfs" and path_protocol !=
default_fs_protocol:
- Logger.info("Skipping creation of {0} since it is not in default
filesystem.".format(self.resource.target))
- return
+ self.can_use_webhdfs = False
+ Logger.info("Cannot use webhdfs for {0} defaultFs = {1} has different
protocol".format(self.resource.target, self.resource.default_fs))
parsed_path = HdfsResourceProvider.parse_path(self.resource.target)
@@ -688,10 +705,15 @@ class HdfsResourceProvider(Provider):
self.action_delayed("download")
def action_execute(self):
- self.get_hdfs_resource_executor().action_execute(self)
+ if not self.has_core_configs:
+ Logger.info("Cannot find core-site or core-site/fs.defaultFs. Assuming
usage of external filesystem for services. Ambari will not manage the
directories.")
+ return
+
+ HdfsResourceWebHDFS().action_execute(self)
+ HdfsResourceJar().action_execute(self)
def get_hdfs_resource_executor(self):
- if WebHDFSUtil.is_webhdfs_available(self.webhdfs_enabled, self.fsType):
+ if self.can_use_webhdfs and
WebHDFSUtil.is_webhdfs_available(self.webhdfs_enabled, self.fsType):
return HdfsResourceWebHDFS()
else:
return HdfsResourceJar()
@@ -709,5 +731,7 @@ class HdfsResourceProvider(Provider):
Execute(format("{kinit_path} -kt {keytab_file} {principal_name}"),
user=user
- )
+ )
+
+
diff --git
a/ambari-server/src/main/resources/stack-hooks/before-START/files/fast-hdfs-resource.jar
b/ambari-server/src/main/resources/stack-hooks/before-START/files/fast-hdfs-resource.jar
index b8f633f..ff018ca 100644
Binary files
a/ambari-server/src/main/resources/stack-hooks/before-START/files/fast-hdfs-resource.jar
and
b/ambari-server/src/main/resources/stack-hooks/before-START/files/fast-hdfs-resource.jar
differ
diff --git
a/ambari-server/src/main/resources/stack-hooks/before-START/scripts/params.py
b/ambari-server/src/main/resources/stack-hooks/before-START/scripts/params.py
index 0f29e24..7988bd3 100644
---
a/ambari-server/src/main/resources/stack-hooks/before-START/scripts/params.py
+++
b/ambari-server/src/main/resources/stack-hooks/before-START/scripts/params.py
@@ -112,6 +112,7 @@ hbase_master_hosts =
default("/clusterHostInfo/hbase_master_hosts", [])
hs_host = default("/clusterHostInfo/historyserver_hosts", [])
jtnode_host = default("/clusterHostInfo/jtnode_hosts", [])
namenode_host = default("/clusterHostInfo/namenode_hosts", [])
+hdfs_client_hosts = default("/clusterHostInfo/hdfs_client_hosts", [])
zk_hosts = default("/clusterHostInfo/zookeeper_server_hosts", [])
ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_hosts", [])
cluster_name = config["clusterName"]
@@ -124,6 +125,8 @@ else:
ams_collector_hosts =
",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
has_namenode = not len(namenode_host) == 0
+has_hdfs_clients = len(hdfs_client_hosts) > 0
+has_hdfs = has_hdfs_clients or has_namenode
has_resourcemanager = not len(rm_host) == 0
has_slaves = not len(slave_hosts) == 0
has_oozie_server = not len(oozie_servers) == 0
diff --git
a/ambari-server/src/main/resources/stack-hooks/before-START/scripts/shared_initialization.py
b/ambari-server/src/main/resources/stack-hooks/before-START/scripts/shared_initialization.py
index a0e6e09..7efd326 100644
---
a/ambari-server/src/main/resources/stack-hooks/before-START/scripts/shared_initialization.py
+++
b/ambari-server/src/main/resources/stack-hooks/before-START/scripts/shared_initialization.py
@@ -67,16 +67,6 @@ def setup_hadoop():
else:
tc_owner = params.hdfs_user
- # if WebHDFS is not enabled we need this jar to create hadoop folders and
copy tarballs to HDFS.
- if params.sysprep_skip_copy_fast_jar_hdfs:
- print "Skipping copying of fast-hdfs-resource.jar as host is sys prepped"
- elif params.dfs_type == 'HCFS' or not
WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.dfs_type):
- # for source-code of jar goto contrib/fast-hdfs-resource
- File(format("{ambari_libs_dir}/fast-hdfs-resource.jar"),
- mode=0644,
- content=StaticFile("fast-hdfs-resource.jar")
- )
-
if os.path.exists(params.hadoop_conf_dir):
File(os.path.join(params.hadoop_conf_dir, 'commons-logging.properties'),
owner=tc_owner,
@@ -104,6 +94,19 @@ def setup_hadoop():
owner=params.hdfs_user,
)
+ create_microsoft_r_dir()
+
+ if params.has_hdfs or params.dfs_type == 'HCFS':
+ # if WebHDFS is not enabled we need this jar to create hadoop folders and
copy tarballs to HDFS.
+ if params.sysprep_skip_copy_fast_jar_hdfs:
+ print "Skipping copying of fast-hdfs-resource.jar as host is sys prepped"
+ else:
+ # for source-code of jar goto contrib/fast-hdfs-resource
+ File(format("{ambari_libs_dir}/fast-hdfs-resource.jar"),
+ mode=0644,
+ content=StaticFile("fast-hdfs-resource.jar")
+ )
+ if os.path.exists(params.hadoop_conf_dir):
if params.hadoop_metrics2_properties_content:
File(os.path.join(params.hadoop_conf_dir,
"hadoop-metrics2.properties"),
owner=params.hdfs_user,
@@ -118,9 +121,7 @@ def setup_hadoop():
)
if params.dfs_type == 'HCFS' and params.has_core_site and 'ECS_CLIENT' in
params.component_list:
- create_dirs()
-
- create_microsoft_r_dir()
+ create_dirs()
def setup_configs():
diff --git
a/ambari-server/src/test/python/stacks/stack-hooks/before-START/test_before_start.py
b/ambari-server/src/test/python/stacks/stack-hooks/before-START/test_before_start.py
index a7aea77..a7e8577 100644
---
a/ambari-server/src/test/python/stacks/stack-hooks/before-START/test_before_start.py
+++
b/ambari-server/src/test/python/stacks/stack-hooks/before-START/test_before_start.py
@@ -62,10 +62,6 @@ class TestHookBeforeStart(RMFTestCase):
create_parents = True,
cd_access = 'a',
)
- self.assertResourceCalled('File',
'/var/lib/ambari-agent/lib/fast-hdfs-resource.jar',
- content = StaticFile('fast-hdfs-resource.jar'),
- mode = 0644,
- )
self.assertResourceCalled('File',
'/etc/hadoop/conf/commons-logging.properties',
content =
Template('commons-logging.properties.j2'),
owner = 'hdfs',
@@ -81,6 +77,10 @@ class TestHookBeforeStart(RMFTestCase):
owner='hdfs',
content=InlineTemplate('log4jproperties\nline2log4jproperties\nline2')
)
+ self.assertResourceCalled('File',
'/var/lib/ambari-agent/lib/fast-hdfs-resource.jar',
+ content = StaticFile('fast-hdfs-resource.jar'),
+ mode = 0644,
+ )
self.assertResourceCalled('File',
'/etc/hadoop/conf/hadoop-metrics2.properties',
content =
InlineTemplate(self.getConfig()['configurations']['hadoop-metrics2.properties']['content']),
group='hadoop',
@@ -146,10 +146,6 @@ class TestHookBeforeStart(RMFTestCase):
create_parents = True,
cd_access = 'a',
)
- self.assertResourceCalled('File',
'/var/lib/ambari-agent/lib/fast-hdfs-resource.jar',
- content = StaticFile('fast-hdfs-resource.jar'),
- mode = 0644,
- )
self.assertResourceCalled('File',
'/etc/hadoop/conf/commons-logging.properties',
content =
Template('commons-logging.properties.j2'),
owner = 'root',
@@ -165,6 +161,10 @@ class TestHookBeforeStart(RMFTestCase):
owner='hdfs',
content=InlineTemplate('log4jproperties\nline2log4jproperties\nline2')
)
+ self.assertResourceCalled('File',
'/var/lib/ambari-agent/lib/fast-hdfs-resource.jar',
+ content = StaticFile('fast-hdfs-resource.jar'),
+ mode = 0644,
+ )
self.assertResourceCalled('File',
'/etc/hadoop/conf/hadoop-metrics2.properties',
content =
InlineTemplate(self.getConfig()['configurations']['hadoop-metrics2.properties']['content']),
group='hadoop',
@@ -235,10 +235,6 @@ class TestHookBeforeStart(RMFTestCase):
create_parents = True,
cd_access = 'a',
)
- self.assertResourceCalled('File',
'/var/lib/ambari-agent/lib/fast-hdfs-resource.jar',
- content = StaticFile('fast-hdfs-resource.jar'),
- mode = 0644,
- )
self.assertResourceCalled('File',
'/etc/hadoop/conf/commons-logging.properties',
content =
Template('commons-logging.properties.j2'),
owner = 'hdfs',
@@ -254,6 +250,10 @@ class TestHookBeforeStart(RMFTestCase):
owner='hdfs',
content=InlineTemplate('log4jproperties\nline2log4jproperties\nline2')
)
+ self.assertResourceCalled('File',
'/var/lib/ambari-agent/lib/fast-hdfs-resource.jar',
+ content = StaticFile('fast-hdfs-resource.jar'),
+ mode = 0644,
+ )
self.assertResourceCalled('File',
'/etc/hadoop/conf/hadoop-metrics2.properties',
content =
InlineTemplate(self.getConfig()['configurations']['hadoop-metrics2.properties']['content']),
group='hadoop',
@@ -326,10 +326,6 @@ class TestHookBeforeStart(RMFTestCase):
create_parents = True,
cd_access = 'a',
)
- self.assertResourceCalled('File',
'/var/lib/ambari-agent/lib/fast-hdfs-resource.jar',
- content = StaticFile('fast-hdfs-resource.jar'),
- mode = 0644,
- )
self.assertResourceCalled('File',
'/etc/hadoop/conf/commons-logging.properties',
content =
Template('commons-logging.properties.j2'),
owner = 'hdfs',
@@ -345,6 +341,10 @@ class TestHookBeforeStart(RMFTestCase):
owner='hdfs',
content=InlineTemplate('log4jproperties\nline2log4jproperties\nline2')
)
+ self.assertResourceCalled('File',
'/var/lib/ambari-agent/lib/fast-hdfs-resource.jar',
+ content = StaticFile('fast-hdfs-resource.jar'),
+ mode = 0644,
+ )
self.assertResourceCalled('File',
'/etc/hadoop/conf/hadoop-metrics2.properties',
content =
InlineTemplate(self.getConfig()['configurations']['hadoop-metrics2.properties']['content']),
group='hadoop',
diff --git
a/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Runner.java
b/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Runner.java
index 9cf0a73..25baf51 100644
---
a/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Runner.java
+++
b/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Runner.java
@@ -59,7 +59,7 @@ public class Runner {
Map<String, FileSystem> fileSystemNameToInstance = new HashMap<String,
FileSystem>();
Map<String, List<Resource>> fileSystemToResource = new HashMap<String,
List<Resource>>();
-
+ boolean failed = false;
try {
// 3 - Load data from JSON
resources = (Resource[]) gson.fromJson(new FileReader(jsonFilePath),
@@ -68,16 +68,30 @@ public class Runner {
Configuration conf = new Configuration();
FileSystem dfs = null;
+ String defaultFsSchema = FileSystem.getDefaultUri(conf).getScheme();
+
// Creating connections
for (Resource resource : resources) {
- String nameservice = resource.getNameservice();
+ String fsName = null;
+ URI targetURI = new URI(resource.getTarget());
+ String targetSchema = targetURI.getScheme();
+
+ if(targetSchema != null && !targetSchema.equals(defaultFsSchema)) {
+ String authority = targetURI.getAuthority();
+ if(authority == null) {
+ authority = "";
+ }
+ fsName = String.format("%s://%s/", targetSchema, authority);
+ } else if(resource.getNameservice() != null) {
+ fsName = resource.getNameservice();
+ }
- if(!fileSystemNameToInstance.containsKey(nameservice)) {
+ if(!fileSystemNameToInstance.containsKey(fsName)) {
URI fileSystemUrl;
- if(nameservice == null) {
+ if(fsName == null) {
fileSystemUrl = FileSystem.getDefaultUri(conf);
} else {
- fileSystemUrl = new URI(nameservice);
+ fileSystemUrl = new URI(fsName);
}
dfs = FileSystem.get(fileSystemUrl, conf);
@@ -86,13 +100,13 @@ public class Runner {
System.out.println("Initializing filesystem uri: " + fileSystemUrl);
dfs.initialize(fileSystemUrl, conf);
- fileSystemNameToInstance.put(nameservice, dfs);
+ fileSystemNameToInstance.put(fsName, dfs);
}
- if(!fileSystemToResource.containsKey(nameservice)) {
- fileSystemToResource.put(nameservice, new ArrayList<Resource>());
+ if(!fileSystemToResource.containsKey(fsName)) {
+ fileSystemToResource.put(fsName, new ArrayList<Resource>());
}
- fileSystemToResource.get(nameservice).add(resource);
+ fileSystemToResource.get(fsName).add(resource);
}
//for (Resource resource : resources) {
@@ -144,14 +158,18 @@ public class Runner {
catch(Exception e) {
System.out.println("Exception occurred, Reason: " + e.getMessage());
e.printStackTrace();
+ failed = true;
}
finally {
for(FileSystem dfs:fileSystemNameToInstance.values()) {
dfs.close();
}
}
-
- System.out.println("All resources created.");
+ if(!failed) {
+ System.out.println("All resources created.");
+ } else {
+ System.exit(1);
+ }
}
}
