This is an automated email from the ASF dual-hosted git repository.
aonishuk pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 7c36855 AMBARI-23700. Stack scripts should only try to reach
corresponding Namenode pair in the context rather than all NNs (aonishuk)
7c36855 is described below
commit 7c36855d73cf6c7ee46bdd2c4df883b6012695d0
Author: Andrew Onishuk <[email protected]>
AuthorDate: Thu Apr 26 11:52:36 2018 +0300
AMBARI-23700. Stack scripts should only try to reach corresponding Namenode
pair in the context rather than all NNs (aonishuk)
---
.../main/python/ambari_agent/CommandStatusDict.py | 1 -
.../libraries/functions/namenode_ha_utils.py | 5 +++--
.../libraries/providers/hdfs_resource.py | 6 +++++-
.../libraries/resources/hdfs_resource.py | 3 +++
.../2.1.0.2.0/package/scripts/hdfs_namenode.py | 17 ++++++++++-----
.../test/python/stacks/2.0.6/HDFS/test_namenode.py | 24 +++++++++++++++++-----
6 files changed, 42 insertions(+), 14 deletions(-)
diff --git a/ambari-agent/src/main/python/ambari_agent/CommandStatusDict.py
b/ambari-agent/src/main/python/ambari_agent/CommandStatusDict.py
index 49d925c..a30063e 100644
--- a/ambari-agent/src/main/python/ambari_agent/CommandStatusDict.py
+++ b/ambari-agent/src/main/python/ambari_agent/CommandStatusDict.py
@@ -107,7 +107,6 @@ class CommandStatusDict():
FAILED. Statuses for COMPLETE or FAILED commands are forgotten after
generation
"""
- logger.info("Reporting {0}".format(self.current_state))
self.generated_reports = []
from ActionQueue import ActionQueue
with self.lock: # Synchronized
diff --git
a/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py
b/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py
index c68d713..2ffd3cc 100644
---
a/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py
+++
b/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py
@@ -18,6 +18,7 @@ See the License for the specific language governing
permissions and
limitations under the License.
'''
from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.is_empty import is_empty
from resource_management.libraries.functions.jmx import get_value_from_jmx
from resource_management.core.base import Fail
from resource_management.core import shell
@@ -367,8 +368,8 @@ def get_name_service_by_hostname(hdfs_site, host_name):
"""
#there has to be a name service - we are in HA at least
name_services_string = hdfs_site['dfs.internal.nameservices']
- if not name_services_string:
- raise Fail('Not a HA setup')
+ if is_empty(name_services_string):
+ return None
name_services = name_services_string.split(',')
if len(name_services) == 1:
return name_services[0]
diff --git
a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
index 897e83c..acd469f 100644
---
a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
+++
b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
@@ -319,7 +319,11 @@ class HdfsResourceWebHDFS:
if main_resource.resource.security_enabled:
main_resource.kinit()
- nameservices =
namenode_ha_utils.get_nameservices(main_resource.resource.hdfs_site)
+ if main_resource.resource.nameservices is None:
+ nameservices =
namenode_ha_utils.get_nameservices(main_resource.resource.hdfs_site)
+ else:
+ nameservices = main_resource.resource.nameservices
+
if not nameservices:
self.action_delayed_for_nameservice(None, action_name, main_resource)
else:
diff --git
a/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py
b/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py
index 0a22147..6d44800 100644
---
a/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py
+++
b/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py
@@ -103,6 +103,9 @@ class HdfsResource(Resource):
# To support HCFS
dfs_type = ResourceArgument(default="")
+ # default None - means all nameservices
+ nameservices = ResourceArgument()
+
#action 'execute' immediately performs all pending actions in an efficient
manner
#action 'create_on_execute/delete_on_execute/download_on_execute' adds to
the list of pending actions
actions = Resource.actions + ["create_on_execute", "delete_on_execute",
"download_on_execute", "execute"]
diff --git
a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py
index d935792..213c8d9 100644
---
a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py
+++
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py
@@ -29,6 +29,7 @@ from resource_management.libraries.functions import
namenode_ha_utils
from resource_management.libraries.functions.decorator import retry
from resource_management.libraries.functions.format import format
from resource_management.libraries.functions.check_process_status import
check_process_status
+from resource_management.libraries.functions.namenode_ha_utils import
get_name_service_by_hostname
from resource_management.libraries.resources.execute_hadoop import
ExecuteHadoop
from resource_management.libraries.functions import Direction
from ambari_commons import OSCheck, OSConst
@@ -176,6 +177,8 @@ def namenode(action=None, hdfs_binary=None, do_format=True,
upgrade_type=None,
Execute(format("{kinit_path_local} -kt {hdfs_user_keytab}
{hdfs_principal_name}"),
user = params.hdfs_user)
+ name_service = get_name_service_by_hostname(params.hdfs_site,
params.hostname)
+
# ___Scenario___________|_Expected safemode state__|_Wait for safemode
OFF____|
# no-HA | ON -> OFF | Yes
|
# HA and active | ON -> OFF | Yes
|
@@ -197,7 +200,7 @@ def namenode(action=None, hdfs_binary=None, do_format=True,
upgrade_type=None,
if params.dfs_ha_enabled:
Logger.info("Waiting for the NameNode to broadcast whether it is Active
or Standby...")
- if is_this_namenode_active() is False:
+ if is_this_namenode_active(name_service) is False:
# we are the STANDBY NN
is_active_namenode = False
@@ -232,7 +235,7 @@ def namenode(action=None, hdfs_binary=None, do_format=True,
upgrade_type=None,
# in the case where safemode was ignored (like during an express upgrade),
then
# NN will be in SafeMode and cannot have directories created
if is_active_namenode and ensure_safemode_off:
- create_hdfs_directories()
+ create_hdfs_directories(name_service)
create_ranger_audit_hdfs_directories()
else:
Logger.info("Skipping creation of HDFS directories since this is either
not the Active NameNode or we did not wait for Safemode to finish.")
@@ -292,20 +295,24 @@ def create_name_dirs(directories):
)
-def create_hdfs_directories():
+def create_hdfs_directories(name_service):
import params
+ name_services = None if name_service is None else [name_service]
+
params.HdfsResource(params.hdfs_tmp_dir,
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
mode=0777,
+ nameservices=name_services,
)
params.HdfsResource(params.smoke_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.smoke_user,
mode=params.smoke_hdfs_user_mode,
+ nameservices=name_services,
)
params.HdfsResource(None,
action="execute",
@@ -588,7 +595,7 @@ def find_timeout():
@retry(sleep_time=5, backoff_factor=2, err_class=Fail,
timeout_func=find_timeout)
-def is_this_namenode_active():
+def is_this_namenode_active(name_service):
"""
Gets whether the current NameNode is Active. This function will wait until
the NameNode is
listed as being either Active or Standby before returning a value. This is
to ensure that
@@ -607,7 +614,7 @@ def is_this_namenode_active():
# 0 1
2
#
namenode_states = namenode_ha_utils.get_namenode_states(params.hdfs_site,
params.security_enabled,
- params.hdfs_user, times=5, sleep_time=5, backoff_factor=2)
+ params.hdfs_user, times=5, sleep_time=5, backoff_factor=2,
name_service=name_service)
# unwraps [('nn1', 'c6401.ambari.apache.org:50070')]
active_namenodes = [] if len(namenode_states[0]) < 1 else namenode_states[0]
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
index fb2aa2b..f46ac4b 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
@@ -119,6 +119,7 @@ class TestNamenode(RMFTestCase):
type = 'directory',
action = ['create_on_execute'],
hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
mode = 0777,
+ nameservices = None,
)
self.assertResourceCalled('HdfsResource', '/user/ambari-qa',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
@@ -136,6 +137,7 @@ class TestNamenode(RMFTestCase):
type = 'directory',
action = ['create_on_execute'],
hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
mode = 0770,
+ nameservices = None,
)
self.assertResourceCalled('HdfsResource', None,
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
@@ -236,6 +238,7 @@ class TestNamenode(RMFTestCase):
type = 'directory',
action = ['create_on_execute'],
hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
mode = 0777,
+ nameservices = None,
)
self.assertResourceCalled('HdfsResource', '/user/ambari-qa',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
@@ -253,6 +256,7 @@ class TestNamenode(RMFTestCase):
type = 'directory',
action = ['create_on_execute'],
hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
mode = 0770,
+ nameservices = None,
)
self.assertResourceCalled('HdfsResource', None,
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
@@ -365,7 +369,8 @@ class TestNamenode(RMFTestCase):
hadoop_conf_dir = '/etc/hadoop/conf',
type = 'directory',
action = ['create_on_execute'],
hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
hdfs_site=self.getConfig()['configurations']['hdfs-site'],
principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020',
- mode = 0777
+ mode = 0777,
+ nameservices = None
)
self.assertResourceCalled('HdfsResource', '/user/ambari-qa',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
@@ -379,7 +384,8 @@ class TestNamenode(RMFTestCase):
hadoop_conf_dir = '/etc/hadoop/conf',
type = 'directory',
action = ['create_on_execute'],
hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
hdfs_site=self.getConfig()['configurations']['hdfs-site'],
principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020',
- mode = 0770
+ mode = 0770,
+ nameservices = None,
)
self.assertResourceCalled('HdfsResource', None,
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
@@ -474,6 +480,7 @@ class TestNamenode(RMFTestCase):
type = 'directory',
action = ['create_on_execute'],
hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
mode = 0777,
+ nameservices = ['ns1'],
)
self.assertResourceCalled('HdfsResource', '/user/ambari-qa',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
@@ -491,6 +498,7 @@ class TestNamenode(RMFTestCase):
type = 'directory',
action = ['create_on_execute'],
hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
mode = 0770,
+ nameservices = ['ns1'],
)
self.assertResourceCalled('HdfsResource', None,
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
@@ -574,6 +582,7 @@ class TestNamenode(RMFTestCase):
type = 'directory',
action = ['create_on_execute'],
hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
mode = 0777,
+ nameservices = ['ns1'],
)
self.assertResourceCalled('HdfsResource', '/user/ambari-qa',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
@@ -591,6 +600,7 @@ class TestNamenode(RMFTestCase):
type = 'directory',
action = ['create_on_execute'],
hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
mode = 0770,
+ nameservices = ['ns1'],
)
self.assertResourceCalled('HdfsResource', None,
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
@@ -679,6 +689,7 @@ class TestNamenode(RMFTestCase):
type = 'directory',
action = ['create_on_execute'],
hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
mode = 0777,
+ nameservices = None,
)
self.assertResourceCalled('HdfsResource', '/user/ambari-qa',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
@@ -696,6 +707,7 @@ class TestNamenode(RMFTestCase):
type = 'directory',
action = ['create_on_execute'],
hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
mode = 0770,
+ nameservices = None,
)
self.assertResourceCalled('HdfsResource', None,
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
@@ -793,6 +805,7 @@ class TestNamenode(RMFTestCase):
type = 'directory',
action = ['create_on_execute'],
hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
mode = 0777,
+ nameservices = None,
)
self.assertResourceCalled('HdfsResource', '/user/ambari-qa',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
@@ -810,6 +823,7 @@ class TestNamenode(RMFTestCase):
type = 'directory',
action = ['create_on_execute'],
hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
mode = 0770,
+ nameservices = None,
)
self.assertResourceCalled('HdfsResource', None,
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
@@ -1723,7 +1737,7 @@ class TestNamenode(RMFTestCase):
unknown_namenodes = []
get_namenode_states_mock.return_value = active_namenodes,
standby_namenodes, unknown_namenodes
- self.assertTrue(is_this_namenode_active())
+ self.assertTrue(is_this_namenode_active(None))
# now test the harder tuple
active_namenodes = [('nn1', 'c6401.ambari.apache.org:50070'), ('nn2',
'c6402.ambari.apache.org:50070')]
@@ -1731,7 +1745,7 @@ class TestNamenode(RMFTestCase):
unknown_namenodes = []
get_namenode_states_mock.return_value = active_namenodes,
standby_namenodes, unknown_namenodes
- self.assertTrue(is_this_namenode_active())
+ self.assertTrue(is_this_namenode_active(None))
# and the negative for good measure
active_namenodes = []
@@ -1739,7 +1753,7 @@ class TestNamenode(RMFTestCase):
unknown_namenodes = []
get_namenode_states_mock.return_value = active_namenodes,
standby_namenodes, unknown_namenodes
- self.assertFalse(is_this_namenode_active())
+ self.assertFalse(is_this_namenode_active(None))
def test_reloadproxyusers(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR +
"/scripts/namenode.py",
--
To stop receiving notification emails like this one, please contact
[email protected].
