Repository: ambari
Updated Branches:
refs/heads/branch-2.4 56145c6c7 -> 14d79d002
AMBARI-16920: Follow up issue for Spark2 stack definition (Jeff Zhang via
jluniya)
Conflicts:
ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/14d79d00
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/14d79d00
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/14d79d00
Branch: refs/heads/branch-2.4
Commit: 14d79d002cde6d4d712ec41d97d771b62af6dcdb
Parents: 56145c6
Author: Jayush Luniya <[email protected]>
Authored: Fri Jun 17 07:48:09 2016 -0700
Committer: Jayush Luniya <[email protected]>
Committed: Fri Jun 17 07:56:21 2016 -0700
----------------------------------------------------------------------
.../libraries/functions/conf_select.py | 6 +
.../libraries/functions/copy_tarball.py | 8 +-
.../2.0.0/package/scripts/spark_service.py | 7 +-
.../stacks/HDP/2.5/role_command_order.json | 4 +-
.../hooks/after-INSTALL/test_after_install.py | 716 ++-----------------
5 files changed, 63 insertions(+), 678 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/14d79d00/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
----------------------------------------------------------------------
diff --git
a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
index 4eb0015..f98f1fc 100644
---
a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
+++
b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
@@ -145,6 +145,12 @@ _PACKAGE_DIRS = {
"current_dir": "{0}/current/spark-client/conf".format(STACK_ROOT_PATTERN)
}
],
+ "spark2": [
+ {
+ "conf_dir": "/etc/spark2/conf",
+ "current_dir":
"{0}/current/spark2-client/conf".format(STACK_ROOT_PATTERN)
+ }
+ ],
"sqoop": [
{
"conf_dir": "/etc/sqoop/conf",
http://git-wip-us.apache.org/repos/asf/ambari/blob/14d79d00/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
----------------------------------------------------------------------
diff --git
a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
index 286df8d..ac4e5b8 100644
---
a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
+++
b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
@@ -59,11 +59,13 @@ TARBALL_MAP = {
"mapreduce": ("{0}/{1}/hadoop/mapreduce.tar.gz".format(STACK_ROOT_PATTERN,
STACK_VERSION_PATTERN),
"/{0}/apps/{1}/mapreduce/mapreduce.tar.gz".format(STACK_NAME_PATTERN,
STACK_VERSION_PATTERN)),
"spark":
("{0}/{1}/spark/lib/spark-{2}-assembly.jar".format(STACK_ROOT_PATTERN,
STACK_VERSION_PATTERN, STACK_NAME_PATTERN),
-
"/{0}/apps/{1}/spark/spark-{0}-assembly.jar".format(STACK_NAME_PATTERN,
STACK_VERSION_PATTERN))
+
"/{0}/apps/{1}/spark/spark-{0}-assembly.jar".format(STACK_NAME_PATTERN,
STACK_VERSION_PATTERN)),
+ "spark2":
("/tmp/spark2/spark2-{0}-yarn-archive.tar.gz".format(STACK_NAME_PATTERN),
+
"/{0}/apps/{1}/spark2/spark2-{0}-yarn-archive.tar.gz".format(STACK_NAME_PATTERN,
STACK_VERSION_PATTERN))
}
-def _get_tarball_paths(name, use_upgrading_version_during_upgrade=True,
custom_source_file=None, custom_dest_file=None):
+def get_tarball_paths(name, use_upgrading_version_during_upgrade=True,
custom_source_file=None, custom_dest_file=None):
"""
For a given tarball name, get the source and destination paths to use.
:param name: Tarball name
@@ -203,7 +205,7 @@ def copy_to_hdfs(name, user_group, owner, file_mode=0444,
custom_source_file=Non
import params
Logger.info("Called copy_to_hdfs tarball: {0}".format(name))
- (success, source_file, dest_file) = _get_tarball_paths(name,
use_upgrading_version_during_upgrade,
+ (success, source_file, dest_file) = get_tarball_paths(name,
use_upgrading_version_during_upgrade,
custom_source_file,
custom_dest_file)
if not success:
http://git-wip-us.apache.org/repos/asf/ambari/blob/14d79d00/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_service.py
----------------------------------------------------------------------
diff --git
a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_service.py
b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_service.py
index 2eae3e7..a2ba4de 100755
---
a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_service.py
+++
b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_service.py
@@ -24,7 +24,7 @@ from contextlib import closing
from resource_management.libraries.script.script import Script
from resource_management.libraries.resources.hdfs_resource import HdfsResource
-from resource_management.libraries.functions.copy_tarball import copy_to_hdfs
+from resource_management.libraries.functions.copy_tarball import copy_to_hdfs,
get_tarball_paths
from resource_management.libraries.functions import format
from resource_management.core.resources.system import File, Execute
from resource_management.libraries.functions.version import
format_stack_version
@@ -42,7 +42,8 @@ def make_tarfile(output_filename, source_dir):
if not os.path.exists(parent_dir):
os.makedirs(parent_dir)
with closing(tarfile.open(output_filename, "w:gz")) as tar:
- tar.add(source_dir, arcname=os.path.basename(source_dir))
+ for file in os.listdir(source_dir):
+ tar.add(os.path.join(source_dir,file),arcname=file)
def spark_service(name, upgrade_type=None, action=None):
@@ -57,7 +58,7 @@ def spark_service(name, upgrade_type=None, action=None):
if effective_version and check_stack_feature(StackFeature.SPARK_16PLUS,
effective_version):
# create & copy spark2-hdp-yarn-archive.tar.gz to hdfs
source_dir=params.spark_home+"/jars"
- tmp_archive_file="/tmp/spark2/spark2-hdp-yarn-archive.tar.gz"
+ tmp_archive_file=get_tarball_paths("spark2")[1]
make_tarfile(tmp_archive_file, source_dir)
copy_to_hdfs("spark2", params.user_group, params.hdfs_user,
host_sys_prepped=params.host_sys_prepped)
# create spark history directory
http://git-wip-us.apache.org/repos/asf/ambari/blob/14d79d00/ambari-server/src/main/resources/stacks/HDP/2.5/role_command_order.json
----------------------------------------------------------------------
diff --git
a/ambari-server/src/main/resources/stacks/HDP/2.5/role_command_order.json
b/ambari-server/src/main/resources/stacks/HDP/2.5/role_command_order.json
index f6011b0..47ba050 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/role_command_order.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/role_command_order.json
@@ -15,7 +15,7 @@
},
"_comment" : "Dependencies that are used when GLUSTERFS is not present in
cluster",
"optional_no_glusterfs": {
- "SPARK2_JOBHISTORYSERVER-START" : ["NAMENODE-START"],
- "SPARK2_THRIFTSERVER-START" : ["HIVE_SERVER-START"]
+ "SPARK2_JOBHISTORYSERVER-START" : ["NAMENODE-START", "DATANODE-START"],
+ "SPARK2_THRIFTSERVER-START" : ["NAMENODE-START", "DATANODE-START",
"HIVE_SERVER-START"]
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/14d79d00/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
----------------------------------------------------------------------
diff --git
a/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
b/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
index 6c7fe18..06a366e 100644
---
a/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
+++
b/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
@@ -21,6 +21,7 @@ limitations under the License.
import json
from mock.mock import MagicMock, call, patch
from stacks.utils.RMFTestCase import *
+from resource_management.libraries.functions import conf_select
@patch("os.path.exists", new = MagicMock(return_value=True))
class TestHookAfterInstall(RMFTestCase):
@@ -80,215 +81,21 @@ class TestHookAfterInstall(RMFTestCase):
configuration_attributes =
self.getConfig()['configuration_attributes']['core-site'],
only_if="ls /usr/hdp/current/hadoop-client/conf")
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/ranger/kms/conf', '/etc/ranger/kms/conf.backup'),
- not_if = 'test -e /etc/ranger/kms/conf.backup',
- sudo = True,)
- self.assertResourceCalled('Directory', '/etc/ranger/kms/conf',
- action = ['delete'],)
- self.assertResourceCalled('Link', '/etc/ranger/kms/conf',
- to = '/usr/hdp/current/ranger-kms/conf',)
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/hive2/conf',
'/etc/hive2/conf.backup'),
- not_if = 'test -e /etc/hive2/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/hive2/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/hive2/conf',
- to = '/usr/hdp/current/hive-server2-hive2/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/zookeeper/conf', '/etc/zookeeper/conf.backup'),
- not_if = 'test -e /etc/zookeeper/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/zookeeper/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/zookeeper/conf',
- to = '/usr/hdp/current/zookeeper-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/pig/conf',
'/etc/pig/conf.backup'),
- not_if = 'test -e /etc/pig/conf.backup',
- sudo = True,)
- self.assertResourceCalled('Directory', '/etc/pig/conf',
- action = ['delete'],)
- self.assertResourceCalled('Link', '/etc/pig/conf',
- to = '/usr/hdp/current/pig-client/conf',)
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/tez/conf',
'/etc/tez/conf.backup'),
- not_if = 'test -e /etc/tez/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/tez/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/tez/conf',
- to = '/usr/hdp/current/tez-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/hive-webhcat/conf', '/etc/hive-webhcat/conf.backup'),
- not_if = 'test -e /etc/hive-webhcat/conf.backup',
- sudo = True,)
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/hive-hcatalog/conf', '/etc/hive-hcatalog/conf.backup'),
- not_if = 'test -e /etc/hive-hcatalog/conf.backup',
- sudo = True,)
-
- self.assertResourceCalled('Directory', '/etc/hive-webhcat/conf',
- action = ['delete'],)
- self.assertResourceCalled('Link', '/etc/hive-webhcat/conf',
- to = '/usr/hdp/current/hive-webhcat/etc/webhcat',)
-
- self.assertResourceCalled('Directory', '/etc/hive-hcatalog/conf',
- action = ['delete'],)
- self.assertResourceCalled('Link', '/etc/hive-hcatalog/conf',
- to = '/usr/hdp/current/hive-webhcat/etc/hcatalog',)
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/hbase/conf',
'/etc/hbase/conf.backup'),
- not_if = 'test -e /etc/hbase/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/hbase/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/hbase/conf',
- to = '/usr/hdp/current/hbase-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/ranger/tagsync/conf', '/etc/ranger/tagsync/conf.backup'),
- not_if = 'test -e
/etc/ranger/tagsync/conf.backup',
- sudo = True,)
- self.assertResourceCalled('Directory', '/etc/ranger/tagsync/conf',
- action = ['delete'],)
- self.assertResourceCalled('Link', '/etc/ranger/tagsync/conf',
- to = '/usr/hdp/current/ranger-tagsync/conf',)
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/ranger/usersync/conf', '/etc/ranger/usersync/conf.backup'),
- not_if = 'test -e
/etc/ranger/usersync/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/ranger/usersync/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/ranger/usersync/conf',
- to = '/usr/hdp/current/ranger-usersync/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/hadoop/conf', '/etc/hadoop/conf.backup'),
- not_if = 'test -e /etc/hadoop/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/hadoop/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/hadoop/conf',
- to = '/usr/hdp/current/hadoop-client/conf')
-
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/mahout/conf', '/etc/mahout/conf.backup'),
- not_if = 'test -e /etc/mahout/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/mahout/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/mahout/conf',
- to = '/usr/hdp/current/mahout-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/storm/conf',
'/etc/storm/conf.backup'),
- not_if = 'test -e /etc/storm/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/storm/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/storm/conf',
- to = '/usr/hdp/current/storm-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/ranger/admin/conf', '/etc/ranger/admin/conf.backup'),
- not_if = 'test -e /etc/ranger/admin/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/ranger/admin/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/ranger/admin/conf',
- to = '/usr/hdp/current/ranger-admin/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/flume/conf',
'/etc/flume/conf.backup'),
- not_if = 'test -e /etc/flume/conf.backup',
- sudo = True,)
- self.assertResourceCalled('Directory', '/etc/flume/conf',
- action = ['delete'],)
- self.assertResourceCalled('Link', '/etc/flume/conf',
- to = '/usr/hdp/current/flume-server/conf',)
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/sqoop/conf',
'/etc/sqoop/conf.backup'),
- not_if = 'test -e /etc/sqoop/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/sqoop/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/sqoop/conf',
- to = '/usr/hdp/current/sqoop-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/accumulo/conf', '/etc/accumulo/conf.backup'),
- not_if = 'test -e /etc/accumulo/conf.backup',
- sudo = True,)
- self.assertResourceCalled('Directory', '/etc/accumulo/conf',
- action = ['delete'],)
- self.assertResourceCalled('Link', '/etc/accumulo/conf',
- to = '/usr/hdp/current/accumulo-client/conf',)
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/phoenix/conf', '/etc/phoenix/conf.backup'),
- not_if = 'test -e /etc/phoenix/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/phoenix/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/phoenix/conf',
- to = '/usr/hdp/current/phoenix-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/storm-slider-client/conf', '/etc/storm-slider-client/conf.backup'),
- not_if = 'test -e /etc/storm-slider-client/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/storm-slider-client/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/storm-slider-client/conf',
- to = '/usr/hdp/current/storm-slider-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/slider/conf', '/etc/slider/conf.backup'),
- not_if = 'test -e /etc/slider/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/slider/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/slider/conf',
- to = '/usr/hdp/current/slider-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/oozie/conf',
'/etc/oozie/conf.backup'),
- not_if = 'test -e /etc/oozie/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/oozie/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/oozie/conf',
- to = '/usr/hdp/current/oozie-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/falcon/conf', '/etc/falcon/conf.backup'),
- not_if = 'test -e /etc/falcon/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/falcon/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/falcon/conf',
- to = '/usr/hdp/current/falcon-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/knox/conf',
'/etc/knox/conf.backup'),
- not_if = 'test -e /etc/knox/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/knox/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/knox/conf',
- to = '/usr/hdp/current/knox-server/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/kafka/conf',
'/etc/kafka/conf.backup'),
- not_if = 'test -e /etc/kafka/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/kafka/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/kafka/conf',
- to = '/usr/hdp/current/kafka-broker/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/hive/conf',
'/etc/hive/conf.backup'),
- not_if = 'test -e /etc/hive/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/hive/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/hive/conf',
- to = '/usr/hdp/current/hive-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/spark/conf',
'/etc/spark/conf.backup'),
- not_if = 'test -e /etc/spark/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/spark/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/spark/conf',
- to = '/usr/hdp/current/spark-client/conf')
+ package_dirs = conf_select.get_package_dirs();
+ for package, dir_defs in package_dirs.iteritems():
+ for dir_def in dir_defs:
+ conf_dir = dir_def['conf_dir']
+ conf_backup_dir = conf_dir + ".backup"
+ self.assertResourceCalled('Execute', ('cp', '-R', '-p', conf_dir,
conf_backup_dir),
+ not_if = 'test -e ' + conf_backup_dir,
+ sudo = True,)
+ for dir_def in dir_defs:
+ conf_dir = dir_def['conf_dir']
+ current_dir = dir_def['current_dir']
+ self.assertResourceCalled('Directory', conf_dir,
+ action = ['delete'],)
+ self.assertResourceCalled('Link', conf_dir,
+ to = current_dir,)
self.assertNoMoreResources()
@@ -335,258 +142,21 @@ class TestHookAfterInstall(RMFTestCase):
configuration_attributes =
self.getConfig()['configuration_attributes']['core-site'],
only_if="ls /usr/hdp/current/hadoop-client/conf")
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/ranger/kms/conf', '/etc/ranger/kms/conf.backup'),
- not_if = 'test -e /etc/ranger/kms/conf.backup',
- sudo = True,)
-
- self.assertResourceCalled('Directory', '/etc/ranger/kms/conf',
- action = ['delete'],)
-
- self.assertResourceCalled('Link', '/etc/ranger/kms/conf',
- to = '/usr/hdp/current/ranger-kms/conf',)
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/hive2/conf',
'/etc/hive2/conf.backup'),
- not_if = 'test -e /etc/hive2/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/hive2/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/hive2/conf',
- to = '/usr/hdp/current/hive-server2-hive2/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/zookeeper/conf', '/etc/zookeeper/conf.backup'),
- not_if = 'test -e /etc/zookeeper/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/zookeeper/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/zookeeper/conf',
- to = '/usr/hdp/current/zookeeper-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/pig/conf',
'/etc/pig/conf.backup'),
- not_if = 'test -e /etc/pig/conf.backup',
- sudo = True,)
-
- self.assertResourceCalled('Directory', '/etc/pig/conf',
- action=['delete'])
-
- self.assertResourceCalled("Link", "/etc/pig/conf",
- to="/usr/hdp/current/pig-client/conf")
-
- # pig fails, so no Directory/Link combo
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/tez/conf',
'/etc/tez/conf.backup'),
- not_if = 'test -e /etc/tez/conf.backup',
- sudo = True)
-
- self.assertResourceCalled('Directory', '/etc/tez/conf',
- action = ['delete'])
-
- self.assertResourceCalled('Link', '/etc/tez/conf',
- to = '/usr/hdp/current/tez-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/hive-webhcat/conf', '/etc/hive-webhcat/conf.backup'),
- not_if = 'test -e /etc/hive-webhcat/conf.backup',
- sudo = True,)
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/hive-hcatalog/conf', '/etc/hive-hcatalog/conf.backup'),
- not_if = 'test -e /etc/hive-hcatalog/conf.backup',
- sudo = True,)
-
- self.assertResourceCalled('Directory', '/etc/hive-webhcat/conf',
- action = ['delete'],)
-
- self.assertResourceCalled('Link', '/etc/hive-webhcat/conf',
- to = '/usr/hdp/current/hive-webhcat/etc/webhcat',)
-
- self.assertResourceCalled('Directory', '/etc/hive-hcatalog/conf',
- action = ['delete'],)
-
- self.assertResourceCalled('Link', '/etc/hive-hcatalog/conf',
- to = '/usr/hdp/current/hive-webhcat/etc/hcatalog',)
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/hbase/conf',
'/etc/hbase/conf.backup'),
- not_if = 'test -e /etc/hbase/conf.backup',
- sudo = True)
-
- self.assertResourceCalled('Directory', '/etc/hbase/conf',
- action = ['delete'])
-
- self.assertResourceCalled('Link', '/etc/hbase/conf',
- to = '/usr/hdp/current/hbase-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/ranger/tagsync/conf', '/etc/ranger/tagsync/conf.backup'),
- not_if = 'test -e
/etc/ranger/tagsync/conf.backup',
- sudo = True,)
-
- self.assertResourceCalled('Directory', '/etc/ranger/tagsync/conf',
- action = ['delete'],)
-
- self.assertResourceCalled('Link', '/etc/ranger/tagsync/conf',
- to = '/usr/hdp/current/ranger-tagsync/conf',)
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/ranger/usersync/conf', '/etc/ranger/usersync/conf.backup'),
- not_if = 'test -e
/etc/ranger/usersync/conf.backup',
- sudo = True)
-
- self.assertResourceCalled('Directory', '/etc/ranger/usersync/conf',
- action = ['delete'])
-
- self.assertResourceCalled('Link', '/etc/ranger/usersync/conf',
- to = '/usr/hdp/current/ranger-usersync/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/hadoop/conf', '/etc/hadoop/conf.backup'),
- not_if = 'test -e /etc/hadoop/conf.backup',
- sudo = True)
-
- self.assertResourceCalled('Directory', '/etc/hadoop/conf',
- action = ['delete'])
-
- self.assertResourceCalled('Link', '/etc/hadoop/conf',
- to = '/usr/hdp/current/hadoop-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/mahout/conf', '/etc/mahout/conf.backup'),
- not_if = 'test -e /etc/mahout/conf.backup',
- sudo = True)
-
- self.assertResourceCalled('Directory', '/etc/mahout/conf',
- action = ['delete'])
-
- self.assertResourceCalled('Link', '/etc/mahout/conf',
- to = '/usr/hdp/current/mahout-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/storm/conf',
'/etc/storm/conf.backup'),
- not_if = 'test -e /etc/storm/conf.backup',
- sudo = True)
-
- self.assertResourceCalled('Directory', '/etc/storm/conf',
- action = ['delete'])
-
- self.assertResourceCalled('Link', '/etc/storm/conf',
- to = '/usr/hdp/current/storm-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/ranger/admin/conf', '/etc/ranger/admin/conf.backup'),
- not_if = 'test -e /etc/ranger/admin/conf.backup',
- sudo = True)
-
- self.assertResourceCalled('Directory', '/etc/ranger/admin/conf',
- action = ['delete'])
-
- self.assertResourceCalled('Link', '/etc/ranger/admin/conf',
- to = '/usr/hdp/current/ranger-admin/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/flume/conf',
'/etc/flume/conf.backup'),
- not_if = 'test -e /etc/flume/conf.backup',
- sudo = True,)
-
- self.assertResourceCalled('Directory', '/etc/flume/conf',
- action = ['delete'],)
-
- self.assertResourceCalled('Link', '/etc/flume/conf',
- to = '/usr/hdp/current/flume-server/conf',)
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/sqoop/conf',
'/etc/sqoop/conf.backup'),
- not_if = 'test -e /etc/sqoop/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/sqoop/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/sqoop/conf',
- to = '/usr/hdp/current/sqoop-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/accumulo/conf', '/etc/accumulo/conf.backup'),
- not_if = 'test -e /etc/accumulo/conf.backup',
- sudo = True,)
-
- self.assertResourceCalled('Directory', '/etc/accumulo/conf',
- action = ['delete'],)
-
- self.assertResourceCalled('Link', '/etc/accumulo/conf',
- to = '/usr/hdp/current/accumulo-client/conf',)
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/phoenix/conf', '/etc/phoenix/conf.backup'),
- not_if = 'test -e /etc/phoenix/conf.backup',
- sudo = True)
-
- self.assertResourceCalled('Directory', '/etc/phoenix/conf',
- action = ['delete'])
-
- self.assertResourceCalled('Link', '/etc/phoenix/conf',
- to = '/usr/hdp/current/phoenix-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/storm-slider-client/conf', '/etc/storm-slider-client/conf.backup'),
- not_if = 'test -e /etc/storm-slider-client/conf.backup',
- sudo = True)
-
- self.assertResourceCalled('Directory', '/etc/storm-slider-client/conf',
- action = ['delete'])
-
- self.assertResourceCalled('Link', '/etc/storm-slider-client/conf',
- to = '/usr/hdp/current/storm-slider-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/slider/conf', '/etc/slider/conf.backup'),
- not_if = 'test -e /etc/slider/conf.backup',
- sudo = True)
-
- self.assertResourceCalled('Directory', '/etc/slider/conf',
- action = ['delete'])
-
- self.assertResourceCalled('Link', '/etc/slider/conf',
- to = '/usr/hdp/current/slider-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/oozie/conf',
'/etc/oozie/conf.backup'),
- not_if = 'test -e /etc/oozie/conf.backup',
- sudo = True)
-
- self.assertResourceCalled('Directory', '/etc/oozie/conf',
- action = ['delete'])
-
- self.assertResourceCalled('Link', '/etc/oozie/conf',
- to = '/usr/hdp/current/oozie-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/falcon/conf', '/etc/falcon/conf.backup'),
- not_if = 'test -e /etc/falcon/conf.backup',
- sudo = True)
-
- self.assertResourceCalled('Directory', '/etc/falcon/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/falcon/conf',
- to = '/usr/hdp/current/falcon-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/knox/conf',
'/etc/knox/conf.backup'),
- not_if = 'test -e /etc/knox/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/knox/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/knox/conf',
- to = '/usr/hdp/current/knox-server/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/kafka/conf',
'/etc/kafka/conf.backup'),
- not_if = 'test -e /etc/kafka/conf.backup',
- sudo = True)
-
- self.assertResourceCalled('Directory', '/etc/kafka/conf',
- action = ['delete'])
-
- self.assertResourceCalled('Link', '/etc/kafka/conf',
- to = '/usr/hdp/current/kafka-broker/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/hive/conf',
'/etc/hive/conf.backup'),
- not_if = 'test -e /etc/hive/conf.backup',
- sudo = True)
-
- self.assertResourceCalled('Directory', '/etc/hive/conf',
- action = ['delete'])
-
- self.assertResourceCalled('Link', '/etc/hive/conf',
- to = '/usr/hdp/current/hive-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/spark/conf',
'/etc/spark/conf.backup'),
- not_if = 'test -e /etc/spark/conf.backup',
- sudo = True)
-
- self.assertResourceCalled('Directory', '/etc/spark/conf',
- action = ['delete'])
-
- self.assertResourceCalled('Link', '/etc/spark/conf',
- to = '/usr/hdp/current/spark-client/conf')
+ package_dirs = conf_select.get_package_dirs();
+ for package, dir_defs in package_dirs.iteritems():
+ for dir_def in dir_defs:
+ conf_dir = dir_def['conf_dir']
+ conf_backup_dir = conf_dir + ".backup"
+ self.assertResourceCalled('Execute', ('cp', '-R', '-p', conf_dir,
conf_backup_dir),
+ not_if = 'test -e ' + conf_backup_dir,
+ sudo = True,)
+ for dir_def in dir_defs:
+ conf_dir = dir_def['conf_dir']
+ current_dir = dir_def['current_dir']
+ self.assertResourceCalled('Directory', conf_dir,
+ action = ['delete'],)
+ self.assertResourceCalled('Link', conf_dir,
+ to = current_dir,)
self.assertNoMoreResources()
@@ -666,214 +236,20 @@ class TestHookAfterInstall(RMFTestCase):
configuration_attributes =
self.getConfig()['configuration_attributes']['core-site'],
only_if="ls /usr/hdp/current/hadoop-client/conf")
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/ranger/kms/conf', '/etc/ranger/kms/conf.backup'),
- not_if = 'test -e /etc/ranger/kms/conf.backup',
- sudo = True,)
- self.assertResourceCalled('Directory', '/etc/ranger/kms/conf',
- action = ['delete'],)
- self.assertResourceCalled('Link', '/etc/ranger/kms/conf',
- to = '/usr/hdp/current/ranger-kms/conf',)
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/hive2/conf',
'/etc/hive2/conf.backup'),
- not_if = 'test -e /etc/hive2/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/hive2/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/hive2/conf',
- to = '/usr/hdp/current/hive-server2-hive2/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/zookeeper/conf', '/etc/zookeeper/conf.backup'),
- not_if = 'test -e /etc/zookeeper/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/zookeeper/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/zookeeper/conf',
- to = '/usr/hdp/current/zookeeper-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/pig/conf',
'/etc/pig/conf.backup'),
- not_if = 'test -e /etc/pig/conf.backup',
- sudo = True,)
- self.assertResourceCalled('Directory', '/etc/pig/conf',
- action = ['delete'],)
- self.assertResourceCalled('Link', '/etc/pig/conf',
- to = '/usr/hdp/current/pig-client/conf',)
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/tez/conf',
'/etc/tez/conf.backup'),
- not_if = 'test -e /etc/tez/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/tez/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/tez/conf',
- to = '/usr/hdp/current/tez-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/hive-webhcat/conf', '/etc/hive-webhcat/conf.backup'),
- not_if = 'test -e /etc/hive-webhcat/conf.backup',
- sudo = True,)
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/hive-hcatalog/conf', '/etc/hive-hcatalog/conf.backup'),
- not_if = 'test -e /etc/hive-hcatalog/conf.backup',
- sudo = True,)
-
- self.assertResourceCalled('Directory', '/etc/hive-webhcat/conf',
- action = ['delete'],)
- self.assertResourceCalled('Link', '/etc/hive-webhcat/conf',
- to = '/usr/hdp/current/hive-webhcat/etc/webhcat',)
-
- self.assertResourceCalled('Directory', '/etc/hive-hcatalog/conf',
- action = ['delete'],)
- self.assertResourceCalled('Link', '/etc/hive-hcatalog/conf',
- to = '/usr/hdp/current/hive-webhcat/etc/hcatalog',)
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/hbase/conf',
'/etc/hbase/conf.backup'),
- not_if = 'test -e /etc/hbase/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/hbase/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/hbase/conf',
- to = '/usr/hdp/current/hbase-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/ranger/tagsync/conf', '/etc/ranger/tagsync/conf.backup'),
- not_if = 'test -e
/etc/ranger/tagsync/conf.backup',
- sudo = True,)
- self.assertResourceCalled('Directory', '/etc/ranger/tagsync/conf',
- action = ['delete'],)
- self.assertResourceCalled('Link', '/etc/ranger/tagsync/conf',
- to = '/usr/hdp/current/ranger-tagsync/conf',)
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/ranger/usersync/conf', '/etc/ranger/usersync/conf.backup'),
- not_if = 'test -e
/etc/ranger/usersync/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/ranger/usersync/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/ranger/usersync/conf',
- to = '/usr/hdp/current/ranger-usersync/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/hadoop/conf', '/etc/hadoop/conf.backup'),
- not_if = 'test -e /etc/hadoop/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/hadoop/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/hadoop/conf',
- to = '/usr/hdp/current/hadoop-client/conf')
-
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/mahout/conf', '/etc/mahout/conf.backup'),
- not_if = 'test -e /etc/mahout/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/mahout/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/mahout/conf',
- to = '/usr/hdp/current/mahout-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/storm/conf',
'/etc/storm/conf.backup'),
- not_if = 'test -e /etc/storm/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/storm/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/storm/conf',
- to = '/usr/hdp/current/storm-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/ranger/admin/conf', '/etc/ranger/admin/conf.backup'),
- not_if = 'test -e /etc/ranger/admin/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/ranger/admin/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/ranger/admin/conf',
- to = '/usr/hdp/current/ranger-admin/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/flume/conf',
'/etc/flume/conf.backup'),
- not_if = 'test -e /etc/flume/conf.backup',
- sudo = True,)
- self.assertResourceCalled('Directory', '/etc/flume/conf',
- action = ['delete'],)
- self.assertResourceCalled('Link', '/etc/flume/conf',
- to = '/usr/hdp/current/flume-server/conf',)
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/sqoop/conf',
'/etc/sqoop/conf.backup'),
- not_if = 'test -e /etc/sqoop/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/sqoop/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/sqoop/conf',
- to = '/usr/hdp/current/sqoop-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/accumulo/conf', '/etc/accumulo/conf.backup'),
- not_if = 'test -e /etc/accumulo/conf.backup',
- sudo = True,)
- self.assertResourceCalled('Directory', '/etc/accumulo/conf',
- action = ['delete'],)
- self.assertResourceCalled('Link', '/etc/accumulo/conf',
- to = '/usr/hdp/current/accumulo-client/conf',)
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/phoenix/conf', '/etc/phoenix/conf.backup'),
- not_if = 'test -e /etc/phoenix/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/phoenix/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/phoenix/conf',
- to = '/usr/hdp/current/phoenix-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/storm-slider-client/conf', '/etc/storm-slider-client/conf.backup'),
- not_if = 'test -e /etc/storm-slider-client/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/storm-slider-client/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/storm-slider-client/conf',
- to = '/usr/hdp/current/storm-slider-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/slider/conf', '/etc/slider/conf.backup'),
- not_if = 'test -e /etc/slider/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/slider/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/slider/conf',
- to = '/usr/hdp/current/slider-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/oozie/conf',
'/etc/oozie/conf.backup'),
- not_if = 'test -e /etc/oozie/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/oozie/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/oozie/conf',
- to = '/usr/hdp/current/oozie-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p',
'/etc/falcon/conf', '/etc/falcon/conf.backup'),
- not_if = 'test -e /etc/falcon/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/falcon/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/falcon/conf',
- to = '/usr/hdp/current/falcon-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/knox/conf',
'/etc/knox/conf.backup'),
- not_if = 'test -e /etc/knox/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/knox/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/knox/conf',
- to = '/usr/hdp/current/knox-server/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/kafka/conf',
'/etc/kafka/conf.backup'),
- not_if = 'test -e /etc/kafka/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/kafka/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/kafka/conf',
- to = '/usr/hdp/current/kafka-broker/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/hive/conf',
'/etc/hive/conf.backup'),
- not_if = 'test -e /etc/hive/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/hive/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/hive/conf',
- to = '/usr/hdp/current/hive-client/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/spark/conf',
'/etc/spark/conf.backup'),
- not_if = 'test -e /etc/spark/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/spark/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/spark/conf',
- to = '/usr/hdp/current/spark-client/conf')
+ package_dirs = conf_select.get_package_dirs();
+ for package, dir_defs in package_dirs.iteritems():
+ for dir_def in dir_defs:
+ conf_dir = dir_def['conf_dir']
+ conf_backup_dir = conf_dir + ".backup"
+ self.assertResourceCalled('Execute', ('cp', '-R', '-p', conf_dir,
conf_backup_dir),
+ not_if = 'test -e ' + conf_backup_dir,
+ sudo = True,)
+ for dir_def in dir_defs:
+ conf_dir = dir_def['conf_dir']
+ current_dir = dir_def['current_dir']
+ self.assertResourceCalled('Directory', conf_dir,
+ action = ['delete'],)
+ self.assertResourceCalled('Link', conf_dir,
+ to = current_dir,)
self.assertNoMoreResources()
