This is an automated email from the ASF dual-hosted git repository. aonishuk pushed a commit to branch branch-3.0-perf in repository https://gitbox.apache.org/repos/asf/ambari.git
commit b6b29fd0fdcf00676f92bf8fe29d97471ce45daa Author: Andrew Onishuk <[email protected]> AuthorDate: Mon Jan 15 16:41:34 2018 +0200 AMBARI-22786. Fix server unit tests on branch-3.0-perf (aonishuk) --- .../libraries/functions/conf_select.py | 2 +- .../libraries/functions/stack_features.py | 2 +- .../custom_actions/scripts/update_repo.py | 11 +- .../src/test/python/TestResourceFilesKeeper.py | 2 +- ambari-server/src/test/python/TestStackFeature.py | 26 ++-- ambari-server/src/test/python/TestStackSelect.py | 16 +-- .../test/python/custom_actions/TestCheckHost.py | 16 +-- .../test/python/custom_actions/TestUpdateRepo.py | 41 +++++- .../configs/install_packages_config.json | 158 +++++++++++++++++++-- .../configs/install_packages_repository_file.json | 158 +++++++++++++++++++-- .../python/custom_actions/test_ru_execute_tasks.py | 4 +- .../test/python/custom_actions/test_ru_set_all.py | 8 +- .../python/stacks/2.2/common/test_conf_select.py | 4 +- .../ru_execute_tasks_namenode_prepare.json | 158 +++++++++++++++++++-- 14 files changed, 512 insertions(+), 94 deletions(-) diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py index 563e02f..2a8d230 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py @@ -322,7 +322,7 @@ def get_restricted_packages(): Logger.info("No services found, there are no restrictions for conf-select") return package_names - stack_name = default("/hostLevelParams/stack_name", None) + stack_name = default("/clusterLevelParams/stack_name", None) if stack_name is None: Logger.info("The stack name is not present in the command. Restricted names skipped.") return package_names diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py index da4f1c0..c9de8b8 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py @@ -98,7 +98,7 @@ def get_stack_feature_version(config): from resource_management.libraries.functions.default import default if "clusterLevelParams" not in config or "commandParams" not in config: - raise Fail("Unable to determine the correct version since hostLevelParams and commandParams were not present in the configuration dictionary") + raise Fail("Unable to determine the correct version since clusterLevelParams and commandParams were not present in the configuration dictionary") # should always be there stack_version = config['clusterLevelParams']['stack_version'] diff --git a/ambari-server/src/main/resources/custom_actions/scripts/update_repo.py b/ambari-server/src/main/resources/custom_actions/scripts/update_repo.py index 98ee91a..6f8a939 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/update_repo.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/update_repo.py @@ -37,13 +37,12 @@ class UpdateRepo(Script): try: - repo_info_json = config['hostLevelParams']['repoInfo'] - repo_info_dict = json.loads(repo_info_json) + repo_info = config['repositoryFile'] - for item in repo_info_dict["repositories"]: - base_url = item["base_url"] - repo_name = item["repo_name"] - repo_id = item["repo_id"] + for item in repo_info["repositories"]: + base_url = item["baseUrl"] + repo_name = item["repoName"] + repo_id = item["repoId"] distribution = item["distribution"] if "distribution" in item else None components = item["components"] if "components" in item else None diff --git a/ambari-server/src/test/python/TestResourceFilesKeeper.py b/ambari-server/src/test/python/TestResourceFilesKeeper.py index d5d1287..94b7bc2 100644 --- a/ambari-server/src/test/python/TestResourceFilesKeeper.py +++ b/ambari-server/src/test/python/TestResourceFilesKeeper.py @@ -59,7 +59,7 @@ class TestResourceFilesKeeper(TestCase): ResourceFilesKeeper.PACKAGE_DIR) if get_platform() != PLATFORM_WINDOWS: - DUMMY_UNCHANGEABLE_PACKAGE_HASH="11c9ed5f7987b41ce5d7adaedd6dd08c9cc9b418" + DUMMY_UNCHANGEABLE_PACKAGE_HASH="ad9dc916d5b113b4763b37b119d8237d72c9c5e5" else: DUMMY_UNCHANGEABLE_PACKAGE_HASH="2e438f4f9862420ed8930a56b8809b8aca359e87" DUMMY_HASH="dummy_hash" diff --git a/ambari-server/src/test/python/TestStackFeature.py b/ambari-server/src/test/python/TestStackFeature.py index ddd3f72..b61dbf7 100644 --- a/ambari-server/src/test/python/TestStackFeature.py +++ b/ambari-server/src/test/python/TestStackFeature.py @@ -34,22 +34,22 @@ class TestStackFeature(TestCase): """ EU Upgrade (HDP 2.5 to HDP 2.6) - STOP - hostLevelParams/stack_name = HDP - hostLevelParams/stack_version = 2.5 + clusterLevelParams/stack_name = HDP + clusterLevelParams/stack_version = 2.5 commandParams/version = 2.5.0.0-1237 - START - hostLevelParams/stack_name = HDP - hostLevelParams/stack_version = 2.6 + clusterLevelParams/stack_name = HDP + clusterLevelParams/stack_version = 2.6 commandParams/version = 2.6.0.0-334 EU Downgrade (HDP 2.6 to HDP 2.5) - STOP - hostLevelParams/stack_name = HDP - hostLevelParams/stack_version = 2.6 + clusterLevelParams/stack_name = HDP + clusterLevelParams/stack_version = 2.6 commandParams/version = 2.6.0.0-334 - START - hostLevelParams/stack_name = HDP - hostLevelParams/stack_version = 2.5 + clusterLevelParams/stack_name = HDP + clusterLevelParams/stack_version = 2.5 commandParams/version = 2.5.0.0-1237 """ @@ -151,7 +151,7 @@ class TestStackFeature(TestCase): return { "serviceName":"HDFS", "roleCommand": "ACTIONEXECUTE", - "hostLevelParams": { + "clusterLevelParams": { "stack_name": "HDP", "stack_version": "2.4", }, @@ -171,7 +171,7 @@ class TestStackFeature(TestCase): return { "serviceName":"HDFS", "roleCommand":"ACTIONEXECUTE", - "hostLevelParams": { + "clusterLevelParams": { "stack_name": "HDP", "stack_version": "2.4", }, @@ -208,7 +208,7 @@ class TestStackFeature(TestCase): return { "serviceName":"HDFS", "roleCommand":"ACTIONEXECUTE", - "hostLevelParams":{ + "clusterLevelParams":{ "stack_name":"HDP", "stack_version":"2.4" }, @@ -246,7 +246,7 @@ class TestStackFeature(TestCase): return { "serviceName":"HDFS", "roleCommand":"STOP", - "hostLevelParams":{ + "clusterLevelParams":{ "stack_name":"HDP", "stack_version":"2.5", }, @@ -283,7 +283,7 @@ class TestStackFeature(TestCase): return { "serviceName":"HDFS", "roleCommand":"CUSTOM_COMMAND", - "hostLevelParams":{ + "clusterLevelParams":{ "stack_name":"HDP", "stack_version":"2.5", "custom_command":"STOP" diff --git a/ambari-server/src/test/python/TestStackSelect.py b/ambari-server/src/test/python/TestStackSelect.py index 8cd8f2f..9e87a22 100644 --- a/ambari-server/src/test/python/TestStackSelect.py +++ b/ambari-server/src/test/python/TestStackSelect.py @@ -59,9 +59,9 @@ class TestStackSelect(TestCase): Script.config = dict() Script.config.update(command_json) - Script.config.update( { "configurations" : { "cluster-env" : {} }, "hostLevelParams": {} } ) + Script.config.update( { "configurations" : { "cluster-env" : {} }, "clusterLevelParams": {} } ) Script.config["configurations"]["cluster-env"]["stack_packages"] = self._get_stack_packages() - Script.config["hostLevelParams"] = { "stack_name" : "HDP" } + Script.config["clusterLevelParams"] = { "stack_name" : "HDP" } stack_select.select_packages(version) @@ -85,9 +85,9 @@ class TestStackSelect(TestCase): Script.config = dict() Script.config.update(command_json) - Script.config.update( { "configurations" : { "cluster-env" : {} }, "hostLevelParams": {} } ) + Script.config.update( { "configurations" : { "cluster-env" : {} }, "clusterLevelParams": {} } ) Script.config["configurations"]["cluster-env"]["stack_packages"] = self._get_stack_packages() - Script.config["hostLevelParams"] = { "stack_name" : "HDP" } + Script.config["clusterLevelParams"] = { "stack_name" : "HDP" } stack_select.select_packages(version) @@ -119,9 +119,9 @@ class TestStackSelect(TestCase): Script.config = dict() Script.config.update(command_json) - Script.config.update( { "configurations" : { "cluster-env" : {} }, "hostLevelParams": {} } ) + Script.config.update( { "configurations" : { "cluster-env" : {} }, "clusterLevelParams": {} } ) Script.config["configurations"]["cluster-env"]["stack_packages"] = self._get_stack_packages_with_legacy() - Script.config["hostLevelParams"] = { "stack_name" : "HDP" } + Script.config["clusterLevelParams"] = { "stack_name" : "HDP" } stack_select.select_packages(version) @@ -136,7 +136,7 @@ class TestStackSelect(TestCase): """ return { "roleCommand":"ACTIONEXECUTE", - "hostLevelParams": { + "clusterLevelParams": { "stack_name": "HDP", "stack_version": "2.4", }, @@ -174,7 +174,7 @@ class TestStackSelect(TestCase): "roleCommand":"ACTIONEXECUTE", "serviceName": "FOO_SERVICE", "role": "FOO_MASTER", - "hostLevelParams": { + "clusterLevelParams": { "stack_name": "HDP", "stack_version": "2.4", }, diff --git a/ambari-server/src/test/python/custom_actions/TestCheckHost.py b/ambari-server/src/test/python/custom_actions/TestCheckHost.py index 9303d5d..7a9e8b8 100644 --- a/ambari-server/src/test/python/custom_actions/TestCheckHost.py +++ b/ambari-server/src/test/python/custom_actions/TestCheckHost.py @@ -88,8 +88,8 @@ class TestCheckHost(TestCase): "user_name" : "test_user_name", "user_passwd" : "test_user_passwd", "jdk_name" : "test_jdk_name"}, - "hostLevelParams": { "agentCacheDir": "/nonexistent_tmp", - "custom_mysql_jdbc_name" : "mysql-connector-java.jar"} + "ambariLevelParams": {"custom_mysql_jdbc_name" : "mysql-connector-java.jar"}, + "agentLevelParams":{"agentCacheDir": "/nonexistent_tmp"} } get_tmp_dir_mock.return_value = "/tmp" download_file_mock.side_effect = Exception("test exception") @@ -116,8 +116,8 @@ class TestCheckHost(TestCase): "user_name" : "test_user_name", "user_passwd" : "test_user_passwd", "jdk_name" : "test_jdk_name"}, - "hostLevelParams": { "agentCacheDir": "/nonexistent_tmp", - "custom_oracle_jdbc_name" : "oracle-jdbc-driver.jar"}} + "agentLevelParams": { "agentCacheDir": "/nonexistent_tmp"}, + "ambariLevelParams": { "custom_oracle_jdbc_name" : "oracle-jdbc-driver.jar"}} format_mock.reset_mock() download_file_mock.reset_mock() p = MagicMock() @@ -146,8 +146,8 @@ class TestCheckHost(TestCase): "user_name" : "test_user_name", "user_passwd" : "test_user_passwd", "jdk_name" : "test_jdk_name"}, - "hostLevelParams": { "agentCacheDir": "/nonexistent_tmp", - "custom_postgres_jdbc_name" : "test-postgres-jdbc.jar"}} + "agentLevelParams": { "agentCacheDir": "/nonexistent_tmp"}, + "ambariLevelParams": { "custom_postgres_jdbc_name" : "oracle-jdbc-driver.jar"}} format_mock.reset_mock() download_file_mock.reset_mock() download_file_mock.side_effect = [p, p] @@ -184,8 +184,8 @@ class TestCheckHost(TestCase): "user_name" : "test_user_name", "user_passwd" : "test_user_passwd", "db_name" : "postgres"}, - "hostLevelParams": { "agentCacheDir": "/nonexistent_tmp", - "custom_postgres_jdbc_name" : "test-postgres-jdbc.jar"}} + "agentLevelParams": { "agentCacheDir": "/nonexistent_tmp"}, + "ambariLevelParams": { "custom_postgres_jdbc_name" : "test-postgres-jdbc.jar"}} isfile_mock.return_value = False diff --git a/ambari-server/src/test/python/custom_actions/TestUpdateRepo.py b/ambari-server/src/test/python/custom_actions/TestUpdateRepo.py index f3ee5ba..f984da9 100644 --- a/ambari-server/src/test/python/custom_actions/TestUpdateRepo.py +++ b/ambari-server/src/test/python/custom_actions/TestUpdateRepo.py @@ -54,16 +54,45 @@ class TestUpdateRepo(TestCase): "repo_ubuntu_template": "REPO_UBUNTU_TEMPLATE" } }, - "hostLevelParams": { - "repo_info": '{"repositories":[{"base_url":"TEST_BASE_URL","repo_name":"TEST_REPO_NAME","repo_id":"TEST_REPO_ID"}]}' - } + "repositoryFile": { + "resolved": True, + "repoVersion": "2.4.3.0-227", + "repositories": [ + { + "mirrorsList": None, + "ambariManaged": True, + "baseUrl": "http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.4.3.0/";, + "repoName": "HDP", + "components": None, + "osType": "redhat6", + "distribution": None, + "repoId": "HDP-2.4-repo-1" + }, + { + "mirrorsList": None, + "ambariManaged": True, + "baseUrl": "http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6";, + "repoName": "HDP-UTILS", + "components": None, + "osType": "redhat6", + "distribution": None, + "repoId": "HDP-UTILS-1.1.0.20-repo-1" + } + ], + "feature": { + "m_isScoped": False, + "m_isPreInstalled": False + }, + "stackName": "HDP", + "repoVersionId": 1 + }, } with Environment('/') as env: updateRepo.actionexecute(None) self.assertTrue(file_mock.called) - self.assertEquals(file_mock.call_args[0][0], "/etc/yum.repos.d/TEST_REPO_NAME.repo") + self.assertEquals(file_mock.call_args[0][0], "/etc/yum.repos.d/HDP-UTILS.repo") self.assertEquals(structured_out_mock.call_args[0][0], {'repo_update': {'message': 'Repository files successfully updated!', 'exit_code': 0}}) ###### invalid repo info @@ -75,9 +104,7 @@ class TestUpdateRepo(TestCase): "repo_ubuntu_template": "REPO_UBUNTU_TEMPLATE" } }, - "hostLevelParams": { - "repo_info": '{}' - } + "repositoryFile": {} } try: with Environment('/') as env: diff --git a/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json b/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json index 36a7896..683eb8b 100644 --- a/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json +++ b/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json @@ -4,25 +4,155 @@ "clusterName": "cc", "hostname": "0b3.vm", "passiveInfo": [], - "hostLevelParams": { - "agent_stack_retry_count": "5", - "agent_stack_retry_on_unavailability": "false", - "jdk_location": "http://0b3.vm:8080/resources/";, - "ambari_db_rca_password": "mapred", - "java_home": "/usr/jdk64/jdk1.7.0_67", - "java_version": "8", - "ambari_db_rca_url": "jdbc:postgresql://0b3.vm/ambarirca", - "jce_name": "UnlimitedJCEPolicyJDK7.zip", - "oracle_jdbc_url": "http://0b3.vm:8080/resources//ojdbc6.jar";, + "clusterLevelParams": { "stack_version": "2.1", + "not_managed_hdfs_path_list": "[\"/apps/hive/warehouse\",\"/apps/falcon\",\"/mr-history/done\",\"/app-logs\",\"/tmp\"]", + "hooks_folder": "stack-hooks", "stack_name": "HDP", + "group_list": "[\"hdfs\",\"hadoop\",\"users\"]", + "user_groups": "{\"hive\":[\"hadoop\"], \"oozie\":[\"hadoop\",\"users\"], \"nobody\":[\"hadoop\",\"nobody\"], \"ambari-qa\":[\"hadoop\",\"users\"], \"flume\":[\"hadoop\"], \"hdfs\":[\"hadoop\"], \"storm\":[\"hadoop\"], \"mapred\":[\"hadoop\"], \"hbase\":[\"hadoop\"], \"tez\":[\"hadoop\",\"users\"], \"zookeeper\":[\"hadoop\"], \"falcon\":[\"hadoop\",\"users\"], \"sqoop\":[\"hadoop\"], \"yarn\":[\"hadoop\"], \"hcat\":[\"hadoop\"]}", + "cluster_name": "c1", + "user_list": "[\"ambari-qa\",\"hdfs\"]" + }, + "ambariLevelParams": { + "jdk_location": "http://c6401.ambari.apache.org:8080/resources/";, + "agent_stack_retry_count": "5", + "db_driver_filename": "mysql-connector-java.jar", + "agent_stack_retry_on_unavailability": "false", + "ambari_db_rca_url": "jdbc:postgresql://c6401.ambari.apache.org/ambarirca", + "jce_name": "jce_policy-7.zip", + "java_version": "7", + "ambari_db_rca_password": "mapred", + "host_sys_prepped": "false", "db_name": "ambari", + "oracle_jdbc_url": "http://c6401.ambari.apache.org/resources//ojdbc6.jar";, "ambari_db_rca_driver": "org.postgresql.Driver", - "jdk_name": "jdk-7u67-linux-x64.tar.gz", "ambari_db_rca_username": "mapred", - "db_driver_filename": "mysql-connector-java.jar", - "mysql_jdbc_url": "http://0b3.vm:8080/resources//mysql-connector-java.jar"; - }, + "jdk_name": "jdk-7u45-linux-x64.tar.gz", + "java_home": "/usr/jdk64/jdk1.7.0_45", + "mysql_jdbc_url": "http://c6401.ambari.apache.org/resources//mysql-connector-java.jar"; + }, + "repositoryFile": { + "resolved": true, + "repoVersion": "2.6.4.0-60", + "repositories": [ + { + "mirrorsList": null, + "ambariManaged": true, + "baseUrl": "http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.6.4.0-60";, + "repoName": "HDP", + "components": null, + "osType": "redhat6", + "distribution": null, + "repoId": "HDP-2.6-repo-1" + }, + { + "mirrorsList": null, + "ambariManaged": true, + "baseUrl": "http://s3.amazonaws.com/dev.hortonworks.com/HDP-GPL/centos6/2.x/BUILDS/2.6.4.0-60";, + "repoName": "HDP-GPL", + "components": null, + "osType": "redhat6", + "distribution": null, + "repoId": "HDP-2.6-GPL-repo-1" + }, + { + "mirrorsList": null, + "ambariManaged": true, + "baseUrl": "http://s3.amazonaws.com/dev.hortonworks.com/HDP-UTILS-1.1.0.22/repos/centos6";, + "repoName": "HDP-UTILS", + "components": null, + "osType": "redhat6", + "distribution": null, + "repoId": "HDP-UTILS-1.1.0.22-repo-1" + } + ], + "feature": { + "m_isScoped": true, + "m_isPreInstalled": false + }, + "stackName": "HDP", + "repoVersionId": 1 + }, + "agentLevelParams": { + "public_hostname": "c6401.ambari.apache.org", + "hostname": "c6401.ambari.apache.org", + "agentConfigParams": { + "agent": { + "parallel_execution": 0, + "use_system_proxy_settings": true + } + }, + "agentCacheDir": "/var/lib/ambari-agent/cache" + }, + "hostLevelParams": { + "recoveryConfig": { + "retryGap": "5", + "windowInMinutes": "60", + "maxLifetimeCount": "1024", + "components": "", + "maxCount": "6", + "type": "AUTO_START" + }, + "hostRepositories": { + "componentRepos": { + "NAMENODE": 1, + "SECONDARY_NAMENODE": 1, + "DATANODE": 1, + "HDFS_CLIENT": 1 + }, + "commandRepos": { + "1": { + "resolved": true, + "repoVersion": "2.1.4.0-60", + "repositories": [ + { + "mirrorsList": null, + "ambariManaged": true, + "baseUrl": "http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.6.4.0-60";, + "repoName": "HDP", + "components": null, + "osType": "redhat6", + "distribution": null, + "repoId": "HDP-2.6-repo-1" + }, + { + "mirrorsList": null, + "ambariManaged": true, + "baseUrl": "http://s3.amazonaws.com/dev.hortonworks.com/HDP-GPL/centos6/2.x/BUILDS/2.6.4.0-60";, + "repoName": "HDP-GPL", + "components": null, + "osType": "redhat6", + "distribution": null, + "repoId": "HDP-2.6-GPL-repo-1" + }, + { + "mirrorsList": null, + "ambariManaged": true, + "baseUrl": "http://s3.amazonaws.com/dev.hortonworks.com/HDP-UTILS-1.1.0.22/repos/centos6";, + "repoName": "HDP-UTILS", + "components": null, + "osType": "redhat6", + "distribution": null, + "repoId": "HDP-UTILS-1.1.0.22-repo-1" + } + ], + "feature": { + "m_isScoped": true, + "m_isPreInstalled": false + }, + "stackName": "HDP", + "repoVersionId": 1 + } + } + } + }, + "serviceLevelParams": { + "credentialStoreEnabled": false, + "status_commands_timeout": 300, + "version": "2.7.3", + "service_package_folder": "common-services/HDFS/2.1.0.2.0/package" + }, "commandType": "EXECUTION_COMMAND", "roleParams": { "stack_id": "HDP-2.2", diff --git a/ambari-server/src/test/python/custom_actions/configs/install_packages_repository_file.json b/ambari-server/src/test/python/custom_actions/configs/install_packages_repository_file.json index bdf062c..a30119e 100644 --- a/ambari-server/src/test/python/custom_actions/configs/install_packages_repository_file.json +++ b/ambari-server/src/test/python/custom_actions/configs/install_packages_repository_file.json @@ -4,25 +4,155 @@ "clusterName": "cc", "hostname": "0b3.vm", "passiveInfo": [], - "hostLevelParams": { - "agent_stack_retry_count": "5", - "agent_stack_retry_on_unavailability": "false", - "jdk_location": "http://0b3.vm:8080/resources/";, - "ambari_db_rca_password": "mapred", - "java_home": "/usr/jdk64/jdk1.7.0_67", - "java_version": "8", - "ambari_db_rca_url": "jdbc:postgresql://0b3.vm/ambarirca", - "jce_name": "UnlimitedJCEPolicyJDK7.zip", - "oracle_jdbc_url": "http://0b3.vm:8080/resources//ojdbc6.jar";, + "clusterLevelParams": { "stack_version": "2.1", + "not_managed_hdfs_path_list": "[\"/apps/hive/warehouse\",\"/apps/falcon\",\"/mr-history/done\",\"/app-logs\",\"/tmp\"]", + "hooks_folder": "stack-hooks", "stack_name": "HDP", + "group_list": "[\"hdfs\",\"hadoop\",\"users\"]", + "user_groups": "{\"hive\":[\"hadoop\"], \"oozie\":[\"hadoop\",\"users\"], \"nobody\":[\"hadoop\",\"nobody\"], \"ambari-qa\":[\"hadoop\",\"users\"], \"flume\":[\"hadoop\"], \"hdfs\":[\"hadoop\"], \"storm\":[\"hadoop\"], \"mapred\":[\"hadoop\"], \"hbase\":[\"hadoop\"], \"tez\":[\"hadoop\",\"users\"], \"zookeeper\":[\"hadoop\"], \"falcon\":[\"hadoop\",\"users\"], \"sqoop\":[\"hadoop\"], \"yarn\":[\"hadoop\"], \"hcat\":[\"hadoop\"]}", + "cluster_name": "c1", + "user_list": "[\"ambari-qa\",\"hdfs\"]" + }, + "ambariLevelParams": { + "jdk_location": "http://c6401.ambari.apache.org:8080/resources/";, + "agent_stack_retry_count": "5", + "db_driver_filename": "mysql-connector-java.jar", + "agent_stack_retry_on_unavailability": "false", + "ambari_db_rca_url": "jdbc:postgresql://c6401.ambari.apache.org/ambarirca", + "jce_name": "jce_policy-7.zip", + "java_version": "7", + "ambari_db_rca_password": "mapred", + "host_sys_prepped": "false", "db_name": "ambari", + "oracle_jdbc_url": "http://c6401.ambari.apache.org/resources//ojdbc6.jar";, "ambari_db_rca_driver": "org.postgresql.Driver", - "jdk_name": "jdk-7u67-linux-x64.tar.gz", "ambari_db_rca_username": "mapred", - "db_driver_filename": "mysql-connector-java.jar", - "mysql_jdbc_url": "http://0b3.vm:8080/resources//mysql-connector-java.jar"; - }, + "jdk_name": "jdk-7u45-linux-x64.tar.gz", + "java_home": "/usr/jdk64/jdk1.7.0_45", + "mysql_jdbc_url": "http://c6401.ambari.apache.org/resources//mysql-connector-java.jar"; + }, + "repositoryFile": { + "resolved": true, + "repoVersion": "2.6.4.0-60", + "repositories": [ + { + "mirrorsList": null, + "ambariManaged": true, + "baseUrl": "http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.6.4.0-60";, + "repoName": "HDP", + "components": null, + "osType": "redhat6", + "distribution": null, + "repoId": "HDP-2.6-repo-1" + }, + { + "mirrorsList": null, + "ambariManaged": true, + "baseUrl": "http://s3.amazonaws.com/dev.hortonworks.com/HDP-GPL/centos6/2.x/BUILDS/2.6.4.0-60";, + "repoName": "HDP-GPL", + "components": null, + "osType": "redhat6", + "distribution": null, + "repoId": "HDP-2.6-GPL-repo-1" + }, + { + "mirrorsList": null, + "ambariManaged": true, + "baseUrl": "http://s3.amazonaws.com/dev.hortonworks.com/HDP-UTILS-1.1.0.22/repos/centos6";, + "repoName": "HDP-UTILS", + "components": null, + "osType": "redhat6", + "distribution": null, + "repoId": "HDP-UTILS-1.1.0.22-repo-1" + } + ], + "feature": { + "m_isScoped": true, + "m_isPreInstalled": false + }, + "stackName": "HDP", + "repoVersionId": 1 + }, + "agentLevelParams": { + "public_hostname": "c6401.ambari.apache.org", + "hostname": "c6401.ambari.apache.org", + "agentConfigParams": { + "agent": { + "parallel_execution": 0, + "use_system_proxy_settings": true + } + }, + "agentCacheDir": "/var/lib/ambari-agent/cache" + }, + "hostLevelParams": { + "recoveryConfig": { + "retryGap": "5", + "windowInMinutes": "60", + "maxLifetimeCount": "1024", + "components": "", + "maxCount": "6", + "type": "AUTO_START" + }, + "hostRepositories": { + "componentRepos": { + "NAMENODE": 1, + "SECONDARY_NAMENODE": 1, + "DATANODE": 1, + "HDFS_CLIENT": 1 + }, + "commandRepos": { + "1": { + "resolved": true, + "repoVersion": "2.1.4.0-60", + "repositories": [ + { + "mirrorsList": null, + "ambariManaged": true, + "baseUrl": "http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.6.4.0-60";, + "repoName": "HDP", + "components": null, + "osType": "redhat6", + "distribution": null, + "repoId": "HDP-2.6-repo-1" + }, + { + "mirrorsList": null, + "ambariManaged": true, + "baseUrl": "http://s3.amazonaws.com/dev.hortonworks.com/HDP-GPL/centos6/2.x/BUILDS/2.6.4.0-60";, + "repoName": "HDP-GPL", + "components": null, + "osType": "redhat6", + "distribution": null, + "repoId": "HDP-2.6-GPL-repo-1" + }, + { + "mirrorsList": null, + "ambariManaged": true, + "baseUrl": "http://s3.amazonaws.com/dev.hortonworks.com/HDP-UTILS-1.1.0.22/repos/centos6";, + "repoName": "HDP-UTILS", + "components": null, + "osType": "redhat6", + "distribution": null, + "repoId": "HDP-UTILS-1.1.0.22-repo-1" + } + ], + "feature": { + "m_isScoped": true, + "m_isPreInstalled": false + }, + "stackName": "HDP", + "repoVersionId": 1 + } + } + } + }, + "serviceLevelParams": { + "credentialStoreEnabled": false, + "status_commands_timeout": 300, + "version": "2.7.3", + "service_package_folder": "common-services/HDFS/2.1.0.2.0/package" + }, "commandType": "EXECUTION_COMMAND", "repositoryFile": { "stackName": "HDP", diff --git a/ambari-server/src/test/python/custom_actions/test_ru_execute_tasks.py b/ambari-server/src/test/python/custom_actions/test_ru_execute_tasks.py index 1d822eb..f77a95b 100644 --- a/ambari-server/src/test/python/custom_actions/test_ru_execute_tasks.py +++ b/ambari-server/src/test/python/custom_actions/test_ru_execute_tasks.py @@ -117,7 +117,7 @@ class TestRUExecuteTasks(RMFTestCase): # Ensure that the json file was actually read. stack_name = default("/clusterLevelParams/stack_name", None) - stack_version = default("/hostLevelParams/stack_version", None) + stack_version = default("/clusterLevelParams/stack_version", None) service_package_folder = default('/roleParams/service_package_folder', None) self.assertEqual(stack_name, "HDP") @@ -165,7 +165,7 @@ class TestRUExecuteTasks(RMFTestCase): # Ensure that the json file was actually read. stack_name = default("/clusterLevelParams/stack_name", None) - stack_version = default("/hostLevelParams/stack_version", None) + stack_version = default("/clusterLevelParams/stack_version", None) service_package_folder = default('/commandParams/service_package_folder', None) self.assertEqual(stack_name, "HDP") diff --git a/ambari-server/src/test/python/custom_actions/test_ru_set_all.py b/ambari-server/src/test/python/custom_actions/test_ru_set_all.py index 400af49..6421397 100644 --- a/ambari-server/src/test/python/custom_actions/test_ru_set_all.py +++ b/ambari-server/src/test/python/custom_actions/test_ru_set_all.py @@ -90,7 +90,7 @@ class TestRUSetAll(RMFTestCase): # Ensure that the json file was actually read. stack_name = default("/clusterLevelParams/stack_name", None) - stack_version = default("/hostLevelParams/stack_version", None) + stack_version = default("/clusterLevelParams/stack_version", None) service_package_folder = default('/roleParams/service_package_folder', None) self.assertEqual(stack_name, "HDP") @@ -115,7 +115,7 @@ class TestRUSetAll(RMFTestCase): with open(json_file_path, "r") as json_file: json_payload = json.load(json_file) - json_payload['hostLevelParams']['stack_name'] = "HDP" + json_payload['clusterLevelParams']['stack_name'] = "HDP" json_payload['clusterLevelParams']['stack_version'] = "2.3" json_payload['commandParams']['version'] = "2.3.0.0-1234" json_payload["configurations"]["cluster-env"]["stack_tools"] = self.get_stack_tools() @@ -131,7 +131,7 @@ class TestRUSetAll(RMFTestCase): # Ensure that the json file was actually read. stack_name = default("/clusterLevelParams/stack_name", None) - stack_version = default("/hostLevelParams/stack_version", None) + stack_version = default("/clusterLevelParams/stack_version", None) service_package_folder = default('/roleParams/service_package_folder', None) self.assertEqual(stack_name, "HDP") @@ -182,7 +182,7 @@ class TestRUSetAll(RMFTestCase): # Ensure that the json file was actually read. stack_name = default("/clusterLevelParams/stack_name", None) - stack_version = default("/hostLevelParams/stack_version", None) + stack_version = default("/clusterLevelParams/stack_version", None) service_package_folder = default('/roleParams/service_package_folder', None) self.assertEqual(stack_name, "HDP") diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_conf_select.py b/ambari-server/src/test/python/stacks/2.2/common/test_conf_select.py index a199d00..a893d75 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_conf_select.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_conf_select.py @@ -35,9 +35,9 @@ class TestConfSelect(RMFTestCase): self.env.__enter__() Script.config = dict() - Script.config.update( { "configurations" : { "cluster-env" : {} }, "hostLevelParams": {} } ) + Script.config.update( { "configurations" : { "cluster-env" : {} }, "clusterLevelParams": {} } ) Script.config["configurations"]["cluster-env"]["stack_packages"] = RMFTestCase.get_stack_packages() - Script.config["hostLevelParams"] = { "stack_name" : "HDP" } + Script.config["clusterLevelParams"] = { "stack_name" : "HDP" } def tearDown(self): diff --git a/ambari-server/src/test/resources/custom_actions/ru_execute_tasks_namenode_prepare.json b/ambari-server/src/test/resources/custom_actions/ru_execute_tasks_namenode_prepare.json index 27c09b5..ef85a34 100644 --- a/ambari-server/src/test/resources/custom_actions/ru_execute_tasks_namenode_prepare.json +++ b/ambari-server/src/test/resources/custom_actions/ru_execute_tasks_namenode_prepare.json @@ -5,23 +5,155 @@ "clusterName": "c1", "componentName": "", "hostname": "u1202.ambari.apache.org", - "hostLevelParams": { - "jdk_location": "http://u1201.ambari.apache.org:8080/resources/";, - "ambari_db_rca_password": "mapred", - "java_home": "/usr/jdk64/jdk1.7.0_67", - "ambari_db_rca_url": "jdbc:postgresql://u1201.ambari.apache.org/ambarirca", - "jce_name": "UnlimitedJCEPolicyJDK7.zip", - "oracle_jdbc_url": "http://u1201.ambari.apache.org:8080/resources//ojdbc6.jar";, + "clusterLevelParams": { "stack_version": "2.2", + "not_managed_hdfs_path_list": "[\"/apps/hive/warehouse\",\"/apps/falcon\",\"/mr-history/done\",\"/app-logs\",\"/tmp\"]", + "hooks_folder": "stack-hooks", "stack_name": "HDP", + "group_list": "[\"hdfs\",\"hadoop\",\"users\"]", + "user_groups": "{\"hive\":[\"hadoop\"], \"oozie\":[\"hadoop\",\"users\"], \"nobody\":[\"hadoop\",\"nobody\"], \"ambari-qa\":[\"hadoop\",\"users\"], \"flume\":[\"hadoop\"], \"hdfs\":[\"hadoop\"], \"storm\":[\"hadoop\"], \"mapred\":[\"hadoop\"], \"hbase\":[\"hadoop\"], \"tez\":[\"hadoop\",\"users\"], \"zookeeper\":[\"hadoop\"], \"falcon\":[\"hadoop\",\"users\"], \"sqoop\":[\"hadoop\"], \"yarn\":[\"hadoop\"], \"hcat\":[\"hadoop\"]}", + "cluster_name": "c1", + "user_list": "[\"ambari-qa\",\"hdfs\"]" + }, + "ambariLevelParams": { + "jdk_location": "http://c6401.ambari.apache.org:8080/resources/";, + "agent_stack_retry_count": "5", + "db_driver_filename": "mysql-connector-java.jar", + "agent_stack_retry_on_unavailability": "false", + "ambari_db_rca_url": "jdbc:postgresql://c6401.ambari.apache.org/ambarirca", + "jce_name": "jce_policy-7.zip", + "java_version": "7", + "ambari_db_rca_password": "mapred", + "host_sys_prepped": "false", "db_name": "ambari", + "oracle_jdbc_url": "http://c6401.ambari.apache.org/resources//ojdbc6.jar";, "ambari_db_rca_driver": "org.postgresql.Driver", - "jdk_name": "jdk-7u67-linux-x64.tar.gz", "ambari_db_rca_username": "mapred", - "db_driver_filename": "mysql-connector-java.jar", - "agentCacheDir": "/var/lib/ambari-agent/cache", - "mysql_jdbc_url": "http://u1201.ambari.apache.org:8080/resources//mysql-connector-java.jar"; - }, + "jdk_name": "jdk-7u45-linux-x64.tar.gz", + "java_home": "/usr/jdk64/jdk1.7.0_45", + "mysql_jdbc_url": "http://c6401.ambari.apache.org/resources//mysql-connector-java.jar"; + }, + "repositoryFile": { + "resolved": true, + "repoVersion": "2.6.4.0-60", + "repositories": [ + { + "mirrorsList": null, + "ambariManaged": true, + "baseUrl": "http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.6.4.0-60";, + "repoName": "HDP", + "components": null, + "osType": "redhat6", + "distribution": null, + "repoId": "HDP-2.6-repo-1" + }, + { + "mirrorsList": null, + "ambariManaged": true, + "baseUrl": "http://s3.amazonaws.com/dev.hortonworks.com/HDP-GPL/centos6/2.x/BUILDS/2.6.4.0-60";, + "repoName": "HDP-GPL", + "components": null, + "osType": "redhat6", + "distribution": null, + "repoId": "HDP-2.6-GPL-repo-1" + }, + { + "mirrorsList": null, + "ambariManaged": true, + "baseUrl": "http://s3.amazonaws.com/dev.hortonworks.com/HDP-UTILS-1.1.0.22/repos/centos6";, + "repoName": "HDP-UTILS", + "components": null, + "osType": "redhat6", + "distribution": null, + "repoId": "HDP-UTILS-1.1.0.22-repo-1" + } + ], + "feature": { + "m_isScoped": true, + "m_isPreInstalled": false + }, + "stackName": "HDP", + "repoVersionId": 1 + }, + "agentLevelParams": { + "public_hostname": "c6401.ambari.apache.org", + "hostname": "c6401.ambari.apache.org", + "agentConfigParams": { + "agent": { + "parallel_execution": 0, + "use_system_proxy_settings": true + } + }, + "agentCacheDir": "/var/lib/ambari-agent/cache" + }, + "hostLevelParams": { + "recoveryConfig": { + "retryGap": "5", + "windowInMinutes": "60", + "maxLifetimeCount": "1024", + "components": "", + "maxCount": "6", + "type": "AUTO_START" + }, + "hostRepositories": { + "componentRepos": { + "NAMENODE": 1, + "SECONDARY_NAMENODE": 1, + "DATANODE": 1, + "HDFS_CLIENT": 1 + }, + "commandRepos": { + "1": { + "resolved": true, + "repoVersion": "2.1.4.0-60", + "repositories": [ + { + "mirrorsList": null, + "ambariManaged": true, + "baseUrl": "http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.6.4.0-60";, + "repoName": "HDP", + "components": null, + "osType": "redhat6", + "distribution": null, + "repoId": "HDP-2.6-repo-1" + }, + { + "mirrorsList": null, + "ambariManaged": true, + "baseUrl": "http://s3.amazonaws.com/dev.hortonworks.com/HDP-GPL/centos6/2.x/BUILDS/2.6.4.0-60";, + "repoName": "HDP-GPL", + "components": null, + "osType": "redhat6", + "distribution": null, + "repoId": "HDP-2.6-GPL-repo-1" + }, + { + "mirrorsList": null, + "ambariManaged": true, + "baseUrl": "http://s3.amazonaws.com/dev.hortonworks.com/HDP-UTILS-1.1.0.22/repos/centos6";, + "repoName": "HDP-UTILS", + "components": null, + "osType": "redhat6", + "distribution": null, + "repoId": "HDP-UTILS-1.1.0.22-repo-1" + } + ], + "feature": { + "m_isScoped": true, + "m_isPreInstalled": false + }, + "stackName": "HDP", + "repoVersionId": 1 + } + } + } + }, + "serviceLevelParams": { + "credentialStoreEnabled": false, + "status_commands_timeout": 300, + "version": "2.7.3", + "service_package_folder": "common-services/HDFS/2.1.0.2.0/package" + }, "commandType": "EXECUTION_COMMAND", "roleParams": { "version": "2.2.1.0-2260", @@ -247,4 +379,4 @@ "u1203.ambari.apache.org" ] } -} \ No newline at end of file +} -- To stop receiving notification emails like this one, please contact "[email protected]" <[email protected]>.
