Merge from branch-2.5
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/333c7015 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/333c7015 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/333c7015 Branch: refs/heads/branch-feature-AMBARI-18901 Commit: 333c70150963e9bee4388d03356443d9e35b5f7f Parents: b749054 7f60b1f Author: Swapan Shridhar <sshrid...@hortonworks.com> Authored: Thu Dec 1 00:13:14 2016 -0800 Committer: Swapan Shridhar <sshrid...@hortonworks.com> Committed: Thu Dec 1 00:13:14 2016 -0800 ---------------------------------------------------------------------- .../views/stackVersions/stackVersionPage.html | 2 +- ambari-agent/conf/unix/agent-multiplier.py | 14 +- .../src/main/python/ambari_agent/ActionQueue.py | 10 +- .../ambari_agent/CustomServiceOrchestrator.py | 21 +- .../TestCustomServiceOrchestrator.py | 25 +- .../libraries/script/script.py | 15 +- .../logfeeder/metrics/LogFeederAMSClient.java | 3 +- .../logfeeder/metrics/MetricsManager.java | 2 +- .../org/apache/ambari/logsearch/LogSearch.java | 15 +- .../logsearch/common/ExternalServerClient.java | 12 +- .../logsearch/common/LogSearchConstants.java | 1 + .../AbstractOperationHolderConverter.java | 9 + ...actServiceLogRequestFacetQueryConverter.java | 44 ++ .../BaseServiceLogRequestQueryConverter.java | 1 + ...ServiceLogAnyGraphRequestQueryConverter.java | 13 + ...eLogComponentLevelRequestQueryConverter.java | 15 +- ...eLogComponentRequestFacetQueryConverter.java | 15 +- ...rviceLogLevelCountRequestQueryConverter.java | 16 + ...eLogLevelDateRangeRequestQueryConverter.java | 16 +- ...erviceLogTreeRequestFacetQueryConverter.java | 17 +- .../ambari/logsearch/doc/DocConstants.java | 1 + .../logsearch/graph/GraphDataGenerator.java | 26 +- .../logsearch/manager/ServiceLogsManager.java | 2 +- .../request/ServiceLogParamDefinition.java | 6 + .../request/impl/BaseServiceLogRequest.java | 13 + .../LogsearchKRBAuthenticationFilter.java | 4 +- .../src/main/resources/logsearch.properties | 2 +- .../dashboard/BubbleGraphTableLayoutView.js | 4 +- .../views/dashboard/ComponentListView.js | 2 +- .../scripts/views/dashboard/HostListView.js | 14 +- .../main/webapp/scripts/views/tabs/TreeView.js | 10 +- .../webapp/templates/tabs/TreeView_tmpl.html | 115 ++-- ...ComponentLevelRequestQueryConverterTest.java | 7 +- ...ComponentRequestFacetQueryConverterTest.java | 4 +- .../ambari-metrics/datasource.js | 6 +- .../conf/unix/ambari-metrics-grafana | 34 +- ambari-server/docs/configuration/index.md | 25 +- ambari-server/pom.xml | 2 + ambari-server/sbin/ambari-server | 6 +- .../server/actionmanager/ActionScheduler.java | 2 +- .../ambari/server/agent/HeartBeatHandler.java | 2 +- .../ambari/server/agent/HeartbeatProcessor.java | 2 +- .../ambari/server/checks/CheckDescription.java | 13 + .../checks/DatabaseConsistencyCheckHelper.java | 10 +- .../server/checks/ServicePresenceCheck.java | 172 ++++++ .../server/configuration/Configuration.java | 30 +- .../internal/ClusterControllerImpl.java | 2 +- .../ServiceConfigVersionResourceProvider.java | 3 +- .../logging/LoggingSearchPropertyProvider.java | 11 +- .../ambari/server/events/AmbariEvent.java | 5 + .../events/ClusterConfigFinishedEvent.java | 53 ++ .../ambari/server/orm/dao/ClusterDAO.java | 24 + .../ambari/server/orm/dao/ServiceConfigDAO.java | 13 +- .../orm/entities/ClusterConfigEntity.java | 7 +- .../entities/ClusterConfigMappingEntity.java | 6 + .../orm/entities/ServiceConfigEntity.java | 5 +- .../server/orm/helpers/dbms/MySqlHelper.java | 18 + .../AmbariLdapAuthenticationProvider.java | 23 +- .../AmbariLdapBindAuthenticator.java | 233 ++++++- .../kerberos/MITKerberosOperationHandler.java | 22 +- .../upgrades/RangerKmsProxyConfig.java | 95 +++ .../server/state/cluster/ClusterImpl.java | 42 +- .../ambari/server/state/quicklinks/Link.java | 21 + .../services/RetryUpgradeActionService.java | 2 +- .../ambari/server/topology/TopologyManager.java | 16 +- .../server/upgrade/SchemaUpgradeHelper.java | 35 +- .../server/upgrade/UpgradeCatalog250.java | 66 +- .../ambari/server/utils/ShellCommandUtil.java | 11 +- .../ambari/server/view/ViewExtractor.java | 2 +- .../apache/ambari/server/view/ViewRegistry.java | 381 +++++++----- ambari-server/src/main/python/ambari-server.py | 149 ++++- .../python/ambari_server/serverConfiguration.py | 9 +- .../main/python/ambari_server/serverUtils.py | 11 + .../main/python/ambari_server/setupActions.py | 1 + .../main/python/ambari_server/setupMpacks.py | 167 ++++- .../src/main/python/ambari_server/utils.py | 24 +- .../src/main/python/ambari_server_main.py | 44 +- .../main/resources/Ambari-DDL-MySQL-CREATE.sql | 1 + .../0.1.0/configuration/storm-site.xml | 84 --- .../AMBARI_METRICS/0.1.0/metainfo.xml | 4 - .../package/scripts/metadata_server.py | 18 + .../FLUME/1.4.0.2.0/metrics.json | 62 -- .../package/files/hbaseSmokeVerify.sh | 2 +- .../HBASE/0.96.0.2.0/package/scripts/upgrade.py | 3 +- .../package/scripts/hive_server_interactive.py | 5 +- .../0.12.0.2.0/package/scripts/params_linux.py | 1 + .../configuration/logfeeder-ambari-config.xml | 37 ++ .../configuration/logfeeder-output-config.xml | 37 ++ .../configuration/logsearch-properties.xml | 40 ++ .../LOGSEARCH/0.5.0/metainfo.xml | 5 + .../LOGSEARCH/0.5.0/package/scripts/params.py | 17 +- .../0.5.0/package/scripts/setup_logfeeder.py | 10 + .../templates/input.config-ambari.json.j2 | 602 ------------------- .../package/templates/output.config.json.j2 | 61 -- .../properties/input.config-ambari.json.j2 | 602 +++++++++++++++++++ .../0.5.0/properties/output.config.json.j2 | 61 ++ .../RANGER/0.4.0/package/scripts/params.py | 14 + .../0.4.0/package/scripts/setup_ranger_xml.py | 21 + .../common-services/RANGER/0.6.0/metainfo.xml | 2 + .../STORM/1.0.1/configuration/storm-site.xml | 54 ++ .../0.6.0.2.5/package/scripts/master.py | 15 +- .../0.6.0.2.5/package/scripts/params.py | 28 +- .../custom_actions/scripts/install_packages.py | 2 +- .../src/main/resources/scripts/stack_advisor.py | 11 +- .../HDP/2.0.6/configuration/cluster-env.xml | 10 +- .../stacks/HDP/2.0.6/services/stack_advisor.py | 50 +- .../stacks/HDP/2.1/services/stack_advisor.py | 20 +- .../stacks/HDP/2.2/services/stack_advisor.py | 7 - .../HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml | 12 +- .../HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml | 12 +- .../stacks/HDP/2.3/upgrades/upgrade-2.5.xml | 16 +- .../stacks/HDP/2.3/upgrades/upgrade-2.6.xml | 16 +- .../HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml | 12 +- .../HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml | 12 +- .../stacks/HDP/2.4/upgrades/upgrade-2.5.xml | 16 +- .../stacks/HDP/2.4/upgrades/upgrade-2.6.xml | 16 +- .../stacks/HDP/2.5/role_command_order.json | 2 +- .../HIVE/configuration/hive-interactive-env.xml | 10 + .../stacks/HDP/2.5/services/KNOX/metainfo.xml | 4 - .../stacks/HDP/2.5/services/OOZIE/metainfo.xml | 4 + .../stacks/HDP/2.5/services/stack_advisor.py | 52 +- .../HDP/2.5/upgrades/host-upgrade-2.5.xml | 11 +- .../services/ATLAS/themes/theme_version_2.json | 28 +- .../configuration/hive-interactive-site.xml | 58 ++ .../HIVE/configuration/tez-interactive-site.xml | 78 +++ .../stacks/HDP/2.6/services/stack_advisor.py | 2 +- .../PERF/1.0/configuration/cluster-env.xml | 6 +- .../GRUMPY/configuration/grumpy-site.xml | 2 +- .../services/HAPPY/configuration/happy-site.xml | 2 +- .../services/HBASE/configuration/hbase-env.xml | 30 +- .../HBASE/configuration/hbase-log4j.xml | 2 +- .../configuration/hbase-logsearch-conf.xml | 6 +- .../HBASE/configuration/hbase-policy.xml | 6 +- .../services/HBASE/configuration/hbase-site.xml | 74 +-- .../services/HDFS/configuration/core-site.xml | 32 +- .../services/HDFS/configuration/hadoop-env.xml | 44 +- .../hadoop-metrics2.properties.xml | 2 +- .../HDFS/configuration/hadoop-policy.xml | 22 +- .../services/HDFS/configuration/hdfs-log4j.xml | 2 +- .../HDFS/configuration/hdfs-logsearch-conf.xml | 6 +- .../services/HDFS/configuration/hdfs-site.xml | 108 ++-- .../services/HDFS/configuration/ssl-client.xml | 14 +- .../services/HDFS/configuration/ssl-server.xml | 16 +- .../SLEEPY/configuration/sleepy-site.xml | 2 +- .../services/SNOW/configuration/snow-site.xml | 2 +- .../YARN/configuration-mapred/mapred-env.xml | 2 +- .../YARN/configuration-mapred/mapred-site.xml | 28 +- .../services/YARN/configuration/yarn-env.xml | 8 +- .../services/YARN/configuration/yarn-log4j.xml | 2 +- .../services/YARN/configuration/yarn-site.xml | 150 ++--- .../src/main/resources/stacks/stack_advisor.py | 200 +----- .../server/checks/ServicePresenceCheckTest.java | 217 +++++++ .../StackDefinedPropertyProviderTest.java | 2 + .../RestMetricsPropertyProviderTest.java | 2 + .../AmbariLdapBindAuthenticatorTest.java | 226 ++++--- .../MITKerberosOperationHandlerTest.java | 23 + .../upgrades/RangerKmsProxyConfigTest.java | 141 +++++ .../QuickLinksConfigurationModuleTest.java | 36 ++ .../ClusterDeployWithStartOnlyTest.java | 2 +- ...InstallWithoutStartOnComponentLevelTest.java | 2 +- .../ClusterInstallWithoutStartTest.java | 2 +- .../server/upgrade/UpgradeCatalog250Test.java | 5 + .../server/utils/TestShellCommandUtil.java | 13 +- .../ambari/server/view/ViewRegistryTest.java | 2 + .../src/test/python/TestAmbariServer.py | 4 +- ambari-server/src/test/python/TestMpacks.py | 223 +++++-- ambari-server/src/test/python/TestUtils.py | 26 +- .../grafana-dashboards/grafana-hdfs-users.json | 270 +++++++++ .../dashboards/service-metrics/STORM.txt | 7 + .../stacks/2.0.6/common/test_stack_advisor.py | 34 +- .../stacks/2.1/common/test_stack_advisor.py | 5 - .../stacks/2.2/common/test_stack_advisor.py | 19 +- .../stacks/2.3/ATLAS/test_metadata_server.py | 5 +- .../stacks/2.4/LOGSEARCH/test_logfeeder.py | 14 +- .../test/python/stacks/2.4/configs/default.json | 6 + .../stacks/2.5/common/test_stack_advisor.py | 79 ++- .../stacks/2.6/common/test_stack_advisor.py | 4 +- .../test/python/stacks/test_stack_adviser.py | 239 -------- .../python/uninstall/common-services/SERVICEA | 1 + .../python/uninstall/common-services/SERVICEB | 1 + .../test/python/uninstall/dashboards/SERVICEA | 1 + .../test/python/uninstall/dashboards/SERVICEB | 1 + .../uninstall/dashboards/files/README.txt | 17 + .../python/uninstall/dashboards/files/STORM.txt | 1 + .../uninstall/dashboards/files/metainfo.xml | 1 + .../test/python/uninstall/extensions/SERVICEA | 1 + .../test/python/uninstall/extensions/SERVICEB | 1 + .../test/python/uninstall/stacks/2.0/SERVICEA | 1 + .../test/python/uninstall/stacks/2.0/SERVICEB | 1 + .../uninstall/stacks/2.0/files/README.txt | 17 + .../uninstall/stacks/2.0/files/metainfo1.xml | 1 + .../uninstall/stacks/2.0/files/metainfo2.xml | 1 + .../child_quicklinks_with_properties.json | 64 ++ .../parent_quicklinks_with_properties.json | 65 ++ ambari-web/app/assets/test/tests.js | 5 + ambari-web/app/controllers.js | 1 + .../main/admin/service_auto_start.js | 58 +- .../admin/stack_upgrade_history_controller.js | 217 +++++++ .../alert_definitions_actions_controller.js | 4 +- ambari-web/app/data/HDP2.3/site_properties.js | 25 +- ambari-web/app/data/HDP2/site_properties.js | 28 + ambari-web/app/mappers.js | 3 +- .../app/mappers/stack_upgrade_history_mapper.js | 54 ++ ambari-web/app/messages.js | 40 +- ambari-web/app/mixins.js | 1 + ambari-web/app/models.js | 2 + .../app/models/finished_upgrade_entity.js | 92 +++ .../stack_version/stack_upgrade_history.js | 37 ++ ambari-web/app/routes/main.js | 7 + ambari-web/app/styles/stack_versions.less | 69 +++ .../highAvailability/journalNode/step2.hbs | 14 +- .../templates/main/admin/service_auto_start.hbs | 12 +- .../admin/stack_upgrade/upgrade_history.hbs | 105 ++++ .../stack_upgrade/upgrade_history_details.hbs | 46 ++ .../app/utils/configs/config_initializer.js | 28 +- .../mount_points_based_initializer_mixin.js | 340 +++++++++++ ambari-web/app/views.js | 2 + ambari-web/app/views/main/admin.js | 3 +- .../app/views/main/admin/service_auto_start.js | 25 +- .../service_auto_start/component_auto_start.js | 10 + .../views/main/admin/stack_upgrade/menu_view.js | 6 + .../upgrade_history_details_view.js | 85 +++ .../admin/stack_upgrade/upgrade_history_view.js | 303 ++++++++++ ambari-web/app/views/main/dashboard/widgets.js | 17 +- .../stack_upgrade_history_controller_test.js | 125 ++++ .../stack_upgrade_history_mapper_test.js | 372 ++++++++++++ .../test/models/finished_upgrade_entity_test.js | 197 ++++++ ambari-web/test/utils/ajax/ajax_test.js | 9 +- .../utils/configs/config_initializer_test.js | 458 ++++++++++++++ .../upgrade_history_details_view_test.js | 248 ++++++++ .../stack_upgrade/upgrade_history_view_test.js | 173 ++++++ .../upgrade_version_box_view_test.js | 2 +- .../test/views/main/dashboard/widgets_test.js | 11 +- contrib/utils/perf/deploy-gce-perf-cluster.py | 405 +++++++++---- contrib/views/ambari-views-package/pom.xml | 123 ++++ .../src/main/package/deb/control/control | 22 + contrib/views/capacity-scheduler/pom.xml | 18 + contrib/views/files/pom.xml | 18 + contrib/views/hawq/pom.xml | 18 + contrib/views/hive-next/pom.xml | 18 + contrib/views/hive/pom.xml | 18 + contrib/views/hueambarimigration/pom.xml | 18 + contrib/views/jobs/pom.xml | 23 + contrib/views/pig/pom.xml | 18 + contrib/views/pom.xml | 5 + contrib/views/slider/pom.xml | 18 + contrib/views/storm/pom.xml | 23 + contrib/views/tez/pom.xml | 18 + contrib/views/wfmanager/pom.xml | 18 + .../resources/ui/app/components/job-details.js | 3 + contrib/views/zeppelin/pom.xml | 18 + 251 files changed, 8434 insertions(+), 2681 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/333c7015/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java index 6de48c1,52de784..1f44e28 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java @@@ -17,8 -17,14 +17,16 @@@ */ package org.apache.ambari.server.upgrade; +import com.google.inject.Inject; +import com.google.inject.Injector; + import java.sql.SQLException; + import java.util.ArrayList; + import java.util.Collections; + import java.util.HashMap; + import java.util.List; + import java.util.Map; + import java.util.Set; + import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.actionmanager.CommandExecutionType; import org.apache.ambari.server.controller.AmbariManagementController; @@@ -32,13 -38,9 +40,6 @@@ import org.apache.commons.lang.StringUt import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import java.sql.SQLException; - import java.util.ArrayList; - import java.util.Collections; - import java.util.HashMap; - import java.util.List; - import java.util.Map; -import com.google.inject.Inject; -import com.google.inject.Injector; -- /** * Upgrade catalog for version 2.5.0. */ @@@ -133,8 -135,7 +134,9 @@@ public class UpgradeCatalog250 extends addNewConfigurationsFromXml(); updateAMSConfigs(); updateKafkaConfigs(); + updateHIVEInteractiveConfigs(); + updateTEZInteractiveConfigs(); + updateHiveLlapConfigs(); } protected void updateHostVersionTable() throws SQLException { http://git-wip-us.apache.org/repos/asf/ambari/blob/333c7015/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py ---------------------------------------------------------------------- diff --cc ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py index 4866432,7335637..2c5ebeb --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py @@@ -277,20 -274,10 +277,21 @@@ class HiveServerInteractiveDefault(Hive unique_name = "llap-slider%s" % datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S') + # Figure out the Slider Anti-affinity to be used. + # YARN does not support anti-affinity, and therefore Slider implements AA by the means of exclusion lists, i.e, it + # starts containers one by one and excludes the nodes it gets (adding a delay of ~2sec./machine). When the LLAP + # container memory size configuration is more than half of YARN node memory, AA is implicit and should be avoided. + slider_placement = 4 + if long(params.llap_daemon_container_size) > (0.5 * long(params.yarn_nm_mem)): + slider_placement = 0 + Logger.info("Setting slider_placement : 0, as llap_daemon_container_size : {0} > 0.5 * " + "YARN NodeManager Memory({1})".format(params.llap_daemon_container_size, params.yarn_nm_mem)) + cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances {params.num_llap_nodes}" - " --slider-am-container-mb {params.slider_am_container_mb} --size {params.llap_daemon_container_size}m " + " --slider-am-container-mb {params.slider_am_container_mb} --size {params.llap_daemon_container_size}m" " --cache {params.hive_llap_io_mem_size}m --xmx {params.llap_heap_size}m --loglevel {params.llap_log_level}" - " --slider-placement {slider_placement} --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}") ++ " --slider-placement {slider_placement} --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}" + " {params.llap_extra_slider_opts} --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}") if params.security_enabled: llap_keytab_splits = params.hive_llap_keytab_file.split("/") Logger.debug("llap_keytab_splits : {0}".format(llap_keytab_splits)) http://git-wip-us.apache.org/repos/asf/ambari/blob/333c7015/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/333c7015/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/333c7015/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/333c7015/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java ---------------------------------------------------------------------- diff --cc ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java index 5b520c3,ce0b387..dc8a7e5 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java @@@ -210,15 -211,13 +210,17 @@@ public class UpgradeCatalog250Test public void testExecuteDMLUpdates() throws Exception { Method updateAmsConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateAMSConfigs"); Method updateKafkaConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateKafkaConfigs"); + Method updateHiveLlapConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateHiveLlapConfigs"); Method addNewConfigurationsFromXml = AbstractUpgradeCatalog.class.getDeclaredMethod("addNewConfigurationsFromXml"); + Method updateHIVEInteractiveConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateHIVEInteractiveConfigs"); + Method updateTEZInteractiveConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateTEZInteractiveConfigs"); UpgradeCatalog250 upgradeCatalog250 = createMockBuilder(UpgradeCatalog250.class) .addMockedMethod(updateAmsConfigs) .addMockedMethod(updateKafkaConfigs) + .addMockedMethod(updateHIVEInteractiveConfigs) + .addMockedMethod(updateTEZInteractiveConfigs) + .addMockedMethod(updateHiveLlapConfigs) .addMockedMethod(addNewConfigurationsFromXml) .createMock(); @@@ -232,12 -231,9 +234,15 @@@ upgradeCatalog250.updateKafkaConfigs(); expectLastCall().once(); + upgradeCatalog250.updateHIVEInteractiveConfigs(); + expectLastCall().once(); + + upgradeCatalog250.updateTEZInteractiveConfigs(); + expectLastCall().once(); + + upgradeCatalog250.updateHiveLlapConfigs(); + expectLastCall().once(); + replay(upgradeCatalog250); upgradeCatalog250.executeDMLUpdates();