Merge remote-tracking branch 'origin/trunk' into branch-feature-AMBARI-14714
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c36afcdd Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c36afcdd Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c36afcdd Branch: refs/heads/branch-feature-AMBARI-14714 Commit: c36afcdd051fa28cff2d299f1b97ed92e32fbbe5 Parents: ba1ec6d e61556c Author: Jayush Luniya <[email protected]> Authored: Sun Oct 8 23:00:20 2017 -0700 Committer: Jayush Luniya <[email protected]> Committed: Sun Oct 8 23:00:20 2017 -0700 ---------------------------------------------------------------------- ambari-agent/pom.xml | 4 +- .../main/python/ambari_agent/AmbariConfig.py | 74 +- .../ambari_agent/CustomServiceOrchestrator.py | 6 +- .../src/main/python/ambari_agent/FileCache.py | 5 +- .../main/python/ambari_agent/ProcessHelper.py | 71 -- .../src/main/python/ambari_agent/StatusCheck.py | 142 ---- .../src/main/python/ambari_agent/main.py | 12 +- .../test/python/ambari_agent/TestFileCache.py | 4 +- .../test/python/ambari_agent/TestLiveStatus.py | 5 +- .../src/test/python/ambari_agent/TestMain.py | 13 +- .../python/ambari_agent/TestProcessHelper.py | 70 -- .../test/python/ambari_agent/TestSecurity.py | 1 - .../test/python/ambari_agent/TestStatusCheck.py | 180 ----- .../resource_management/TestPackageResource.py | 6 +- .../python/resource_management/TestScript.py | 26 +- .../core/providers/package/__init__.py | 4 +- .../core/providers/package/apt.py | 35 +- .../core/providers/package/choco.py | 8 +- .../core/providers/package/yumrpm.py | 10 +- .../core/providers/package/zypper.py | 7 +- .../core/resources/packaging.py | 14 +- .../libraries/functions/component_version.py | 26 +- .../libraries/functions/conf_select.py | 362 +++------ .../libraries/functions/repository_util.py | 8 +- .../libraries/functions/stack_select.py | 69 +- .../libraries/script/script.py | 22 +- ambari-infra/ambari-infra-assembly/pom.xml | 8 + .../src/main/package/deb/manager/postinst | 5 + .../src/main/package/deb/solr-client/postinst | 13 + .../src/main/package/rpm/manager/postinstall.sh | 20 + .../main/package/rpm/solr-client/postinstall.sh | 28 + .../org/apache/ambari/infra/InfraManager.java | 4 +- .../conf/batch/InfraManagerBatchConfig.java | 86 -- .../infra/job/dummy/DummyJobConfiguration.java | 108 +++ .../src/main/resources/infraManager.sh | 10 +- ambari-infra/ambari-infra-solr-client/build.xml | 3 + .../src/main/python/solrDataManager.py | 150 ++-- .../src/main/resources/solrIndexHelper.sh | 5 +- .../logsearch/steps/LogSearchUISteps.java | 4 +- .../resources/stories/selenium/login.ui.story | 4 +- .../ambari-logsearch-web/package.json | 17 +- ambari-logsearch/ambari-logsearch-web/pom.xml | 6 +- .../ambari-logsearch-web/src/app/app.module.ts | 6 + .../src/app/classes/active-service-log-entry.ts | 23 + .../src/app/classes/histogram-options.ts | 36 + .../src/app/classes/list-item.class.ts | 25 - .../src/app/classes/list-item.ts | 26 + .../src/app/classes/models/app-settings.ts | 27 + .../src/app/classes/models/app-state.ts | 43 + .../src/app/classes/models/audit-log-field.ts | 225 ++++++ .../src/app/classes/models/audit-log.ts | 46 ++ .../src/app/classes/models/bar-graph.ts | 24 + .../src/app/classes/models/common-entry.ts | 22 + .../src/app/classes/models/count.ts | 22 + .../src/app/classes/models/filter.ts | 25 + .../src/app/classes/models/graph.ts | 23 + .../src/app/classes/models/log-field.ts | 27 + .../src/app/classes/models/log.ts | 38 + .../src/app/classes/models/node.ts | 30 + .../src/app/classes/models/service-log-field.ts | 107 +++ .../src/app/classes/models/service-log.ts | 27 + .../app/classes/models/solr-collection-state.ts | 23 + .../src/app/classes/models/store.ts | 180 +++++ .../src/app/classes/models/user-config.ts | 26 + .../queries/audit-logs-query-params.class.ts | 46 -- .../classes/queries/audit-logs-query-params.ts | 46 ++ .../app/classes/queries/query-params.class.ts | 23 - .../src/app/classes/queries/query-params.ts | 23 + ...ce-logs-histogram-query-params.class.spec.ts | 203 ----- ...service-logs-histogram-query-params.class.ts | 70 -- .../service-logs-histogram-query-params.spec.ts | 203 +++++ .../service-logs-histogram-query-params.ts | 70 ++ .../queries/service-logs-query-params.class.ts | 30 - .../queries/service-logs-query-params.ts | 30 + .../service-logs-truncated-query-params.ts | 36 + .../app/classes/service-log-context-entry.ts | 26 + .../dropdown-button.component.html | 3 +- .../dropdown-button.component.spec.ts | 5 +- .../dropdown-button.component.ts | 2 +- .../dropdown-list/dropdown-list.component.html | 2 +- .../dropdown-list.component.spec.ts | 12 +- .../dropdown-list/dropdown-list.component.ts | 11 +- .../filter-button.component.spec.ts | 5 +- .../filter-button/filter-button.component.ts | 2 +- .../filter-dropdown.component.spec.ts | 5 +- .../filters-panel/filters-panel.component.html | 3 +- .../filters-panel/filters-panel.component.less | 2 +- .../filters-panel.component.spec.ts | 5 +- .../filters-panel/filters-panel.component.ts | 4 +- .../log-context/log-context.component.html | 33 + .../log-context/log-context.component.less | 23 + .../log-context/log-context.component.spec.ts | 108 +++ .../log-context/log-context.component.ts | 91 +++ .../log-file-entry.component.html | 20 + .../log-file-entry.component.less | 31 + .../log-file-entry.component.spec.ts | 56 ++ .../log-file-entry/log-file-entry.component.ts | 51 ++ .../logs-container.component.html | 12 +- .../logs-container.component.spec.ts | 5 +- .../logs-container/logs-container.component.ts | 44 +- .../logs-list/logs-list.component.html | 30 +- .../logs-list/logs-list.component.less | 36 +- .../logs-list/logs-list.component.spec.ts | 3 + .../components/logs-list/logs-list.component.ts | 34 +- .../main-container.component.html | 7 + .../main-container.component.less | 4 + .../main-container.component.spec.ts | 13 +- .../main-container/main-container.component.ts | 32 +- .../menu-button/menu-button.component.spec.ts | 5 +- .../menu-button/menu-button.component.ts | 2 +- .../search-box/search-box.component.ts | 2 +- .../time-histogram.component.less | 22 +- .../time-histogram/time-histogram.component.ts | 94 ++- .../timezone-picker.component.spec.ts | 5 +- .../src/app/components/variables.less | 30 + .../src/app/models/app-settings.model.ts | 27 - .../src/app/models/app-state.model.ts | 35 - .../src/app/models/audit-log-field.model.ts | 225 ------ .../src/app/models/audit-log.model.ts | 46 -- .../src/app/models/bar-graph.model.ts | 24 - .../src/app/models/common-entry.model.ts | 22 - .../src/app/models/count.model.ts | 22 - .../src/app/models/filter.model.ts | 25 - .../src/app/models/graph.model.ts | 23 - .../src/app/models/log-field.model.ts | 27 - .../src/app/models/log.model.ts | 37 - .../src/app/models/node.model.ts | 29 - .../src/app/models/service-log-field.model.ts | 107 --- .../src/app/models/service-log.model.ts | 27 - .../app/models/solr-collection-state.model.ts | 23 - .../src/app/models/store.model.ts | 169 ---- .../src/app/models/user-config.model.ts | 26 - .../services/component-actions.service.spec.ts | 8 +- .../app/services/component-actions.service.ts | 52 +- .../component-generator.service.spec.ts | 10 +- .../app/services/component-generator.service.ts | 23 +- .../src/app/services/filtering.service.spec.ts | 27 + .../src/app/services/filtering.service.ts | 97 ++- .../src/app/services/http-client.service.ts | 15 +- .../app/services/logs-container.service.spec.ts | 8 +- .../src/app/services/logs-container.service.ts | 70 +- .../services/storage/app-settings.service.ts | 4 +- .../app/services/storage/app-state.service.ts | 4 +- .../storage/audit-logs-fields.service.ts | 2 +- .../app/services/storage/audit-logs.service.ts | 2 +- .../app/services/storage/clusters.service.ts | 2 +- .../app/services/storage/components.service.ts | 2 +- .../src/app/services/storage/filters.service.ts | 2 +- .../src/app/services/storage/graphs.service.ts | 2 +- .../src/app/services/storage/hosts.service.ts | 2 +- .../app/services/storage/reducers.service.ts | 2 + .../storage/service-logs-fields.service.ts | 2 +- .../service-logs-histogram-data.service.ts | 2 +- .../storage/service-logs-truncated.service.ts | 32 + .../services/storage/service-logs.service.ts | 2 +- .../services/storage/user-configs.service.ts | 2 +- .../src/assets/i18n/en.json | 7 +- .../ambari-logsearch-web/webpack.config.js | 21 +- ambari-logsearch/ambari-logsearch-web/yarn.lock | 288 +++++-- .../timeline/HBaseTimelineMetricStore.java | 8 +- .../timeline/HBaseTimelineMetricStoreTest.java | 36 +- ambari-server/pom.xml | 2 + ambari-server/src/main/assemblies/server.xml | 20 +- .../actionmanager/ExecutionCommandWrapper.java | 3 +- .../ambari/server/actionmanager/Stage.java | 3 +- .../ambari/server/agent/HeartbeatMonitor.java | 3 +- .../ambari/server/agent/StatusCommand.java | 16 + .../server/api/services/AmbariMetaInfo.java | 61 +- .../AmbariManagementControllerImpl.java | 22 +- .../server/controller/ControllerModule.java | 4 + .../server/controller/KerberosHelperImpl.java | 16 +- .../controller/ResourceProviderFactory.java | 8 + .../server/controller/StackVersionResponse.java | 29 - .../AbstractControllerResourceProvider.java | 4 + .../internal/AlertTargetResourceProvider.java | 3 +- .../internal/ClientConfigResourceProvider.java | 2 - .../internal/DefaultProviderModule.java | 4 - .../internal/StackArtifactResourceProvider.java | 35 +- .../internal/UpgradeResourceProvider.java | 77 +- .../internal/ViewInstanceResourceProvider.java | 147 ++-- .../upgrade/HostVersionOutOfSyncListener.java | 4 + .../ambari/server/mpack/MpackGenerator.java | 17 - .../orm/entities/RepositoryVersionEntity.java | 34 +- .../server/orm/entities/WidgetLayoutEntity.java | 6 +- .../upgrades/AbstractUpgradeServerAction.java | 6 +- .../upgrades/AutoSkipFailedSummaryAction.java | 15 +- .../upgrades/ComponentVersionCheckAction.java | 2 +- .../serveraction/upgrades/ConfigureAction.java | 11 +- .../upgrades/FinalizeUpgradeAction.java | 7 +- .../FixCapacitySchedulerOrderingPolicy.java | 12 +- .../serveraction/upgrades/FixLzoCodecPath.java | 10 +- .../upgrades/FixOozieAdminUsers.java | 10 +- .../upgrades/FixYarnWebServiceUrl.java | 11 +- .../upgrades/HBaseConfigCalculation.java | 10 +- .../HBaseEnvMaxDirectMemorySizeAction.java | 11 +- .../upgrades/HiveEnvClasspathAction.java | 11 +- .../upgrades/HiveZKQuorumConfigAction.java | 13 +- .../upgrades/KerberosKeytabsAction.java | 13 +- .../upgrades/ManualStageAction.java | 3 +- .../upgrades/OozieConfigCalculation.java | 11 +- .../upgrades/PreconfigureKerberosAction.java | 2 +- .../upgrades/RangerConfigCalculation.java | 11 +- .../RangerKerberosConfigCalculation.java | 11 +- .../upgrades/RangerKmsProxyConfig.java | 11 +- .../RangerUsersyncConfigCalculation.java | 11 +- .../upgrades/RangerWebAlertConfigAction.java | 9 +- .../upgrades/SparkShufflePropertyConfig.java | 11 +- .../upgrades/UpdateDesiredRepositoryAction.java | 10 +- .../upgrades/UpgradeUserKerberosDescriptor.java | 6 +- .../upgrades/YarnConfigCalculation.java | 11 +- .../ambari/server/stack/ServiceDirectory.java | 3 +- .../ambari/server/stack/StackContext.java | 15 +- .../ambari/server/stack/StackDirectory.java | 74 +- .../ambari/server/stack/StackManager.java | 11 +- .../apache/ambari/server/stack/StackModule.java | 16 - .../org/apache/ambari/server/state/Host.java | 17 + .../apache/ambari/server/state/ServiceInfo.java | 2 +- .../apache/ambari/server/state/StackInfo.java | 39 - .../server/state/cluster/ClusterImpl.java | 2 +- .../ambari/server/state/host/HostImpl.java | 47 ++ .../AbstractKerberosDescriptorContainer.java | 26 +- .../state/kerberos/KerberosDescriptor.java | 3 +- .../ambari/server/state/stack/UpgradePack.java | 6 + .../stack/upgrade/ServerSideActionTask.java | 15 + .../state/stack/upgrade/TaskParameter.java | 41 + .../svccomphost/ServiceComponentHostImpl.java | 9 +- .../server/topology/AsyncCallableService.java | 110 ++- .../ambari/server/topology/TopologyManager.java | 26 +- .../topology/tasks/ConfigureClusterTask.java | 124 +-- .../server/upgrade/UpgradeCatalog260.java | 31 + .../ambari_server/dbConfiguration_linux.py | 9 +- .../python/ambari_server/resourceFilesKeeper.py | 7 +- .../python/ambari_server/serverConfiguration.py | 2 +- .../main/python/ambari_server/setupMpacks.py | 2 +- .../1.6.1.2.2.0/configuration/accumulo-env.xml | 6 + .../0.1.0/configuration/infra-solr-env.xml | 6 + .../0.1.0/configuration/ams-env.xml | 6 + .../AMBARI_METRICS/0.1.0/service_advisor.py | 787 +++++++++++++++++++ .../ATLAS/0.1.0.2.3/configuration/atlas-env.xml | 6 + .../ATLAS/0.7.0.3.0/configuration/atlas-env.xml | 6 + .../ATLAS/0.7.0.3.0/service_advisor.py | 5 +- .../DRUID/0.10.1/configuration/druid-broker.xml | 106 +++ .../DRUID/0.10.1/configuration/druid-common.xml | 270 +++++++ .../0.10.1/configuration/druid-coordinator.xml | 43 + .../DRUID/0.10.1/configuration/druid-env.xml | 248 ++++++ .../0.10.1/configuration/druid-historical.xml | 94 +++ .../DRUID/0.10.1/configuration/druid-log4j.xml | 84 ++ .../0.10.1/configuration/druid-logrotate.xml | 68 ++ .../configuration/druid-middlemanager.xml | 122 +++ .../0.10.1/configuration/druid-overlord.xml | 52 ++ .../DRUID/0.10.1/configuration/druid-router.xml | 59 ++ .../common-services/DRUID/0.10.1/metainfo.xml | 223 ++++++ .../DRUID/0.10.1/package/scripts/broker.py | 28 + .../DRUID/0.10.1/package/scripts/coordinator.py | 28 + .../DRUID/0.10.1/package/scripts/druid.py | 307 ++++++++ .../DRUID/0.10.1/package/scripts/druid_node.py | 114 +++ .../DRUID/0.10.1/package/scripts/historical.py | 28 + .../0.10.1/package/scripts/middlemanager.py | 28 + .../DRUID/0.10.1/package/scripts/overlord.py | 28 + .../DRUID/0.10.1/package/scripts/params.py | 200 +++++ .../DRUID/0.10.1/package/scripts/router.py | 28 + .../0.10.1/package/scripts/service_check.py | 44 ++ .../0.10.1/package/scripts/status_params.py | 24 + .../DRUID/0.10.1/quicklinks/quicklinks.json | 37 + .../DRUID/0.10.1/role_command_order.json | 17 + .../DRUID/0.10.1/themes/theme.json | 120 +++ .../DRUID/0.9.2/configuration/druid-broker.xml | 100 --- .../DRUID/0.9.2/configuration/druid-common.xml | 270 ------- .../0.9.2/configuration/druid-coordinator.xml | 43 - .../DRUID/0.9.2/configuration/druid-env.xml | 242 ------ .../0.9.2/configuration/druid-historical.xml | 88 --- .../DRUID/0.9.2/configuration/druid-log4j.xml | 84 -- .../0.9.2/configuration/druid-logrotate.xml | 68 -- .../0.9.2/configuration/druid-middlemanager.xml | 122 --- .../0.9.2/configuration/druid-overlord.xml | 52 -- .../DRUID/0.9.2/configuration/druid-router.xml | 59 -- .../common-services/DRUID/0.9.2/metainfo.xml | 223 ------ .../DRUID/0.9.2/package/scripts/broker.py | 28 - .../DRUID/0.9.2/package/scripts/coordinator.py | 28 - .../DRUID/0.9.2/package/scripts/druid.py | 307 -------- .../DRUID/0.9.2/package/scripts/druid_node.py | 114 --- .../DRUID/0.9.2/package/scripts/historical.py | 28 - .../0.9.2/package/scripts/middlemanager.py | 28 - .../DRUID/0.9.2/package/scripts/overlord.py | 28 - .../DRUID/0.9.2/package/scripts/params.py | 200 ----- .../DRUID/0.9.2/package/scripts/router.py | 28 - .../0.9.2/package/scripts/service_check.py | 44 -- .../0.9.2/package/scripts/status_params.py | 24 - .../DRUID/0.9.2/quicklinks/quicklinks.json | 37 - .../DRUID/0.9.2/role_command_order.json | 17 - .../DRUID/0.9.2/themes/theme.json | 120 --- .../0.5.0.2.1/configuration/falcon-env.xml | 6 + .../FLUME/1.4.0.2.0/configuration/flume-env.xml | 6 + .../0.96.0.2.0/configuration/hbase-env.xml | 6 + .../HBASE/2.0.0.3.0/configuration/hbase-env.xml | 6 + .../HDFS/2.1.0.2.0/configuration/hadoop-env.xml | 10 + .../HDFS/2.1.0.2.0/configuration/hdfs-site.xml | 1 + .../common-services/HDFS/2.1.0.2.0/widgets.json | 4 +- .../HDFS/3.0.0.3.0/configuration/hadoop-env.xml | 10 + .../HDFS/3.0.0.3.0/configuration/hdfs-site.xml | 1 + .../HIVE/0.12.0.2.0/configuration/hive-env.xml | 12 + .../package/scripts/hive_server_upgrade.py | 5 - .../0.12.0.2.0/package/scripts/params_linux.py | 5 +- .../0.12.0.2.0/package/scripts/status_params.py | 8 +- .../HIVE/0.12.0.2.0/package/scripts/webhcat.py | 2 +- .../HIVE/2.1.0.3.0/configuration/hive-env.xml | 12 + .../0.10.0.3.0/configuration/kafka-env.xml | 6 + .../KAFKA/0.8.1/configuration/kafka-env.xml | 6 + .../KNOX/0.5.0.2.2/configuration/knox-env.xml | 10 + .../KNOX/0.5.0.3.0/configuration/knox-env.xml | 10 + .../0.5.0/configuration/logsearch-env.xml | 6 + .../1.0.0.2.3/configuration/mahout-env.xml | 6 + .../OOZIE/4.0.0.2.0/configuration/oozie-env.xml | 10 + .../OOZIE/4.2.0.3.0/configuration/oozie-env.xml | 10 + .../RANGER/0.4.0/configuration/ranger-env.xml | 10 + .../0.4.0/package/scripts/ranger_admin.py | 13 +- .../1.0.0.3.0/configuration/ranger-env.xml | 10 + .../0.5.0.2.3/configuration/kms-env.xml | 10 + .../1.0.0.3.0/configuration/kms-env.xml | 10 + .../RANGER_KMS/1.0.0.3.0/service_advisor.py | 3 +- .../SPARK/1.2.1/configuration/spark-env.xml | 10 + .../SPARK/2.2.0/configuration/livy-env.xml | 10 + .../SPARK/2.2.0/configuration/spark-env.xml | 10 + .../SPARK2/2.0.0/configuration/spark2-env.xml | 10 + .../SQOOP/1.4.4.2.0/configuration/sqoop-env.xml | 6 + .../SQOOP/1.4.4.3.0/configuration/sqoop-env.xml | 6 + .../STORM/0.9.1/configuration/storm-env.xml | 6 + .../STORM/1.0.1.3.0/configuration/storm-env.xml | 6 + .../0.15.0/configuration/superset-env.xml | 6 + .../TEZ/0.4.0.2.1/configuration/tez-env.xml | 10 + .../TEZ/0.9.0.3.0/configuration/tez-env.xml | 10 + .../configuration-mapred/mapred-env.xml | 6 + .../YARN/2.1.0.2.0/configuration/yarn-env.xml | 6 + .../configuration-mapred/mapred-env.xml | 6 + .../YARN/3.0.0.3.0/configuration/yarn-env.xml | 6 + .../0.6.0/configuration/zeppelin-env.xml | 10 + .../0.7.0/configuration/zeppelin-env.xml | 10 + .../ZEPPELIN/0.7.0/package/scripts/master.py | 25 +- .../3.4.5/configuration/zookeeper-env.xml | 6 + .../src/main/resources/configuration-schema.xsd | 2 +- .../custom_actions/scripts/install_packages.py | 35 +- ambari-server/src/main/resources/kerberos.json | 79 ++ .../main/resources/scripts/Ambaripreupload.py | 4 +- .../scripts/post-user-creation-hook.sh | 2 +- .../stack-hooks/after-INSTALL/scripts/hook.py | 37 + .../stack-hooks/after-INSTALL/scripts/params.py | 108 +++ .../scripts/shared_initialization.py | 132 ++++ .../before-ANY/files/changeToSecureUid.sh | 64 ++ .../stack-hooks/before-ANY/scripts/hook.py | 36 + .../stack-hooks/before-ANY/scripts/params.py | 254 ++++++ .../before-ANY/scripts/shared_initialization.py | 273 +++++++ .../stack-hooks/before-INSTALL/scripts/hook.py | 37 + .../before-INSTALL/scripts/params.py | 115 +++ .../scripts/repo_initialization.py | 75 ++ .../scripts/shared_initialization.py | 37 + .../stack-hooks/before-RESTART/scripts/hook.py | 29 + .../before-START/files/checkForFormat.sh | 65 ++ .../before-START/files/fast-hdfs-resource.jar | Bin 0 -> 28296600 bytes .../before-START/files/task-log4j.properties | 134 ++++ .../before-START/files/topology_script.py | 66 ++ .../before-START/scripts/custom_extensions.py | 173 ++++ .../stack-hooks/before-START/scripts/hook.py | 43 + .../stack-hooks/before-START/scripts/params.py | 380 +++++++++ .../before-START/scripts/rack_awareness.py | 48 ++ .../scripts/shared_initialization.py | 256 ++++++ .../templates/commons-logging.properties.j2 | 43 + .../templates/exclude_hosts_list.j2 | 21 + .../templates/hadoop-metrics2.properties.j2 | 107 +++ .../before-START/templates/health_check.j2 | 81 ++ .../templates/include_hosts_list.j2 | 21 + .../templates/topology_mappings.data.j2 | 24 + .../HDP/2.0.6/configuration/cluster-env.xml | 10 + .../2.0.6/hooks/after-INSTALL/scripts/hook.py | 37 - .../2.0.6/hooks/after-INSTALL/scripts/params.py | 115 --- .../scripts/shared_initialization.py | 132 ---- .../hooks/before-ANY/files/changeToSecureUid.sh | 64 -- .../HDP/2.0.6/hooks/before-ANY/scripts/hook.py | 36 - .../2.0.6/hooks/before-ANY/scripts/params.py | 277 ------- .../before-ANY/scripts/shared_initialization.py | 281 ------- .../2.0.6/hooks/before-INSTALL/scripts/hook.py | 37 - .../hooks/before-INSTALL/scripts/params.py | 115 --- .../scripts/repo_initialization.py | 75 -- .../scripts/shared_initialization.py | 37 - .../2.0.6/hooks/before-RESTART/scripts/hook.py | 29 - .../hooks/before-START/files/checkForFormat.sh | 65 -- .../before-START/files/fast-hdfs-resource.jar | Bin 28296600 -> 0 bytes .../before-START/files/task-log4j.properties | 134 ---- .../hooks/before-START/files/topology_script.py | 66 -- .../before-START/scripts/custom_extensions.py | 173 ---- .../2.0.6/hooks/before-START/scripts/hook.py | 43 - .../2.0.6/hooks/before-START/scripts/params.py | 387 --------- .../before-START/scripts/rack_awareness.py | 48 -- .../scripts/shared_initialization.py | 256 ------ .../templates/commons-logging.properties.j2 | 43 - .../templates/exclude_hosts_list.j2 | 21 - .../templates/hadoop-metrics2.properties.j2 | 107 --- .../before-START/templates/health_check.j2 | 81 -- .../templates/include_hosts_list.j2 | 21 - .../templates/topology_mappings.data.j2 | 24 - .../resources/stacks/HDP/2.0.6/kerberos.json | 79 -- .../stacks/HDP/2.0.6/services/stack_advisor.py | 562 +------------ .../resources/stacks/HDP/2.0.6/widgets.json | 95 --- .../services/HBASE/configuration/hbase-env.xml | 6 + .../stacks/HDP/2.2/services/stack_advisor.py | 1 - .../services/ECS/package/scripts/ecs_client.py | 2 +- .../services/ECS/package/scripts/params.py | 2 +- .../services/ATLAS/configuration/atlas-env.xml | 6 + .../HIVE/configuration/llap-daemon-log4j.xml | 2 +- .../services/SPARK/configuration/livy-env.xml | 10 + .../stacks/HDP/2.5/services/stack_advisor.py | 3 +- .../stacks/HDP/2.6/kerberos_preconfigure.json | 9 + .../stacks/HDP/2.6/services/DRUID/metainfo.xml | 5 +- .../HIVE/configuration/tez-interactive-site.xml | 6 + .../services/SPARK/configuration/livy-env.xml | 10 + .../services/YARN/configuration/yarn-site.xml | 18 + .../ZEPPELIN/configuration/zeppelin-env.xml | 10 + .../stacks/HDP/2.6/services/stack_advisor.py | 28 +- .../HDP/3.0/configuration/cluster-env.xml | 10 + .../HDP/3.0/hooks/after-INSTALL/scripts/hook.py | 37 - .../3.0/hooks/after-INSTALL/scripts/params.py | 109 --- .../scripts/shared_initialization.py | 140 ---- .../hooks/before-ANY/files/changeToSecureUid.sh | 53 -- .../HDP/3.0/hooks/before-ANY/scripts/hook.py | 36 - .../HDP/3.0/hooks/before-ANY/scripts/params.py | 259 ------ .../before-ANY/scripts/shared_initialization.py | 239 ------ .../3.0/hooks/before-INSTALL/scripts/hook.py | 37 - .../3.0/hooks/before-INSTALL/scripts/params.py | 115 --- .../scripts/repo_initialization.py | 76 -- .../scripts/shared_initialization.py | 37 - .../3.0/hooks/before-RESTART/scripts/hook.py | 29 - .../hooks/before-START/files/checkForFormat.sh | 65 -- .../before-START/files/fast-hdfs-resource.jar | Bin 28296600 -> 0 bytes .../before-START/files/task-log4j.properties | 134 ---- .../hooks/before-START/files/topology_script.py | 66 -- .../HDP/3.0/hooks/before-START/scripts/hook.py | 40 - .../3.0/hooks/before-START/scripts/params.py | 364 --------- .../before-START/scripts/rack_awareness.py | 47 -- .../scripts/shared_initialization.py | 249 ------ .../templates/commons-logging.properties.j2 | 43 - .../templates/exclude_hosts_list.j2 | 21 - .../templates/hadoop-metrics2.properties.j2 | 107 --- .../before-START/templates/health_check.j2 | 81 -- .../templates/include_hosts_list.j2 | 21 - .../templates/topology_mappings.data.j2 | 24 - .../main/resources/stacks/HDP/3.0/kerberos.json | 79 -- .../main/resources/stacks/HDP/3.0/widgets.json | 95 --- .../src/main/resources/upgrade-pack.xsd | 9 + ambari-server/src/main/resources/widgets.json | 95 +++ .../server/api/services/AmbariMetaInfoTest.java | 49 +- .../AmbariManagementControllerImplTest.java | 6 +- .../AmbariManagementControllerTest.java | 17 + .../internal/UpgradeResourceProviderTest.java | 33 +- .../upgrade/StackVersionListenerTest.java | 1 - .../apache/ambari/server/orm/OrmTestHelper.java | 2 +- .../FixCapacitySchedulerOrderingPolicyTest.java | 2 +- .../upgrades/FixOozieAdminUsersTest.java | 2 +- .../upgrades/FixYarnWebServiceUrlTest.java | 2 +- .../HBaseEnvMaxDirectMemorySizeActionTest.java | 2 +- .../upgrades/HiveEnvClasspathActionTest.java | 2 +- .../upgrades/HiveZKQuorumConfigActionTest.java | 2 +- .../upgrades/KerberosKeytabsActionTest.java | 17 + .../upgrades/RangerConfigCalculationTest.java | 2 +- .../RangerKerberosConfigCalculationTest.java | 2 +- .../upgrades/RangerKmsProxyConfigTest.java | 2 +- .../RangerUsersyncConfigCalculationTest.java | 2 +- .../RangerWebAlertConfigActionTest.java | 6 +- .../SparkShufflePropertyConfigTest.java | 2 +- .../UpgradeUserKerberosDescriptorTest.java | 4 +- .../ambari/server/stack/StackManagerTest.java | 8 - .../state/kerberos/KerberosDescriptorTest.java | 56 ++ .../KerberosDescriptorUpdateHelperTest.java | 1 + .../kerberos/VariableReplacementHelperTest.java | 35 +- .../svccomphost/ServiceComponentHostTest.java | 102 ++- .../topology/AsyncCallableServiceTest.java | 89 +-- .../ClusterDeployWithStartOnlyTest.java | 6 + ...InstallWithoutStartOnComponentLevelTest.java | 6 + .../ClusterInstallWithoutStartTest.java | 6 + .../topology/ConfigureClusterTaskTest.java | 64 +- .../server/topology/TopologyManagerTest.java | 14 +- .../server/upgrade/UpgradeCatalog260Test.java | 14 + .../src/test/python/TestAmbariServer.py | 4 +- ambari-server/src/test/python/TestMpacks.py | 12 +- .../src/test/python/TestResourceFilesKeeper.py | 1 + .../src/test/python/TestUpgradeSummary.py | 6 +- .../AMBARI_METRICS/test_service_advisor.py | 596 ++++++++++++++ .../HIVE/test_jdbc_driver_config.py | 18 +- .../RANGER/test_db_flavor_config.py | 17 +- .../RANGER_KMS/test_db_flavor_config.py | 17 +- .../SQOOP/test_jdbc_driver_config.py | 16 +- .../configs/ranger_admin_default.json | 55 ++ .../custom_actions/TestInstallPackages.py | 50 +- .../stacks/2.0.6/HBASE/test_hbase_client.py | 1 - .../stacks/2.0.6/HBASE/test_hbase_master.py | 6 +- .../2.0.6/HBASE/test_phoenix_queryserver.py | 7 + .../python/stacks/2.0.6/HDFS/test_datanode.py | 38 +- .../python/stacks/2.0.6/HDFS/test_namenode.py | 5 +- .../stacks/2.0.6/HIVE/test_hive_metastore.py | 42 +- .../stacks/2.0.6/HIVE/test_hive_server.py | 158 ++-- .../2.0.6/HIVE/test_hive_service_check.py | 4 +- .../stacks/2.0.6/HIVE/test_webhcat_server.py | 41 +- .../stacks/2.0.6/OOZIE/test_oozie_server.py | 9 + .../2.0.6/OOZIE/test_oozie_service_check.py | 5 +- .../stacks/2.0.6/YARN/test_historyserver.py | 5 +- .../stacks/2.0.6/YARN/test_mapreduce2_client.py | 1 + .../stacks/2.0.6/common/test_stack_advisor.py | 576 -------------- .../python/stacks/2.0.6/configs/default.json | 1 + .../hooks/after-INSTALL/test_after_install.py | 88 +-- .../2.0.6/hooks/before-ANY/test_before_any.py | 24 +- .../hooks/before-INSTALL/test_before_install.py | 14 +- .../hooks/before-START/test_before_start.py | 21 +- .../stacks/2.1/FALCON/test_falcon_server.py | 26 +- .../stacks/2.1/HIVE/test_hive_metastore.py | 54 +- .../stacks/2.2/PIG/test_pig_service_check.py | 13 + .../stacks/2.2/SPARK/test_job_history_server.py | 18 +- .../stacks/2.2/common/test_conf_select.py | 13 +- .../stacks/2.2/common/test_stack_advisor.py | 511 ------------ .../2.2/common/test_stack_advisor_perf.py | 66 +- .../stacks/2.3/MAHOUT/test_mahout_client.py | 2 +- .../2.3/MAHOUT/test_mahout_service_check.py | 28 +- .../2.3/SPARK/test_spark_thrift_server.py | 8 +- .../stacks/2.3/common/test_stack_advisor.py | 2 +- .../stacks/2.5/RANGER_KMS/test_kms_server.py | 24 +- .../python/stacks/2.5/SPARK/test_spark_livy.py | 16 +- .../stacks/2.5/ZEPPELIN/test_zeppelin_060.py | 20 +- .../stacks/2.5/configs/ranger-kms-secured.json | 6 +- .../test/python/stacks/2.6/DRUID/test_druid.py | 22 +- .../stacks/2.6/SPARK2/test_spark_livy2.py | 16 +- .../stacks/2.6/ZEPPELIN/test_zeppelin_070.py | 236 +++--- .../stacks/2.6/common/test_stack_advisor.py | 190 ++++- .../src/test/python/stacks/utils/RMFTestCase.py | 43 +- ambari-server/src/test/resources/kerberos.json | 42 + .../resources/stacks/HDP/2.0.8/kerberos.json | 42 - ambari-server/src/test/resources/widgets.json | 95 +++ ambari-web/app/controllers/installer.js | 8 +- .../journalNode/progress_controller.js | 4 +- .../journalNode/step4_controller.js | 6 +- .../nameNode/step5_controller.js | 6 +- .../highAvailability/progress_controller.js | 4 +- .../progress_popup_controller.js | 2 +- .../main/admin/kerberos/step2_controller.js | 7 +- .../main/admin/stack_and_upgrade_controller.js | 4 +- ambari-web/app/controllers/main/host/details.js | 2 - .../controllers/main/service/info/summary.js | 1 + .../main/service/reassign/step4_controller.js | 2 - .../app/controllers/wizard/step6_controller.js | 15 +- .../app/controllers/wizard/step8_controller.js | 15 +- .../app/mappers/repository_version_mapper.js | 2 +- ambari-web/app/messages.js | 8 +- .../app/mixins/common/configs/configs_saver.js | 26 +- .../main/service/configs/config_overridable.js | 1 - .../models/stack_version/repository_version.js | 2 +- ambari-web/app/routes/add_kerberos_routes.js | 22 +- ambari-web/app/styles/alerts.less | 14 +- ambari-web/app/styles/application.less | 7 +- ambari-web/app/styles/bootstrap_overrides.less | 4 + ambari-web/app/styles/dashboard.less | 22 +- ambari-web/app/styles/modal_popups.less | 2 +- ambari-web/app/styles/stack_versions.less | 13 +- .../app/styles/theme/bootstrap-ambari.css | 17 +- ambari-web/app/styles/top-nav.less | 5 +- ambari-web/app/styles/wizard.less | 9 + ambari-web/app/templates/common/breadcrumbs.hbs | 4 +- .../stack_upgrade/stack_upgrade_wizard.hbs | 2 +- .../main/service/info/service_alert_popup.hbs | 6 +- .../app/templates/main/service/info/summary.hbs | 21 +- .../service/info/summary/master_components.hbs | 4 - .../templates/main/service/services/hdfs.hbs | 38 +- ambari-web/app/templates/wizard/step6.hbs | 2 +- ambari-web/app/templates/wizard/step9.hbs | 12 +- ambari-web/app/utils/ajax/ajax.js | 2 - .../stack_upgrade/upgrade_version_box_view.js | 70 +- .../upgrade_version_column_view.js | 2 +- .../admin/stack_upgrade/upgrade_wizard_view.js | 26 +- .../app/views/main/service/info/summary.js | 8 + .../app/views/main/service/services/hdfs.js | 2 - ambari-web/test/controllers/installer_test.js | 37 +- .../journalNode/progress_controller_test.js | 2 - .../progress_controller_test.js | 3 - .../progress_popup_controller_test.js | 4 +- .../admin/kerberos/step2_controller_test.js | 9 +- .../mixins/common/configs/configs_saver_test.js | 28 +- .../stack_version/repository_version_test.js | 12 +- .../upgrade_version_box_view_test.js | 149 ++-- .../upgrade_version_column_view_test.js | 6 + .../stack_upgrade/upgrade_wizard_view_test.js | 38 + .../src/main/assemblies/hdf-ambari-mpack.xml | 1 + .../HIVE/package/scripts/hive_client.py | 2 - .../HIVE/package/scripts/hive_metastore.py | 1 - .../HIVE/package/scripts/hive_server.py | 2 +- .../package/scripts/hive_server_interactive.py | 1 - .../HIVE/package/scripts/webhcat_server.py | 2 - .../scripts/application_timeline_server.py | 1 - .../YARN/package/scripts/historyserver.py | 1 - .../YARN/package/scripts/mapreduce2_client.py | 2 - .../YARN/package/scripts/nodemanager.py | 1 - .../YARN/package/scripts/resourcemanager.py | 1 - .../YARN/package/scripts/yarn_client.py | 1 - .../ui/hive-web/app/adapters/application.js | 18 + .../app/controllers/visualization-ui.js | 10 +- .../ui/hive-web/app/utils/constants.js | 1 + .../savedQueries/SavedQueryResourceManager.java | 17 +- .../resources/ui/app/components/job-item.js | 49 +- .../resources/ui/app/components/jobs-browser.js | 3 + .../ui/app/components/query-result-table.js | 2 +- .../src/main/resources/ui/app/routes/jobs.js | 11 + .../main/resources/ui/app/routes/queries/new.js | 2 + .../resources/ui/app/routes/queries/query.js | 240 ++++-- .../src/main/resources/ui/app/services/jobs.js | 31 +- .../src/main/resources/ui/app/styles/app.scss | 8 + .../ui/app/templates/components/job-item.hbs | 2 +- .../app/templates/components/jobs-browser.hbs | 2 +- .../main/resources/ui/app/templates/jobs.hbs | 1 + .../hive20/src/main/resources/ui/yarn.lock | 2 +- .../apache/ambari/view/utils/hdfs/HdfsApi.java | 2 +- 614 files changed, 13386 insertions(+), 14066 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/assemblies/server.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/agent/StatusCommand.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/agent/StatusCommand.java index a5b3622,9f8a095..5cc5dd5 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/StatusCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/StatusCommand.java @@@ -39,9 -39,9 +39,12 @@@ public class StatusCommand extends Agen @SerializedName("serviceName") private String serviceName; + @SerializedName("serviceType") + private String serviceType; + + @SerializedName("role") + private String role; + @SerializedName("componentName") private String componentName; http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java index 8883ced,46ee65a..c07df43 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java @@@ -134,8 -136,9 +145,10 @@@ public class AmbariMetaInfo private File commonServicesRoot; private File extensionsRoot; private File serverVersionFile; + private File commonWidgetsDescriptorFile; private File customActionRoot; + private File mpacksV2Staging; + private String commonKerberosDescriptorFileLocation; private Map<String, VersionDefinitionXml> versionDefinitions = null; @@@ -243,9 -228,8 +256,11 @@@ customActionRoot = new File(conf.getCustomActionDefinitionPath()); + String mpacksV2StagingPath = conf.getMpacksV2StagingPath(); + mpacksV2Staging = new File(mpacksV2StagingPath); + + commonKerberosDescriptorFileLocation = new File(conf.getResourceDirPath(), KERBEROS_DESCRIPTOR_FILE_NAME).getAbsolutePath(); + commonWidgetsDescriptorFile = new File(conf.getResourceDirPath(), WIDGETS_DESCRIPTOR_FILE_NAME); } /** @@@ -1414,10 -1360,9 +1425,10 @@@ /** * Ensures that the map of version definition files is populated */ - private void ensureVersionDefinitions() { + private synchronized void ensureVersionDefinitions() { if (null != versionDefinitions) { - return; + if(versionDefinitions.size() > 0) + return; } versionDefinitions = new HashMap<>(); @@@ -1519,6 -1446,9 +1530,10 @@@ } return null; + } + + public File getCommonWidgetsDescriptorFile() { + return commonWidgetsDescriptorFile; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java index c56483d,dc97871..a4339ab --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java @@@ -71,9 -72,9 +72,10 @@@ import org.apache.ambari.server.control import org.apache.ambari.server.controller.internal.KerberosDescriptorResourceProvider; import org.apache.ambari.server.controller.internal.MemberResourceProvider; import org.apache.ambari.server.controller.internal.RepositoryVersionResourceProvider; +import org.apache.ambari.server.controller.internal.ServiceGroupResourceProvider; import org.apache.ambari.server.controller.internal.ServiceResourceProvider; import org.apache.ambari.server.controller.internal.UpgradeResourceProvider; + import org.apache.ambari.server.controller.internal.ViewInstanceResourceProvider; import org.apache.ambari.server.controller.logging.LoggingRequestHelperFactory; import org.apache.ambari.server.controller.logging.LoggingRequestHelperFactoryImpl; import org.apache.ambari.server.controller.metrics.MetricPropertyProviderFactory; http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/controller/ResourceProviderFactory.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractControllerResourceProvider.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/DefaultProviderModule.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/HostVersionOutOfSyncListener.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackGenerator.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackGenerator.java index 5d4b832,0000000..130ff62 mode 100644,000000..100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackGenerator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackGenerator.java @@@ -1,561 -1,0 +1,544 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.mpack; + +import java.io.BufferedOutputStream; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.Marshaller; + +import org.apache.ambari.server.stack.ServiceDirectory; +import org.apache.ambari.server.stack.StackDirectory; +import org.apache.ambari.server.stack.StackManager; +import org.apache.ambari.server.state.Mpack; +import org.apache.ambari.server.state.Packlet; +import org.apache.ambari.server.state.PropertyInfo; +import org.apache.ambari.server.state.QuickLinksConfigurationInfo; +import org.apache.ambari.server.state.ServiceInfo; +import org.apache.ambari.server.state.StackId; +import org.apache.ambari.server.state.StackInfo; +import org.apache.ambari.server.state.ThemeInfo; +import org.apache.ambari.server.state.quicklinks.QuickLinks; +import org.apache.ambari.server.state.repository.ManifestServiceInfo; +import org.apache.ambari.server.state.repository.VersionDefinitionXml; +import org.apache.ambari.server.state.stack.ConfigurationXml; +import org.apache.ambari.server.state.stack.RepositoryXml; +import org.apache.ambari.server.state.stack.ServiceMetainfoXml; +import org.apache.ambari.server.state.stack.StackMetainfoXml; +import org.apache.ambari.server.state.stack.StackRoleCommandOrder; +import org.apache.ambari.server.state.theme.Theme; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; +import org.apache.commons.compress.utils.IOUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringUtils; +import org.codehaus.jackson.map.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +/** + * Generate management pack from stack definition + */ +public class MpackGenerator { + private static final Logger LOG = LoggerFactory.getLogger + (MpackGenerator.class); + + private static final ObjectMapper mapper = new ObjectMapper(); + private File commonServicesRoot; + private File stackRoot; + private File mpacksRoot; + private StackId srcStackId; + private StackId dstStackId; + private StackManager stackManager; + private VersionDefinitionXml vdf; + private Gson gson = new GsonBuilder().setPrettyPrinting().create(); + + /** + * {@link MpackGenerator} constructor + * @param stackRoot Stack root directory + * @param commonServicesRoot Common services root directory + * @param srcStackId Source stack id + * @param vdfUrl Version Definition File + * @param mpacksRoot Management pack root directory + * @throws Exception + */ + public MpackGenerator(File stackRoot, File commonServicesRoot, StackId srcStackId, URL vdfUrl, File mpacksRoot) + throws Exception { + this.commonServicesRoot = commonServicesRoot; + this.stackRoot = stackRoot; + this.mpacksRoot = mpacksRoot; + this.srcStackId = srcStackId; + this.stackManager = new StackManager(stackRoot, commonServicesRoot, false); + this.vdf = VersionDefinitionXml.load(vdfUrl); + this.dstStackId = new StackId(vdf.release.stackId); + } + + /** + * Generate management pack + * @throws Exception + */ + public void generateMpack() throws Exception { + String dstStackName = dstStackId.getStackName(); + String version = vdf.release.version; + String build = vdf.release.build; + + System.out.println("==========================================================="); + System.out.println("Source Stack Id: " + srcStackId); + System.out.println("Destination Stack Id: " + dstStackId); + System.out.println("==========================================================="); + + String mpackName = dstStackName.toLowerCase() + "-ambari-mpack" + "-" + version + "-" + build; + File mpackRootDir = new File(mpacksRoot.getAbsolutePath() + File.separator + mpackName); + if (!mpacksRoot.exists()) { + mpacksRoot.mkdirs(); + } + if (mpackRootDir.exists()) { + FileUtils.deleteDirectory(mpackRootDir); + } + mpackRootDir.mkdir(); + + File mpackPackletsDir = new File(mpackRootDir.getAbsolutePath() + File.separator + "packlets"); + if (mpackPackletsDir.exists()) { + mpackPackletsDir.delete(); + } + mpackPackletsDir.mkdir(); + + StackInfo srcStackInfo = stackManager.getStack(srcStackId.getStackName(), srcStackId.getStackVersion()); + StackRoleCommandOrder stackRoleCommandOrder = srcStackInfo.getRoleCommandOrder(); + FileWriter stackRCOFile = new FileWriter( + mpackRootDir.getAbsolutePath() + File.separator + StackDirectory.RCO_FILE_NAME); + mapper.writerWithDefaultPrettyPrinter().writeValue(stackRCOFile, stackRoleCommandOrder.getContent()); + + // Export stack configs + File stackConfigDir = new File( + mpackRootDir.getAbsolutePath() + File.separator + StackDirectory.SERVICE_CONFIG_FOLDER_NAME); + exportConfigs(srcStackInfo.getProperties(), stackConfigDir); + - // Export widgets.json - exportFile(new File(srcStackInfo.getWidgetsDescriptorFileLocation()), mpackRootDir); - - // Export kerberos.json - exportFile(new File(srcStackInfo.getKerberosDescriptorFileLocation()), mpackRootDir); - + // Export repoinfo.xml + RepositoryXml repositoryXml = srcStackInfo.getRepositoryXml(); + JAXBContext ctx = JAXBContext.newInstance(RepositoryXml.class); + Marshaller marshaller = ctx.createMarshaller(); + marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); + File reposDir = new File(mpackRootDir.getAbsolutePath() + File.separator + "repos"); + if (!reposDir.exists()) { + reposDir.mkdir(); + } + FileOutputStream repoXmlFileStream = new FileOutputStream( + reposDir.getAbsolutePath() + File.separator + "repoinfo.xml"); + marshaller.marshal(repositoryXml, repoXmlFileStream); + repoXmlFileStream.flush(); + repoXmlFileStream.close(); + + // Copy JSON cluster property files (example: stacks/HDP/2.0.6/properties/stack_tools.json) + File destPropertiesDir = new File(mpackRootDir.getAbsoluteFile() + File.separator + "properties"); + if(!destPropertiesDir.exists()) { + destPropertiesDir.mkdir(); + } + String srcStackName = srcStackId.getStackName(); + String currentStackVersion = srcStackId.getStackVersion(); + + while (!StringUtils.isEmpty(currentStackVersion)) { + StackInfo currentStackInfo = stackManager.getStack(srcStackName, currentStackVersion); + File srcPropertiesDir = new File(stackRoot.getAbsolutePath() + File.separator + srcStackName + File.separator + currentStackVersion + File.separator + "properties"); + if (srcPropertiesDir.exists() && srcPropertiesDir.isDirectory()) { + for (File srcPropertiesFile : srcPropertiesDir.listFiles()) { + File destPropertiesFile = new File(destPropertiesDir.getAbsolutePath() + File.separator + srcPropertiesFile.getName()); + if (!destPropertiesFile.exists()) { + FileUtils.copyFile(srcPropertiesFile, destPropertiesFile); + } + } + } + currentStackVersion = currentStackInfo.getParentStackVersion(); + } + - // Copy stack hooks folder - String srcStackHooksFolder = srcStackInfo.getStackHooksFolder(); - srcStackHooksFolder = stackRoot.getAbsolutePath() + File.separator + srcStackHooksFolder; - File srcStackHooksFile = new File(srcStackHooksFolder); - if (srcStackHooksFile != null && srcStackHooksFile.exists()) { - File destStackHooksFile = new File( - mpackRootDir.getAbsolutePath() + File.separator - + srcStackHooksFile.getName()); - FileUtils.copyDirectory(srcStackHooksFile, destStackHooksFile); - } - + // Export stack metainfo.xml + String parentStackVersion = srcStackInfo.getParentStackVersion(); + StackMetainfoXml stackMetainfoXml = new StackMetainfoXml(); + stackMetainfoXml.setMinJdk(srcStackInfo.getMinJdk()); + stackMetainfoXml.setMaxJdk(srcStackInfo.getMaxJdk()); + StackMetainfoXml.Version ver = new StackMetainfoXml.Version(); + ver.setActive(srcStackInfo.isActive()); + stackMetainfoXml.setVersion(ver); + ctx = JAXBContext.newInstance(StackMetainfoXml.class); + marshaller = ctx.createMarshaller(); + marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); + FileOutputStream stackMetainfoFileStream = new FileOutputStream( + mpackRootDir.getAbsolutePath() + File.separator + "metainfo.xml"); + marshaller.marshal(stackMetainfoXml, stackMetainfoFileStream); + stackMetainfoFileStream.flush(); + stackMetainfoFileStream.close(); + + // Export stack advisors + File stackAdvisorsDir = new File(mpackRootDir.getAbsolutePath() + File.separator + "stack-advisor"); + if(!stackAdvisorsDir.exists()) { + stackAdvisorsDir.mkdir(); + } + + currentStackVersion = srcStackId.getStackVersion(); + String baseStackAdvisor = null; + String baseStackAdvisorModule = null; + while (!StringUtils.isEmpty(currentStackVersion)) { + // Copy all inherited stack advisors from source stack to "stack-advisor" folder + StackInfo currentStackInfo = stackManager.getStack(srcStackName, currentStackVersion); + File srcStackAdvisor = new File(stackRoot.getAbsolutePath() + File.separator + srcStackName + File.separator + currentStackVersion + File.separator + "services" + File.separator + "stack_advisor.py"); + if(srcStackAdvisor.exists()) { + if(baseStackAdvisor == null) { + baseStackAdvisor = srcStackName.toUpperCase() + currentStackVersion.replace(".", "") + "StackAdvisor"; + baseStackAdvisorModule = "stack_advisor_" + srcStackName.toLowerCase() + currentStackVersion.replace(".", ""); + } + File dstStackAdvisor = new File( + stackAdvisorsDir.getAbsolutePath() + File.separator + "stack_advisor_" + srcStackName.toLowerCase() + + currentStackVersion.replace(".", "") + ".py"); + FileUtils.copyFile(srcStackAdvisor, dstStackAdvisor); + } + currentStackVersion = currentStackInfo.getParentStackVersion(); + } + if(baseStackAdvisor != null) { + File mpackServicesDir = new File(mpackRootDir.getAbsolutePath() + File.separator + "services"); + if (!mpackServicesDir.exists()) { + mpackServicesDir.mkdir(); + } + String mpackStackAdvisorName = dstStackId.getStackName().toUpperCase() + dstStackId.getStackVersion().replace(".", "") + "StackAdvisor"; + if(baseStackAdvisor.equalsIgnoreCase(mpackStackAdvisorName)) { + // Use top level stack advisor from source stack as mpack stack advisor + String srcPath = stackAdvisorsDir.getAbsolutePath() + File.separator + baseStackAdvisorModule + ".py"; + String dstPath = mpackServicesDir.getAbsolutePath() + File.separator + "stack_advisor.py"; + Files.move(Paths.get(srcPath), Paths.get(dstPath)); + } else { + // Create mpack stack advisor that inherits from top level stack advisor from source stack + FileWriter fileWriter = new FileWriter( + mpackServicesDir.getAbsolutePath() + File.separator + "stack_advisor.py"); + BufferedWriter bw = new BufferedWriter(fileWriter); + bw.write("from " + baseStackAdvisorModule + " import *"); + bw.newLine(); + bw.write("class " + mpackStackAdvisorName + "(" + baseStackAdvisor + ")"); + bw.newLine(); + bw.write(" pass"); + bw.newLine(); + bw.flush(); + fileWriter.flush(); + bw.close(); + fileWriter.close(); + } + } + + Mpack mpack = new Mpack(); + mpack.setName(dstStackName); + mpack.setVersion(vdf.release.version); + mpack.setBuildNumber(vdf.release.getFullVersion()); + mpack.setStackId(dstStackId.getStackId()); + mpack.setDescription(dstStackName + " Ambari Management Pack"); + Map<String, String> prereqs = new HashMap<>(); + prereqs.put("min-ambari-version", "3.0.0.0"); + mpack.setPrerequisites(prereqs); + List<Packlet> packlets = new ArrayList<>(); + mpack.setPacklets(packlets); + + for (ManifestServiceInfo manifestServiceInfo : vdf.getStackServices(srcStackInfo, true /* skipMissingServices = true */)) { + ServiceInfo serviceInfo = srcStackInfo.getService(manifestServiceInfo.getName()); + String serviceName = manifestServiceInfo.getName(); + String serviceVersion = (String) manifestServiceInfo.getVersions().toArray()[0]; + ServiceInfo clonedServiceInfo = (ServiceInfo) serviceInfo.clone(); + clonedServiceInfo.setVersion(serviceVersion); + clonedServiceInfo.setParent(null); + if (serviceInfo.getMetricsFile() != null) { + clonedServiceInfo.setMetricsFileName(serviceInfo.getMetricsFile().getName()); + } + if( serviceInfo.getWidgetsDescriptorFile() != null) { + clonedServiceInfo.setWidgetsFileName(serviceInfo.getWidgetsDescriptorFile().getName()); + } + + System.out.println("Processing service=" + serviceInfo.getName() + ", version=" + serviceVersion); + System.out.println("Service Parent : " + serviceInfo.getParent()); + String packletDirName = serviceName + "-packlet-" + serviceVersion; + String packletTarName = packletDirName + ".tar.gz"; + File packletDir = new File( + mpackPackletsDir.getAbsolutePath() + File.separator + packletDirName); + if (!packletDir.exists()) { + packletDir.mkdir(); + } + Packlet packlet = new Packlet(); + packlet.setType(Packlet.PackletType.SERVICE_PACKLET); + packlet.setName(serviceName); + packlet.setVersion(serviceVersion); + packlet.setServiceId(serviceVersion); + packlet.setSourceLocation("packlets" + File.separator + packletTarName); + packlets.add(packlet); + + // Export service metainfo.xml + ServiceMetainfoXml serviceMetainfoXml = new ServiceMetainfoXml(); + serviceMetainfoXml.setSchemaVersion(clonedServiceInfo.getSchemaVersion()); + List<ServiceInfo> serviceInfos = Collections.singletonList(clonedServiceInfo); + serviceMetainfoXml.setServices(serviceInfos); + ctx = JAXBContext.newInstance(ServiceMetainfoXml.class); + marshaller = ctx.createMarshaller(); + marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); + FileOutputStream serviceMetainfoFileStream = new FileOutputStream( + packletDir.getAbsolutePath() + File.separator + "metainfo.xml"); + marshaller.marshal(serviceMetainfoXml, serviceMetainfoFileStream); + serviceMetainfoFileStream.flush(); + serviceMetainfoFileStream.close(); + + // Export mertrics.json + File srcMetricsFile = serviceInfo.getMetricsFile(); + exportFile(srcMetricsFile, packletDir); + + // Export widgets.json + File srcWidgetsFile = serviceInfo.getWidgetsDescriptorFile(); + exportFile(srcWidgetsFile, packletDir); + + // Export alerts.json + File srcAlertsFile = serviceInfo.getAlertsFile(); + exportFile(srcAlertsFile, packletDir); + + // Export kerberos.json + File srcKerberosFile = serviceInfo.getKerberosDescriptorFile(); + exportFile(srcKerberosFile, packletDir); + + // Export quicklinks + for (Map.Entry<String, QuickLinksConfigurationInfo> entry : serviceInfo.getQuickLinksConfigurationsMap() + .entrySet()) { + QuickLinksConfigurationInfo quickLinksConfigurationInfo = entry.getValue(); + String quickLinksFileName = quickLinksConfigurationInfo.getFileName(); + for (Map.Entry<String, QuickLinks> quickLinksEntry : quickLinksConfigurationInfo + .getQuickLinksConfigurationMap().entrySet()) { + File quickLinksDir = new File( + packletDir.getAbsolutePath() + File.separator + serviceInfo + .getQuickLinksConfigurationsDir()); + if (!quickLinksDir.exists()) { + quickLinksDir.mkdir(); + } + FileWriter quickLinksFileWriter = new FileWriter( + quickLinksDir.getAbsolutePath() + File.separator + quickLinksFileName, true); + mapper.writerWithDefaultPrettyPrinter() + .writeValue(quickLinksFileWriter, quickLinksEntry.getValue()); + } + } + + // Export themes + for (Map.Entry<String, ThemeInfo> entry : serviceInfo.getThemesMap().entrySet()) { + ThemeInfo themeInfo = entry.getValue(); + String themeFileName = themeInfo.getFileName(); + for (Map.Entry<String, Theme> themeEntry : themeInfo.getThemeMap().entrySet()) { + File themesDir = new File( + packletDir.getAbsolutePath() + File.separator + serviceInfo.getThemesDir()); + if (!themesDir.exists()) { + themesDir.mkdir(); + } + FileWriter themesFileWriter = new FileWriter( + themesDir.getAbsolutePath() + File.separator + themeFileName, true); + mapper.writerWithDefaultPrettyPrinter().writeValue(themesFileWriter, themeEntry.getValue()); + } + } + + // Export package folder + String srcPackageFolder = serviceInfo.getServicePackageFolder(); + if (srcPackageFolder.startsWith("common-services")) { + srcPackageFolder = srcPackageFolder + .replace("common-services", commonServicesRoot.getAbsolutePath()); + } else { + srcPackageFolder = srcPackageFolder.replace("stacks", stackRoot.getAbsolutePath()); + } + File srcPackageFile = new File(srcPackageFolder); + if (srcPackageFile != null && srcPackageFile.exists()) { + File destPackageFile = new File( + packletDir.getAbsolutePath() + File.separator + + ServiceDirectory.PACKAGE_FOLDER_NAME); + FileUtils.copyDirectory(srcPackageFile, destPackageFile); + } + + // Export merged configs + File configDir = new File( + packletDir.getAbsolutePath() + File.separator + serviceInfo.getConfigDir()); + exportConfigs(serviceInfo.getProperties(), configDir); + + // Copy service advisor + File srcServiceAdvisor = serviceInfo.getAdvisorFile(); + File destServiceAdvisor = new File(packletDir.getAbsolutePath() + File.separator + "service_advisor.py"); + if(srcServiceAdvisor != null && srcServiceAdvisor.exists()) { + FileUtils.copyFile(srcServiceAdvisor, destServiceAdvisor); + } + + // TODO: Export upgrade packs + + // Create packlet tarball + createTarGzip(packletDir.getAbsolutePath()); + if(packletDir.exists()) { + FileUtils.deleteDirectory(packletDir); + } + } + + // Create mpack.json + String mpackFilePath = mpackRootDir.getAbsolutePath() + File.separator + "mpack.json"; + FileWriter mpackFileWriter = new FileWriter(mpackFilePath); + gson.toJson(mpack, Mpack.class, mpackFileWriter); + mpackFileWriter.flush(); + mpackFileWriter.close(); + + // Create mpack tarball + createTarGzip(mpackRootDir.getAbsolutePath()); + if(mpackRootDir.exists()) { + FileUtils.deleteDirectory(mpackRootDir); + } + } + + public static void exportFile(File srcFile, File destRootDir) throws Exception { + if (srcFile != null && srcFile.exists()) { + Path srcPath = Paths.get(srcFile.getAbsolutePath()); + Path destPath = Paths.get( + destRootDir.getAbsolutePath() + File.separator + srcFile.getName()); + Files.copy(srcPath, destPath, StandardCopyOption.COPY_ATTRIBUTES, + StandardCopyOption.REPLACE_EXISTING); + } + } + + public void createTarGzip(String path) throws FileNotFoundException, IOException { + File file = new File(path); + if(!file.exists()) { + throw new FileNotFoundException(path + " does not exist"); + } + String parentDirName = file.getParent(); + String tarGzPath = parentDirName + File.separator + file.getName() + ".tar.gz"; + // Delete existing tarball + File f = new File(tarGzPath); + if(f.exists()) { + f.delete(); + } + FileOutputStream fOut = null; + BufferedOutputStream bOut = null; + GzipCompressorOutputStream gzOut = null; + TarArchiveOutputStream tOut = null; + try { + fOut = new FileOutputStream(new File(tarGzPath)); + bOut = new BufferedOutputStream(fOut); + gzOut = new GzipCompressorOutputStream(bOut); + tOut = new TarArchiveOutputStream(gzOut); + addFileToTarGz(tOut, path, ""); + System.out.println("Compressed " + path + " -> " + tarGzPath); + } finally { + if(tOut != null) { + tOut.finish(); + tOut.close(); + } + if(gzOut != null) { + gzOut.close(); + } + if(bOut != null) { + bOut.close(); + } + if(fOut != null) { + fOut.close(); + } + } + + } + + private void addFileToTarGz(TarArchiveOutputStream tOut, String path, String base) + throws IOException + { + File f = new File(path); + String entryName = base + f.getName(); + TarArchiveEntry tarEntry = new TarArchiveEntry(f, entryName); + tOut.putArchiveEntry(tarEntry); + + if (f.isFile()) { + IOUtils.copy(new FileInputStream(f), tOut); + tOut.closeArchiveEntry(); + } else { + tOut.closeArchiveEntry(); + File[] children = f.listFiles(); + if (children != null) { + for (File child : children) { + addFileToTarGz(tOut, child.getAbsolutePath(), entryName + "/"); + } + } + } + } + + public static void exportConfigs(List<PropertyInfo> properties, File configDir) throws Exception { + if (!configDir.exists()) { + configDir.mkdir(); + } + + Map<String, List<PropertyInfo>> configFilesMap = new HashMap<>(); + for (PropertyInfo propertyInfo : properties) { + String fileName = propertyInfo.getFilename(); + if (!configFilesMap.containsKey(fileName)) { + configFilesMap.put(fileName, new ArrayList<PropertyInfo>()); + } + configFilesMap.get(fileName).add(propertyInfo); + } + + for (Map.Entry<String, List<PropertyInfo>> entry : configFilesMap.entrySet()) { + String fileName = entry.getKey(); + ConfigurationXml configXml = new ConfigurationXml(); + configXml.setProperties(entry.getValue()); + JAXBContext ctx = JAXBContext.newInstance(ConfigurationXml.class); + Marshaller marshaller = ctx.createMarshaller(); + marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); + FileOutputStream configFileStream = new FileOutputStream( + configDir.getAbsolutePath() + File.separator + fileName); + marshaller.marshal(configXml, configFileStream); + configFileStream.flush(); + configFileStream.close(); + } + } + + /** + * Main method for generating mpack + * @param args + * @throws Exception + */ + public static void main(String[] args) throws Exception { + System.out.println("Mpack Generator Started"); + String stackDir = args[0]; + String commonServicesDir = args[1]; + String srcStack = args[2]; + String vdfPath = args[3]; + String mpacksDir = args[4]; + MpackGenerator mpackGenerator = new MpackGenerator( + new File(stackDir), + new File(commonServicesDir), + new StackId(srcStack), + new URL(vdfPath), + new File(mpacksDir)); + mpackGenerator.generateMpack(); + System.out.println("Mpack Generator Finished"); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java index da23ebc,eb6737a..9b740d0 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java @@@ -205,91 -193,6 +210,85 @@@ public class StackManager populateDB(stackDao, extensionDao); } + /*** + * Constructor. Initialize StackManager for merging service definitions and creating management packs + * @param stackRoot + * @param commonServicesRoot + */ + public StackManager(File stackRoot, File commonServicesRoot, boolean validate) throws AmbariException{ + LOG.info("Initializing the stack manager..."); + + if (validate) { + validateStackDirectory(stackRoot); + validateCommonServicesDirectory(commonServicesRoot); + } + + stackMap = new HashMap<>(); + + parseDirectories(stackRoot, commonServicesRoot, null); + + fullyResolveCommonServices(stackModules, commonServiceModules, extensionModules); + fullyResolveExtensions(stackModules, commonServiceModules, extensionModules); + fullyResolveStacks(stackModules, commonServiceModules, extensionModules); + } + + protected void updateArchives( + File resourcesRoot, File stackRoot, Map<String, StackModule> stackModules, Map<String, ServiceModule> commonServiceModules, + Map<String, ExtensionModule> extensionModules ) throws AmbariException { + + LOG.info("Refreshing archives ..."); + + LOG.debug("Refreshing archives for stacks"); + for (StackModule stackModule : stackModules.values()) { + LOG.debug("Refreshing archives for stack : " + stackModule.getId()); - String hooksDir = stackModule.getStackDirectory().getHooksDir(); - if(hooksDir != null) { - LOG.debug("Refreshing archive for stack hooks directory : " + hooksDir); - String hooksAbsolutePath = stackRoot.getAbsolutePath() + File.separator + hooksDir; - ResourceFilesKeeperHelper.updateDirectoryArchive(hooksAbsolutePath, false); - } + for(ServiceModule serviceModule : stackModule.getServiceModules().values()) { + String packageDir = serviceModule.getServiceDirectory().getPackageDir(); + if(packageDir != null) { + LOG.debug("Refreshing archive for stack service package directory : " + packageDir); + String packageAbsoluteDir = resourcesRoot.getAbsolutePath() + File.separator + packageDir; + ResourceFilesKeeperHelper.updateDirectoryArchive(packageAbsoluteDir, false); + } + } + } + + LOG.debug("Refreshing archives for common services"); + for(ServiceModule serviceModule : commonServiceModules.values()) { + String packageDir = serviceModule.getServiceDirectory().getPackageDir(); + if(packageDir != null) { + LOG.debug("Refreshing archive for common service package directory : " + packageDir); + String packageAbsoluteDir = resourcesRoot.getAbsolutePath() + File.separator + packageDir; + ResourceFilesKeeperHelper.updateDirectoryArchive(packageAbsoluteDir, false); + } + } + + LOG.debug("Refreshing archives for extensions"); + for(ExtensionModule extensionModule : extensionModules.values()) { + LOG.debug("Refreshing archives for extension module" + extensionModule.getId()); + for(ServiceModule serviceModule : extensionModule.getServiceModules().values()) { + String packageDir = serviceModule.getServiceDirectory().getPackageDir(); + if(packageDir != null) { + LOG.debug("Refreshing archive for extension service package directory : " + packageDir); + String packageAbsoluteDir = resourcesRoot.getAbsolutePath() + File.separator + packageDir; + ResourceFilesKeeperHelper.updateDirectoryArchive(packageAbsoluteDir, false); + } + } + } + + List<String> miscDirs = new ArrayList<String>() {{ + add(CUSTOM_ACTIONS_DIR); + add(HOST_SCRIPTS_DIR); + add(DASHBOARDS_DIR); + }}; + + LOG.debug("Refreshing archives for misc directories"); + for(String miscDir : miscDirs) { + LOG.debug("Refreshing archive for misc directory : " + miscDir); + String miscAbsolutePath = resourcesRoot.getAbsolutePath() + File.separator + miscDir; + ResourceFilesKeeperHelper.updateDirectoryArchive(miscAbsolutePath, false); + } + LOG.info("Refreshing archives finished!"); + } + protected void parseDirectories(File stackRoot, File commonServicesRoot, File extensionRoot) throws AmbariException { commonServiceModules = parseCommonServicesDirectory(commonServicesRoot); stackModules = parseStackDirectory(stackRoot); http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java index 56d352e,4af83ef..13ceeb9 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java @@@ -64,7 -67,8 +67,9 @@@ import org.apache.ambari.server.state.H import org.apache.ambari.server.state.HostHealthStatus.HealthStatus; import org.apache.ambari.server.state.HostState; import org.apache.ambari.server.state.MaintenanceState; ++import org.apache.ambari.server.state.Service; import org.apache.ambari.server.state.StackId; + import org.apache.ambari.server.state.UpgradeState; import org.apache.ambari.server.state.configgroup.ConfigGroup; import org.apache.ambari.server.state.fsm.InvalidStateTransitionException; import org.apache.ambari.server.state.fsm.SingleArcTransition; @@@ -1168,6 -1172,45 +1173,48 @@@ public class HostImpl implements Host return false; } + + /** + * {@inheritDoc} + */ + @Override + public boolean isRepositoryVersionCorrect(RepositoryVersionEntity repositoryVersion) + throws AmbariException { + HostEntity hostEntity = getHostEntity(); + Collection<HostComponentStateEntity> hostComponentStates = hostEntity.getHostComponentStateEntities(); + + // for every host component, if it matches the desired repo and has reported + // the correct version then we're good + for (HostComponentStateEntity hostComponentState : hostComponentStates) { + ServiceComponentDesiredStateEntity desiredComponmentState = hostComponentState.getServiceComponentDesiredStateEntity(); + RepositoryVersionEntity desiredRepositoryVersion = desiredComponmentState.getDesiredRepositoryVersion(); + ++ Long clusterId = hostComponentState.getClusterId(); ++ Cluster cluster = clusters.getCluster(clusterId); ++ Service s = cluster.getService(hostComponentState.getServiceId()); + ComponentInfo componentInfo = ambariMetaInfo.getComponent( + desiredRepositoryVersion.getStackName(), desiredRepositoryVersion.getStackVersion(), - hostComponentState.getServiceName(), hostComponentState.getComponentName()); ++ s.getName(), hostComponentState.getComponentName()); + + // skip components which don't advertise a version + if (!componentInfo.isVersionAdvertised()) { + continue; + } + + // we only care about checking the specified repo version for this host + if (!repositoryVersion.equals(desiredRepositoryVersion)) { + continue; + } + + String versionAdvertised = hostComponentState.getVersion(); + if (hostComponentState.getUpgradeState() == UpgradeState.IN_PROGRESS + || !StringUtils.equals(versionAdvertised, repositoryVersion.getVersion())) { + return false; + } + } + + return true; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.3.0/service_advisor.py ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/service_advisor.py ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java ---------------------------------------------------------------------- diff --cc ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java index dbe2eed,9285526..8c44632 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java @@@ -73,6 -71,6 +73,7 @@@ import org.apache.ambari.server.state.O import org.apache.ambari.server.state.PropertyDependencyInfo; import org.apache.ambari.server.state.PropertyInfo; import org.apache.ambari.server.state.RepositoryInfo; ++import org.apache.ambari.server.state.ServiceGroup; import org.apache.ambari.server.state.ServiceInfo; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.StackInfo; @@@ -1738,8 -1716,8 +1721,8 @@@ public class AmbariMetaInfoTest RepositoryVersionEntity repositoryVersion = ormHelper.getOrCreateRepositoryVersion( cluster.getCurrentStackVersion(), repoVersion); -- -- cluster.addService("HDFS", repositoryVersion); ++ ServiceGroup sg = cluster.addServiceGroup("core"); ++ cluster.addService(sg, "HDFS", "HDFS", repositoryVersion); metaInfo.reconcileAlertDefinitions(clusters); @@@ -1962,7 -1947,7 +1952,8 @@@ Properties properties = new Properties(); properties.setProperty(Configuration.METADATA_DIR_PATH.getKey(), stackRoot.getPath()); properties.setProperty(Configuration.SERVER_VERSION_FILE.getKey(), versionFile.getPath()); + properties.setProperty(Configuration.MPACKS_V2_STAGING_DIR_PATH.getKey(),"src/test/resources/mpacks-v2"); + properties.setProperty(Configuration.RESOURCES_DIR.getKey(), resourcesRoot.getPath()); Configuration configuration = new Configuration(properties); TestAmbariMetaInfo metaInfo = new TestAmbariMetaInfo(configuration); http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorUpdateHelperTest.java ---------------------------------------------------------------------- diff --cc ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorUpdateHelperTest.java index f7f7ac6,a3d13b1..049fe27 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorUpdateHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorUpdateHelperTest.java @@@ -69,11 -66,10 +69,12 @@@ public class KerberosDescriptorUpdateHe properties.put("common.services.path", "src/main/resources/common-services"); properties.put("server.version.file", "target/version"); properties.put("custom.action.definitions", "/tmp/nofile"); + properties.put("mpacks-v2.staging.path","src/test/resources/mpacks-v2"); + properties.put("resources.dir", "src/main/resources"); Configuration configuration = new Configuration(properties); - install(new FactoryModuleBuilder().build(StackManagerFactory.class)); + install(new FactoryModuleBuilder().implement( + StackManager.class, StackManagerMock.class).build(StackManagerFactory.class)); bind(Clusters.class).toInstance(createNiceMock(Clusters.class)); bind(DBAccessor.class).toInstance(createNiceMock(DBAccessor.class)); http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-web/app/controllers/installer.js ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-web/app/controllers/wizard/step8_controller.js ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-web/app/utils/ajax/ajax.js ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-web/test/controllers/installer_test.js ---------------------------------------------------------------------- diff --cc ambari-web/test/controllers/installer_test.js index 37c2303,65a1e5c..d2d4f2c --- a/ambari-web/test/controllers/installer_test.js +++ b/ambari-web/test/controllers/installer_test.js @@@ -494,27 -468,44 +494,44 @@@ describe('App.InstallerController', fun }; beforeEach(function () { + sinon.spy(checker, 'loadStacks'); - installerController.loadMap['1'][0].callback.call(checker); + installerController.loadMap['step1'][0].callback.call(checker); }); - it('stack info is loaded', function () { - expect(loadStacks).to.be.true; + afterEach(function() { + checker.loadStacks.restore(); + }); + + it('should call loadStacks, stack info not loaded', function () { + expect(checker.loadStacks.calledOnce).to.be.true; }); }); - describe ('Should load stacks async', function() { - var loadStacksVersions = false; + describe('Should load stacks async', function() { var checker = { - loadStacksVersions: function() { - loadStacksVersions = true; - } + loadStacksVersions: Em.K }; + beforeEach(function () { + sinon.spy(checker, 'loadStacksVersions'); + }); + + afterEach(function() { + checker.loadStacksVersions.restore(); + }); + it('stack versions are loaded', function () { - installerController.loadMap['1'][1].callback.call(checker, true).then(function(data){ + installerController.loadMap['step1'][1].callback.call(checker, true).then(function(data){ expect(data).to.be.true; }); - expect(loadStacksVersions).to.be.false; + expect(checker.loadStacksVersions.called).to.be.false; + }); + + it('should call loadStacksVersions, stack versions not loaded', function () { + installerController.loadMap['1'][1].callback.call(checker, false).then(function(data){ + expect(data).to.be.true; + }); + expect(checker.loadStacksVersions.calledOnce).to.be.true; }); }); http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-web/test/mixins/common/configs/configs_saver_test.js ----------------------------------------------------------------------
