Updated Branches: refs/heads/branch-1.4.1 5fcf8fdc8 -> e5d31fe70
AMBARI-3442. Update upgrade script with latest config mapping. Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/e5d31fe7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/e5d31fe7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/e5d31fe7 Branch: refs/heads/branch-1.4.1 Commit: e5d31fe700ef4a5a3eca09d05630302cbea4e0eb Parents: 5fcf8fd Author: Sumit Mohanty <[email protected]> Authored: Sat Oct 5 01:34:49 2013 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Sat Oct 5 01:36:35 2013 -0700 ---------------------------------------------------------------------- .../src/main/python/UpgradeHelper_HDP2.py | 60 ++++++++++++-------- .../src/test/python/TestUpgradeScript_HDP2.py | 6 +- 2 files changed, 40 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/e5d31fe7/ambari-server/src/main/python/UpgradeHelper_HDP2.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/UpgradeHelper_HDP2.py b/ambari-server/src/main/python/UpgradeHelper_HDP2.py index 752cb20..985400b 100644 --- a/ambari-server/src/main/python/UpgradeHelper_HDP2.py +++ b/ambari-server/src/main/python/UpgradeHelper_HDP2.py @@ -27,7 +27,6 @@ import logging import shutil import json import subprocess -import urllib import time @@ -368,6 +367,7 @@ MAPRED_SITE = { "mapreduce.framework.name": "yarn", "mapreduce.history.server.embedded": "DELETE_OLD", "mapreduce.history.server.http.address": "DELETE_OLD", + "mapreduce.job.jvm.numtasks": "DELETE_OLD", "mapreduce.jobhistory.address": "REPLACE_JH_HOST:10020", "mapreduce.jobhistory.done-dir": "/mr-history/done", "mapreduce.jobhistory.intermediate-done-dir": "/mr-history/tmp", @@ -385,7 +385,6 @@ MAPRED_SITE = { "mapreduce.jobtracker.jobhistory.completed.location": "DELETE_OLD", "mapreduce.jobtracker.jobhistory.location": "DELETE_OLD", "mapreduce.jobtracker.jobhistory.lru.cache.size": "DELETE_OLD", - "mapreduce.jobtracker.staging.root.dir": "DELETE_OLD", "mapreduce.jobtracker.maxmapmemory.mb": "DELETE_OLD", "mapreduce.jobtracker.maxreducememory.mb": "DELETE_OLD", "mapreduce.jobtracker.maxtasks.perjob": "DELETE_OLD", @@ -406,12 +405,18 @@ MAPRED_SITE = { "mapreduce.jobtracker.webinterface.trusted": "DELETE_OLD", "mapreduce.map.java.opts": "-Xmx756m", "mapreduce.map.log.level": "INFO", + "mapreduce.map.memory.mb": "1024", + "mapreduce.map.output.compress": "false", + "mapreduce.map.output.compress.codec": "DELETE_OLD", "mapreduce.map.sort.spill.percent": "0.7", + "mapreduce.output.fileoutputformat.compress": "false", "mapreduce.reduce.input.limit": "DELETE_OLD", "mapreduce.reduce.java.opts": "-Xmx756m", "mapreduce.reduce.log.level": "INFO", + "mapreduce.reduce.memory.mb": "1024", "mapreduce.reduce.merge.inmem.threshold": "DELETE_OLD", "mapreduce.shuffle.port": "13562", + "mapreduce.task.timeout": "300000", "mapreduce.tasktracker.cache.local.size": "DELETE_OLD", "mapreduce.tasktracker.contention.tracking": "DELETE_OLD", "mapreduce.tasktracker.dns.interface": "DELETE_OLD", @@ -434,7 +439,6 @@ MAPRED_SITE = { "mapreduce.tasktracker.reduce.tasks.maximum": "DELETE_OLD", "mapreduce.tasktracker.report.address": "DELETE_OLD", "mapreduce.tasktracker.resourcecalculatorplugin": "DELETE_OLD", - "mapreduce.tasktracker.resourcecalculatorplugin": "DELETE_OLD", "mapreduce.tasktracker.taskcontroller": "DELETE_OLD", "mapreduce.tasktracker.taskmemorymanager.monitoringinterval": "DELETE_OLD", "mapreduce.tasktracker.tasks.sleeptimebeforesigkill": "DELETE_OLD", @@ -446,7 +450,7 @@ MAPRED_SITE = { } GLOBAL = { - "datanode_du_reserved": "1", + "datanode_du_reserved": "1073741824", "dfs_block_local_path_access_user": "DELETE_OLD", "dfs_datanode_data_dir": "REPLACE_WITH_dfs_data_dir", "dfs_exclude": "dfs.exclude", @@ -456,8 +460,11 @@ GLOBAL = { "dfs_namenode_name_dir": "REPLACE_WITH_dfs_name_dir", "fs_checkpoint_size": "0.5", "io_sort_spill_percent": "DELETE_OLD", - "hadoop_conf_dir" : "/etc/hadoop/conf", - "hfile_blockcache_size" : "0.25", + "hadoop_conf_dir": "/etc/hadoop/conf", + "hdfs_support_append": "DELETE_OLD", + "hfile_blockcache_size": "0.25", + "hregion_majorcompaction": "604800000", + "hstore_blockingstorefiles": "10", "jtnode_heapsize": "DELETE_OLD", "jtnode_opt_maxnewsize": "DELETE_OLD", "jtnode_opt_newsize": "DELETE_OLD", @@ -468,18 +475,20 @@ GLOBAL = { "mapred_cluster_red_mem_mb": "DELETE_OLD", "mapred_hosts_exclude": "mapred.exclude", "mapred_hosts_include": "mapred.include", + "mapred_job_map_mem_mb": "DELETE_OLD", + "mapred_job_red_mem_mb": "DELETE_OLD", "mapred_local_dir": "DELETE_OLD", "mapred_log_dir_prefix": "/var/log/hadoop-mapreduce", "mapred_map_tasks_max": "DELETE_OLD", "mapred_pid_dir_prefix": "/var/run/hadoop-mapreduce", "mapred_red_tasks_max": "DELETE_OLD", "mapreduce_jobtracker_system_dir": "REPLACE_WITH_mapred_system_dir", - "mapreduce_map_memory_mb": "REPLACE_WITH_mapred_job_map_mem_mb", - "mapreduce_reduce_memory_mb": "REPLACE_WITH_mapred_job_red_mem_mb", + "mapreduce_map_memory_mb": "DELETE_OLD", + "mapreduce_reduce_memory_mb": "DELETE_OLD", "mapreduce_task_io_sort_mb": "REPLACE_WITH_io_sort_mb", "maxtasks_per_job": "DELETE_OLD", "mapreduce_userlog_retainhours": "DELETE_OLD", - "namenode_opt_maxnewsize" : "640m", + "namenode_opt_maxnewsize": "640m", "nodemanager_heapsize": "1024", "rca_enabled": "DELETE_OLD", "resourcemanager_heapsize": "1024", @@ -489,7 +498,8 @@ GLOBAL = { "yarn_heapsize": "1024", "yarn_log_dir_prefix": "/var/log/hadoop-yarn", "yarn_pid_dir_prefix": "/var/run/hadoop-yarn", - "yarn_user": "yarn" + "yarn_user": "yarn", + "zookeeper_sessiontimeout": "30000" } HDFS_SITE = { @@ -497,6 +507,7 @@ HDFS_SITE = { "dfs.client.read.shortcircuit": "true", "dfs.client.read.shortcircuit.streams.cache.size": "4096", "dfs.datanode.du.pct": "DELETE_OLD", + "dfs.datanode.du.reserved": "1073741824", "dfs.datanode.socket.write.timeout": "DELETE_OLD", "dfs.domain.socket.path": "/var/lib/hadoop-hdfs/dn_socket", "dfs.hosts": "DELETE_OLD", @@ -505,7 +516,8 @@ HDFS_SITE = { "dfs.web.ugi": "DELETE_OLD", "fs.permissions.umask-mode": "022", "ipc.server.max.response.size": "DELETE_OLD", - "ipc.server.read.threadpool.size": "DELETE_OLD" + "ipc.server.read.threadpool.size": "DELETE_OLD", + "dfs.support.append": "true" } CORE_SITE = { @@ -518,8 +530,8 @@ CORE_SITE = { } YARN_SITE = { - "yarn.acl.enable" : "true", - "yarn.admin.acl" : "*", + "yarn.acl.enable": "true", + "yarn.admin.acl": "*", "yarn.application.classpath": "/etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*", "yarn.log-aggregation-enable": "true", "yarn.log-aggregation.retain-seconds": "2592000", @@ -556,17 +568,19 @@ YARN_SITE = { "yarn.scheduler.minimum-allocation-mb": "512" } - HBASE_SITE = { + "dfs.client.read.shortcircuit": "DELETE_OLD", "dfs.support.append": "DELETE_OLD", + "hbase.defaults.for.version.skip": "true", "hbase.hregion.majorcompaction": "604800000", "hbase.hregion.max.filesize": "10737418240", "hbase.hstore.blockingStoreFiles": "10", + "hbase.hstore.flush.retries.number": "120", "hbase.regionserver.global.memstore.lowerLimit": "0.38", "hbase.regionserver.handler.count": "60", "hbase.rpc.engine": "DELETE_OLD", "hfile.block.cache.size": "0.40", - "zookeeper.session.timeout": "30" + "zookeeper.session.timeout": "30000" } @@ -884,7 +898,7 @@ def modify_configs(options, config_type): def rename_all_properties(properties, name_mapping): - for key,val in name_mapping.items(): + for key, val in name_mapping.items(): if (key in properties.keys()) and (val not in properties.keys()): properties[val] = properties[key] del properties[key] @@ -963,12 +977,12 @@ def install_services(options): """{"RequestInfo":{"context":"Install MapReduce2"}, "Body":{"ServiceInfo": {"state":"INSTALLED"}}}"""] err_retcode = 0 err_message = "" - for index in [0,1]: + for index in [0, 1]: response = curl(options.printonly, '-u', - AUTH_FORMAT.format(options.user, options.password), - '-X', 'PUT', '-d', - PUT_IN_INSTALLED[index], - SERVICE_URL_FORMAT.format(options.hostname, options.clustername, SERVICES[index])) + AUTH_FORMAT.format(options.user, options.password), + '-X', 'PUT', '-d', + PUT_IN_INSTALLED[index], + SERVICE_URL_FORMAT.format(options.hostname, options.clustername, SERVICES[index])) retcode, errdata = validate_response(response, not options.printonly) if not retcode == 0: err_retcode = retcode @@ -979,7 +993,7 @@ def install_services(options): raise FatalException(err_retcode, error_msg + "(Services may already be installed or agents are not yet started.)") options.exit_message = "Requests has been submitted to install YARN and MAPREDUCE2. Use Ambari Web to monitor " \ - "the status of the install requests." + "the status of the install requests." pass @@ -1022,7 +1036,7 @@ def curl(print_only, *args): # def main(): parser = optparse.OptionParser(usage="usage: %prog [options] action\n Valid actions: " + VALID_ACTIONS - + "\n update-configs accepts type, e.g. hdfs-site to update specific configs",) + + "\n update-configs accepts type, e.g. hdfs-site to update specific configs", ) parser.add_option("-n", "--printonly", action="store_true", dest="printonly", default=False, http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/e5d31fe7/ambari-server/src/test/python/TestUpgradeScript_HDP2.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestUpgradeScript_HDP2.py b/ambari-server/src/test/python/TestUpgradeScript_HDP2.py index d29fb84..49c81ba 100644 --- a/ambari-server/src/test/python/TestUpgradeScript_HDP2.py +++ b/ambari-server/src/test/python/TestUpgradeScript_HDP2.py @@ -514,12 +514,12 @@ class TestUpgradeHDP2Script(TestCase): return deleted, replaced deleted, replaced = count_tags(UpgradeHelper_HDP2.GLOBAL) - self.assertEqual(8, replaced) - self.assertEqual(19, deleted) + self.assertEqual(6, replaced) + self.assertEqual(24, deleted) deleted, replaced = count_tags(UpgradeHelper_HDP2.MAPRED_SITE) self.assertEqual(0, replaced) - self.assertEqual(83, deleted) + self.assertEqual(85, deleted) deleted, replaced = count_tags(UpgradeHelper_HDP2.CORE_SITE) self.assertEqual(0, replaced)
