[ https://issues.apache.org/jira/browse/AMBARI-18536?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Andrew Onischuk updated AMBARI-18536: ------------------------------------- Resolution: Fixed Status: Resolved (was: Patch Available) Committed to trunk and branch-2.4 and branch-2.5 > Components can't be started because of failed webhdfs request > ------------------------------------------------------------- > > Key: AMBARI-18536 > URL: https://issues.apache.org/jira/browse/AMBARI-18536 > Project: Ambari > Issue Type: Bug > Reporter: Andrew Onischuk > Assignee: Andrew Onischuk > Fix For: 2.5.0 > > Attachments: AMBARI-18536.patch > > > Different components fail to start, for instance: > > > > Traceback (most recent call last):\n File > \"/var/lib/ambari-agent/cache/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py\", > line 182, in <module>\n HistoryServer().execute()\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/script/script.py\", > line 219, in execute\n method(env)\n File > \"/var/lib/ambari-agent/cache/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py\", > line 100, in start\n host_sys_prepped=params.host_sys_prepped)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/functions/copy_tarball.py\", > line 204, in copy_to_hdfs\n > replace_existing_files=replace_existing_files,\n File > \"/usr/lib/python2.6/site-packages/resource_management/core/base.py\", line > 154, in __init__\n self.env.run()\n File > \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", > line 160, in run\n self.run_action(resource, action)\n File > \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", > line 124, in run_action\n provider_action()\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 463, in action_create_on_execute\n self.action_delayed(\"create\")\n > File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 460, in action_delayed\n > self.get_hdfs_resource_executor().action_delayed(action_name, self)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 258, in action_delayed\n self._create_resource()\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 272, in _create_resource\n > self._create_file(self.main_resource.resource.target, > source=self.main_resource.resource.source, mode=self.mode)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 325, in _create_file\n self.util.run_command(target, 'CREATE', > method='PUT', overwrite=True, assertable_result=False, file_to_put=source, > **kwargs)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 182, in run_command\n _, out, err = get_user_call_output(cmd, > user=self.run_user, logoutput=self.logoutput, quiet=False)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/functions/get_user_call_output.py\", > line 61, in get_user_call_output\n raise > Fail(err_msg)\nresource_management.core.exceptions.Fail: Execution of 'curl > -sS -L -w '%{http_code}' -X PUT --data-binary > @/usr/hdp/2.4.3.0-227/hadoop/mapreduce.tar.gz --negotiate -u : > 'http://nat-r6-tzrs-ambari-alerts-1.openstacklocal:50070/webhdfs/v1/hdp/apps/2.4.3.0-227/mapreduce/mapreduce.tar.gz?op=CREATE&user.name=cstm-hdfs&overwrite=True&permission=444' > 1>/tmp/tmpNxSlRa 2>/tmp/tmpXlLRgB' returned 52. curl: (52) Empty reply from > server\n100 > > > > Traceback (most recent call last):\n File > \"/var/lib/ambari-agent/cache/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py\", > line 165, in <module>\n FalconServer().execute()\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/script/script.py\", > line 219, in execute\n method(env)\n File > \"/var/lib/ambari-agent/cache/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py\", > line 46, in start\n self.configure(env, upgrade_type=upgrade_type)\n > File > \"/var/lib/ambari-agent/cache/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py\", > line 41, in configure\n falcon('server', action='config', > upgrade_type=upgrade_type)\n File > \"/usr/lib/python2.6/site-packages/ambari_commons/os_family_impl.py\", line > 89, in thunk\n return fn(*args, **kwargs)\n File > \"/var/lib/ambari-agent/cache/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py\", > line 127, in falcon\n mode = 0777)\n File > \"/usr/lib/python2.6/site-packages/resource_management/core/base.py\", line > 154, in __init__\n self.env.run()\n File > \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", > line 160, in run\n self.run_action(resource, action)\n File > \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", > line 124, in run_action\n provider_action()\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 463, in action_create_on_execute\n self.action_delayed(\"create\")\n > File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 460, in action_delayed\n > self.get_hdfs_resource_executor().action_delayed(action_name, self)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 250, in action_delayed\n self._assert_valid()\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 234, in _assert_valid\n self.target_status = > self._get_file_status(target)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 295, in _get_file_status\n list_status = > self.util.run_command(target, 'GETFILESTATUS', method='GET', > ignore_status_codes=['404'], assertable_result=False)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 182, in run_command\n _, out, err = get_user_call_output(cmd, > user=self.run_user, logoutput=self.logoutput, quiet=False)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/functions/get_user_call_output.py\", > line 61, in get_user_call_output\n raise > Fail(err_msg)\nresource_management.core.exceptions.Fail: Execution of 'curl > -sS -L -w '%{http_code}' -X GET --negotiate -u : > 'http://nat-r6-tzrs-ambari-alerts-1.openstacklocal:50070/webhdfs/v1/apps/falcon?op=GETFILESTATUS&user.name=cstm-hdfs' > 1>/tmp/tmp2adaiW 2>/tmp/tmpEu_LF7' returned 7. curl: (7) couldn't connect to > host\n000 > > > > > Traceback (most recent call last):\n File > \"/var/lib/ambari-agent/cache/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py\", > line 149, in <module>\n HbaseMaster().execute()\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/script/script.py\", > line 219, in execute\n method(env)\n File > \"/var/lib/ambari-agent/cache/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py\", > line 83, in start\n self.configure(env) # for security\n File > \"/var/lib/ambari-agent/cache/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py\", > line 39, in configure\n hbase(name='master')\n File > \"/usr/lib/python2.6/site-packages/ambari_commons/os_family_impl.py\", line > 89, in thunk\n return fn(*args, **kwargs)\n File > \"/var/lib/ambari-agent/cache/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py\", > line 184, in hbase\n owner=params.hbase_user\n File > \"/usr/lib/python2.6/site-packages/resource_management/core/base.py\", line > 154, in __init__\n self.env.run()\n File > \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", > line 160, in run\n self.run_action(resource, action)\n File > \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", > line 124, in run_action\n provider_action()\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 463, in action_create_on_execute\n self.action_delayed(\"create\")\n > File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 460, in action_delayed\n > self.get_hdfs_resource_executor().action_delayed(action_name, self)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 250, in action_delayed\n self._assert_valid()\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 234, in _assert_valid\n self.target_status = > self._get_file_status(target)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 295, in _get_file_status\n list_status = > self.util.run_command(target, 'GETFILESTATUS', method='GET', > ignore_status_codes=['404'], assertable_result=False)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 182, in run_command\n _, out, err = get_user_call_output(cmd, > user=self.run_user, logoutput=self.logoutput, quiet=False)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/functions/get_user_call_output.py\", > line 61, in get_user_call_output\n raise > Fail(err_msg)\nresource_management.core.exceptions.Fail: Execution of 'curl > -sS -L -w '%{http_code}' -X GET --negotiate -u : > 'http://nat-r6-tzrs-ambari-alerts-1.openstacklocal:50070/webhdfs/v1/apps/hbase/data?op=GETFILESTATUS&user.name=cstm-hdfs' > 1>/tmp/tmp7NmmTu 2>/tmp/tmpHeD3Y8' returned 7. curl: (7) couldn't connect to > host\n000 > > > > > Traceback (most recent call last):\n File > \"/var/lib/ambari-agent/cache/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py\", > line 203, in <module>\n HiveServer().execute()\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/script/script.py\", > line 219, in execute\n method(env)\n File > \"/var/lib/ambari-agent/cache/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py\", > line 84, in start\n self.configure(env) # FOR SECURITY\n File > \"/var/lib/ambari-agent/cache/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py\", > line 55, in configure\n hive(name='hiveserver2')\n File > \"/usr/lib/python2.6/site-packages/ambari_commons/os_family_impl.py\", line > 89, in thunk\n return fn(*args, **kwargs)\n File > \"/var/lib/ambari-agent/cache/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py\", > line 124, in hive\n mode=params.hcat_hdfs_user_mode\n File > \"/usr/lib/python2.6/site-packages/resource_management/core/base.py\", line > 154, in __init__\n self.env.run()\n File > \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", > line 160, in run\n self.run_action(resource, action)\n File > \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", > line 124, in run_action\n provider_action()\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 463, in action_create_on_execute\n self.action_delayed(\"create\")\n > File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 460, in action_delayed\n > self.get_hdfs_resource_executor().action_delayed(action_name, self)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 250, in action_delayed\n self._assert_valid()\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 234, in _assert_valid\n self.target_status = > self._get_file_status(target)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 295, in _get_file_status\n list_status = > self.util.run_command(target, 'GETFILESTATUS', method='GET', > ignore_status_codes=['404'], assertable_result=False)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 182, in run_command\n _, out, err = get_user_call_output(cmd, > user=self.run_user, logoutput=self.logoutput, quiet=False)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/functions/get_user_call_output.py\", > line 61, in get_user_call_output\n raise > Fail(err_msg)\nresource_management.core.exceptions.Fail: Execution of 'curl > -sS -L -w '%{http_code}' -X GET --negotiate -u : > 'http://nat-r6-tzrs-ambari-alerts-1.openstacklocal:50070/webhdfs/v1/user/hcat?op=GETFILESTATUS&user.name=cstm-hdfs' > 1>/tmp/tmpQxm4XF 2>/tmp/tmpfD0pbF' returned 7. curl: (7) couldn't connect to > host\n000 > > > > > Traceback (most recent call last):\n File > \"/var/lib/ambari-agent/cache/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py\", > line 210, in <module>\n OozieServer().execute()\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/script/script.py\", > line 219, in execute\n method(env)\n File > \"/var/lib/ambari-agent/cache/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py\", > line 88, in start\n self.configure(env)\n File > \"/var/lib/ambari-agent/cache/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py\", > line 82, in configure\n oozie(is_server=True)\n File > \"/usr/lib/python2.6/site-packages/ambari_commons/os_family_impl.py\", line > 89, in thunk\n return fn(*args, **kwargs)\n File > \"/var/lib/ambari-agent/cache/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py\", > line 99, in oozie\n mode=params.oozie_hdfs_user_mode\n File > \"/usr/lib/python2.6/site-packages/resource_management/core/base.py\", line > 154, in __init__\n self.env.run()\n File > \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", > line 160, in run\n self.run_action(resource, action)\n File > \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", > line 124, in run_action\n provider_action()\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 463, in action_create_on_execute\n self.action_delayed(\"create\")\n > File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 460, in action_delayed\n > self.get_hdfs_resource_executor().action_delayed(action_name, self)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 250, in action_delayed\n self._assert_valid()\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 234, in _assert_valid\n self.target_status = > self._get_file_status(target)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 295, in _get_file_status\n list_status = > self.util.run_command(target, 'GETFILESTATUS', method='GET', > ignore_status_codes=['404'], assertable_result=False)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 182, in run_command\n _, out, err = get_user_call_output(cmd, > user=self.run_user, logoutput=self.logoutput, quiet=False)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/functions/get_user_call_output.py\", > line 61, in get_user_call_output\n raise > Fail(err_msg)\nresource_management.core.exceptions.Fail: Execution of 'curl > -sS -L -w '%{http_code}' -X GET --negotiate -u : > 'http://nat-r6-tzrs-ambari-alerts-1.openstacklocal:50070/webhdfs/v1/user/cstm-oozie?op=GETFILESTATUS&user.name=cstm-hdfs' > 1>/tmp/tmpG5odxb 2>/tmp/tmp9KV3kC' returned 7. curl: (7) couldn't connect to > host\n000 > > > > > Traceback (most recent call last):\n File > \"/var/lib/ambari-agent/cache/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py\", > line 95, in <module>\n JobHistoryServer().execute()\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/script/script.py\", > line 219, in execute\n method(env)\n File > \"/var/lib/ambari-agent/cache/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py\", > line 54, in start\n self.configure(env)\n File > \"/var/lib/ambari-agent/cache/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py\", > line 48, in configure\n setup_spark(env, 'server', > upgrade_type=upgrade_type, action = 'config')\n File > \"/var/lib/ambari-agent/cache/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py\", > line 46, in setup_spark\n mode=0775\n File > \"/usr/lib/python2.6/site-packages/resource_management/core/base.py\", line > 154, in __init__\n self.env.run()\n File > \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", > line 160, in run\n self.run_action(resource, action)\n File > \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", > line 124, in run_action\n provider_action()\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 463, in action_create_on_execute\n self.action_delayed(\"create\")\n > File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 460, in action_delayed\n > self.get_hdfs_resource_executor().action_delayed(action_name, self)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 250, in action_delayed\n self._assert_valid()\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 234, in _assert_valid\n self.target_status = > self._get_file_status(target)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 295, in _get_file_status\n list_status = > self.util.run_command(target, 'GETFILESTATUS', method='GET', > ignore_status_codes=['404'], assertable_result=False)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 182, in run_command\n _, out, err = get_user_call_output(cmd, > user=self.run_user, logoutput=self.logoutput, quiet=False)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/functions/get_user_call_output.py\", > line 61, in get_user_call_output\n raise > Fail(err_msg)\nresource_management.core.exceptions.Fail: Execution of 'curl > -sS -L -w '%{http_code}' -X GET --negotiate -u : > 'http://nat-r6-tzrs-ambari-alerts-1.openstacklocal:50070/webhdfs/v1/user/cstm-spark?op=GETFILESTATUS&user.name=cstm-hdfs' > 1>/tmp/tmpDcu_eG 2>/tmp/tmpdF157D' returned 7. curl: (7) couldn't connect to > host\n000 > > > > > Traceback (most recent call last):\n File > \"/var/lib/ambari-agent/cache/common-services/SPARK/1.2.0.2.2/package/scripts/spark_thrift_server.py\", > line 79, in <module>\n SparkThriftServer().execute()\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/script/script.py\", > line 219, in execute\n method(env)\n File > \"/var/lib/ambari-agent/cache/common-services/SPARK/1.2.0.2.2/package/scripts/spark_thrift_server.py\", > line 53, in start\n self.configure(env)\n File > \"/var/lib/ambari-agent/cache/common-services/SPARK/1.2.0.2.2/package/scripts/spark_thrift_server.py\", > line 47, in configure\n setup_spark(env, 'server', upgrade_type = > upgrade_type, action = 'config')\n File > \"/var/lib/ambari-agent/cache/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py\", > line 46, in setup_spark\n mode=0775\n File > \"/usr/lib/python2.6/site-packages/resource_management/core/base.py\", line > 154, in __init__\n self.env.run()\n File > \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", > line 160, in run\n self.run_action(resource, action)\n File > \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", > line 124, in run_action\n provider_action()\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 463, in action_create_on_execute\n self.action_delayed(\"create\")\n > File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 460, in action_delayed\n > self.get_hdfs_resource_executor().action_delayed(action_name, self)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 250, in action_delayed\n self._assert_valid()\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 234, in _assert_valid\n self.target_status = > self._get_file_status(target)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 295, in _get_file_status\n list_status = > self.util.run_command(target, 'GETFILESTATUS', method='GET', > ignore_status_codes=['404'], assertable_result=False)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", > line 182, in run_command\n _, out, err = get_user_call_output(cmd, > user=self.run_user, logoutput=self.logoutput, quiet=False)\n File > \"/usr/lib/python2.6/site-packages/resource_management/libraries/functions/get_user_call_output.py\", > line 61, in get_user_call_output\n raise > Fail(err_msg)\nresource_management.core.exceptions.Fail: Execution of 'curl > -sS -L -w '%{http_code}' -X GET --negotiate -u : > 'http://nat-r6-tzrs-ambari-alerts-1.openstacklocal:50070/webhdfs/v1/user/cstm-spark?op=GETFILESTATUS&user.name=cstm-hdfs' > 1>/tmp/tmpL8hR95 2>/tmp/tmpwE5KZf' returned 7. curl: (7) couldn't connect to > host\n000 > -- This message was sent by Atlassian JIRA (v6.3.4#6332)