KIRY4 commented on issue #10860:
URL: https://github.com/apache/airflow/issues/10860#issuecomment-692101574
Here is full logs from scheduler pod after restarting of scheduler POD.
DAG's wasn't scheduled so it means that this errors producing whole time and
not related to DAG's:
```
*** running scheduler...
____________ _____________
____ |__( )_________ __/__ /________ __
____ /| |_ /__ ___/_ /_ __ /_ __ \_ | /| / /
___ ___ | / _ / _ __/ _ / / /_/ /_ |/ |/ /
_/_/ |_/_/ /_/ /_/ /_/ \____/____/|__/
/home/airflow/.local/lib/python3.6/site-packages/airflow/kubernetes/pod_generator.py:39:
DeprecationWarning: This module is deprecated. Please use
`airflow.kubernetes.pod`.
from airflow.contrib.kubernetes.pod import _extract_volume_mounts
[2020-09-14 14:30:02,552] {__init__.py:50} INFO - Using executor
KubernetesExecutor
[2020-09-14 14:30:02,571] {scheduler_job.py:1367} INFO - Starting the
scheduler
[2020-09-14 14:30:02,572] {scheduler_job.py:1375} INFO - Running execute
loop for -1 seconds
[2020-09-14 14:30:02,573] {scheduler_job.py:1376} INFO - Processing each
file at most -1 times
[2020-09-14 14:30:02,573] {scheduler_job.py:1379} INFO - Searching for files
in /opt/airflow/dags
[2020-09-14 14:30:02,905] {scheduler_job.py:1381} INFO - There are 4 files
in /opt/airflow/dags
[2020-09-14 14:30:02,905] {kubernetes_executor.py:770} INFO - Start
Kubernetes executor
[2020-09-14 14:30:02,958] {kubernetes_executor.py:302} INFO - Event: and now
my watch begins starting at resource_version: 0
[2020-09-14 14:30:02,965] {kubernetes_executor.py:698} INFO - When executor
started up, found 0 queued task instances
[2020-09-14 14:30:02,967] {scheduler_job.py:1438} INFO - Resetting orphaned
tasks for active dag runs
[2020-09-14 14:30:02,980] {dag_processing.py:562} INFO - Launched
DagFileProcessorManager with pid: 81
[2020-09-14 14:30:03,091] {settings.py:55} INFO - Configured default
timezone <Timezone [UTC]>
[2020-09-14 14:31:03,046] {kubernetes_executor.py:290} WARNING - There was a
timeout error accessing the Kube API. Retrying request.
Traceback (most recent call last):
File
"/home/airflow/.local/lib/python3.6/site-packages/urllib3/response.py", line
436, in _error_catcher
yield
File
"/home/airflow/.local/lib/python3.6/site-packages/urllib3/response.py", line
763, in read_chunked
self._update_chunk_length()
File
"/home/airflow/.local/lib/python3.6/site-packages/urllib3/response.py", line
693, in _update_chunk_length
line = self._fp.fp.readline()
File "/usr/local/lib/python3.6/socket.py", line 586, in readinto
return self._sock.recv_into(b)
File "/usr/local/lib/python3.6/ssl.py", line 1012, in recv_into
return self.read(nbytes, buffer)
File "/usr/local/lib/python3.6/ssl.py", line 874, in read
return self._sslobj.read(len, buffer)
File "/usr/local/lib/python3.6/ssl.py", line 631, in read
v = self._sslobj.read(len, buffer)
socket.timeout: The read operation timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File
"/home/airflow/.local/lib/python3.6/site-packages/airflow/executors/kubernetes_executor.py",
line 287, in run
self.worker_uuid, self.kube_config)
File
"/home/airflow/.local/lib/python3.6/site-packages/airflow/executors/kubernetes_executor.py",
line 323, in _run
for event in list_worker_pods():
File
"/home/airflow/.local/lib/python3.6/site-packages/kubernetes/watch/watch.py",
line 144, in stream
for line in iter_resp_lines(resp):
File
"/home/airflow/.local/lib/python3.6/site-packages/kubernetes/watch/watch.py",
line 46, in iter_resp_lines
for seg in resp.read_chunked(decode_content=False):
File
"/home/airflow/.local/lib/python3.6/site-packages/urllib3/response.py", line
792, in read_chunked
self._original_response.close()
File "/usr/local/lib/python3.6/contextlib.py", line 99, in __exit__
self.gen.throw(type, value, traceback)
File
"/home/airflow/.local/lib/python3.6/site-packages/urllib3/response.py", line
441, in _error_catcher
raise ReadTimeoutError(self._pool, None, "Read timed out.")
urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='10.158.0.1',
port=443): Read timed out.
[2020-09-14 14:31:04,050] {kubernetes_executor.py:302} INFO - Event: and now
my watch begins starting at resource_version: 0
[2020-09-14 14:32:04,132] {kubernetes_executor.py:290} WARNING - There was a
timeout error accessing the Kube API. Retrying request.
Traceback (most recent call last):
File
"/home/airflow/.local/lib/python3.6/site-packages/urllib3/response.py", line
436, in _error_catcher
yield
File
"/home/airflow/.local/lib/python3.6/site-packages/urllib3/response.py", line
763, in read_chunked
self._update_chunk_length()
File
"/home/airflow/.local/lib/python3.6/site-packages/urllib3/response.py", line
693, in _update_chunk_length
line = self._fp.fp.readline()
File "/usr/local/lib/python3.6/socket.py", line 586, in readinto
return self._sock.recv_into(b)
File "/usr/local/lib/python3.6/ssl.py", line 1012, in recv_into
return self.read(nbytes, buffer)
File "/usr/local/lib/python3.6/ssl.py", line 874, in read
return self._sslobj.read(len, buffer)
File "/usr/local/lib/python3.6/ssl.py", line 631, in read
v = self._sslobj.read(len, buffer)
socket.timeout: The read operation timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File
"/home/airflow/.local/lib/python3.6/site-packages/airflow/executors/kubernetes_executor.py",
line 287, in run
self.worker_uuid, self.kube_config)
File
"/home/airflow/.local/lib/python3.6/site-packages/airflow/executors/kubernetes_executor.py",
line 323, in _run
for event in list_worker_pods():
File
"/home/airflow/.local/lib/python3.6/site-packages/kubernetes/watch/watch.py",
line 144, in stream
for line in iter_resp_lines(resp):
File
"/home/airflow/.local/lib/python3.6/site-packages/kubernetes/watch/watch.py",
line 46, in iter_resp_lines
for seg in resp.read_chunked(decode_content=False):
File
"/home/airflow/.local/lib/python3.6/site-packages/urllib3/response.py", line
792, in read_chunked
self._original_response.close()
File "/usr/local/lib/python3.6/contextlib.py", line 99, in __exit__
self.gen.throw(type, value, traceback)
File
"/home/airflow/.local/lib/python3.6/site-packages/urllib3/response.py", line
441, in _error_catcher
raise ReadTimeoutError(self._pool, None, "Read timed out.")
urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='10.158.0.1',
port=443): Read timed out.
[2020-09-14 14:32:05,135] {kubernetes_executor.py:302} INFO - Event: and now
my watch begins starting at resource_version: 0
[2020-09-14 14:33:05,217] {kubernetes_executor.py:290} WARNING - There was a
timeout error accessing the Kube API. Retrying request.
Traceback (most recent call last):
File
"/home/airflow/.local/lib/python3.6/site-packages/urllib3/response.py", line
436, in _error_catcher
yield
File
"/home/airflow/.local/lib/python3.6/site-packages/urllib3/response.py", line
763, in read_chunked
self._update_chunk_length()
File
"/home/airflow/.local/lib/python3.6/site-packages/urllib3/response.py", line
693, in _update_chunk_length
line = self._fp.fp.readline()
File "/usr/local/lib/python3.6/socket.py", line 586, in readinto
return self._sock.recv_into(b)
File "/usr/local/lib/python3.6/ssl.py", line 1012, in recv_into
return self.read(nbytes, buffer)
File "/usr/local/lib/python3.6/ssl.py", line 874, in read
return self._sslobj.read(len, buffer)
File "/usr/local/lib/python3.6/ssl.py", line 631, in read
v = self._sslobj.read(len, buffer)
socket.timeout: The read operation timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File
"/home/airflow/.local/lib/python3.6/site-packages/airflow/executors/kubernetes_executor.py",
line 287, in run
self.worker_uuid, self.kube_config)
File
"/home/airflow/.local/lib/python3.6/site-packages/airflow/executors/kubernetes_executor.py",
line 323, in _run
for event in list_worker_pods():
File
"/home/airflow/.local/lib/python3.6/site-packages/kubernetes/watch/watch.py",
line 144, in stream
for line in iter_resp_lines(resp):
File
"/home/airflow/.local/lib/python3.6/site-packages/kubernetes/watch/watch.py",
line 46, in iter_resp_lines
for seg in resp.read_chunked(decode_content=False):
File
"/home/airflow/.local/lib/python3.6/site-packages/urllib3/response.py", line
792, in read_chunked
self._original_response.close()
File "/usr/local/lib/python3.6/contextlib.py", line 99, in __exit__
self.gen.throw(type, value, traceback)
File
"/home/airflow/.local/lib/python3.6/site-packages/urllib3/response.py", line
441, in _error_catcher
raise ReadTimeoutError(self._pool, None, "Read timed out.")
urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='10.158.0.1',
port=443): Read timed out.
[2020-09-14 14:33:06,218] {kubernetes_executor.py:302} INFO - Event: and now
my watch begins starting at resource_version: 0
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
