tvalentyn opened a new issue, #23613:
URL: https://github.com/apache/beam/issues/23613
### What happened?
(from postCommitIT-df-py39)
```
Error Message
concurrent.futures._base.TimeoutError: Operation did not complete within the
designated timeout.
Stacktrace
target = functools.partial(<bound method PollingFuture._done_or_raise of
<google.api_core.operation.Operation object at 0x7f01330a1f10>>)
predicate = <function if_exception_type.<locals>.if_exception_type_predicate
at 0x7f010e26bca0>
sleep_generator = <generator object exponential_sleep_generator at
0x7f0132a917b0>
deadline = 10, on_error = None
def retry_target(target, predicate, sleep_generator, deadline,
on_error=None):
"""Call a function and retry if it fails.
This is the lowest-level retry helper. Generally, you'll use the
higher-level retry helper :class:`Retry`.
Args:
target(Callable): The function to call and retry. This must be a
nullary function - apply arguments with `functools.partial`.
predicate (Callable[Exception]): A callable used to determine if
an
exception raised by the target should be considered
retryable.
It should return True to retry or False otherwise.
sleep_generator (Iterable[float]): An infinite iterator that
determines
how long to sleep between retries.
deadline (float): How long to keep retrying the target. The last
sleep
period is shortened as necessary, so that the last retry
runs at
``deadline`` (and not considerably beyond it).
on_error (Callable[Exception]): A function to call while
processing a
retryable exception. Any error raised by this function will
*not*
be caught.
Returns:
Any: the return value of the target function.
Raises:
google.api_core.RetryError: If the deadline is exceeded while
retrying.
ValueError: If the sleep generator stops yielding values.
Exception: If the target raises a method that isn't retryable.
"""
if deadline is not None:
deadline_datetime = datetime_helpers.utcnow() +
datetime.timedelta(
seconds=deadline
)
else:
deadline_datetime = None
last_exc = None
for sleep in sleep_generator:
try:
> return target()
../../build/gradleenv/-1734967050/lib/python3.9/site-packages/google/api_core/retry.py:190:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
_ _
self = <google.api_core.operation.Operation object at 0x7f01330a1f10>
retry = <google.api_core.retry.Retry object at 0x7f010e280280>
def _done_or_raise(self, retry=DEFAULT_RETRY):
"""Check if the future is done and raise if it's not."""
kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
if not self.done(**kwargs):
> raise _OperationNotComplete()
E google.api_core.future.polling._OperationNotComplete
../../build/gradleenv/-1734967050/lib/python3.9/site-packages/google/api_core/future/polling.py:89:
_OperationNotComplete
The above exception was the direct cause of the following exception:
self = <google.api_core.operation.Operation object at 0x7f01330a1f10>
timeout = 10, retry = <google.api_core.retry.Retry object at 0x7f010e280280>
def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY):
"""Poll and wait for the Future to be resolved.
Args:
timeout (int):
How long (in seconds) to wait for the operation to complete.
If None, wait indefinitely.
"""
if self._result_set:
return
retry_ = self._retry.with_deadline(timeout)
try:
kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
> retry_(self._done_or_raise)(**kwargs)
../../build/gradleenv/-1734967050/lib/python3.9/site-packages/google/api_core/future/polling.py:110:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
_ _
args = (), kwargs = {}
target = functools.partial(<bound method PollingFuture._done_or_raise of
<google.api_core.operation.Operation object at 0x7f01330a1f10>>)
sleep_generator = <generator object exponential_sleep_generator at
0x7f0132a917b0>
@functools.wraps(func)
def retry_wrapped_func(*args, **kwargs):
"""A wrapper that calls target function with retry."""
target = functools.partial(func, *args, **kwargs)
sleep_generator = exponential_sleep_generator(
self._initial, self._maximum, multiplier=self._multiplier
)
> return retry_target(
target,
self._predicate,
sleep_generator,
self._deadline,
on_error=on_error,
)
../../build/gradleenv/-1734967050/lib/python3.9/site-packages/google/api_core/retry.py:283:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
_ _
target = functools.partial(<bound method PollingFuture._done_or_raise of
<google.api_core.operation.Operation object at 0x7f01330a1f10>>)
predicate = <function if_exception_type.<locals>.if_exception_type_predicate
at 0x7f010e26bca0>
sleep_generator = <generator object exponential_sleep_generator at
0x7f0132a917b0>
deadline = 10, on_error = None
def retry_target(target, predicate, sleep_generator, deadline,
on_error=None):
"""Call a function and retry if it fails.
This is the lowest-level retry helper. Generally, you'll use the
higher-level retry helper :class:`Retry`.
Args:
target(Callable): The function to call and retry. This must be a
nullary function - apply arguments with `functools.partial`.
predicate (Callable[Exception]): A callable used to determine if
an
exception raised by the target should be considered
retryable.
It should return True to retry or False otherwise.
sleep_generator (Iterable[float]): An infinite iterator that
determines
how long to sleep between retries.
deadline (float): How long to keep retrying the target. The last
sleep
period is shortened as necessary, so that the last retry
runs at
``deadline`` (and not considerably beyond it).
on_error (Callable[Exception]): A function to call while
processing a
retryable exception. Any error raised by this function will
*not*
be caught.
Returns:
Any: the return value of the target function.
Raises:
google.api_core.RetryError: If the deadline is exceeded while
retrying.
ValueError: If the sleep generator stops yielding values.
Exception: If the target raises a method that isn't retryable.
"""
if deadline is not None:
deadline_datetime = datetime_helpers.utcnow() +
datetime.timedelta(
seconds=deadline
)
else:
deadline_datetime = None
last_exc = None
for sleep in sleep_generator:
try:
return target()
# pylint: disable=broad-except
# This function explicitly must deal with broad exceptions.
except Exception as exc:
if not predicate(exc):
raise
last_exc = exc
if on_error is not None:
on_error(exc)
now = datetime_helpers.utcnow()
if deadline_datetime is not None:
if deadline_datetime <= now:
> raise exceptions.RetryError(
"Deadline of {:.1f}s exceeded while calling target
function".format(
deadline
),
last_exc,
) from last_exc
E google.api_core.exceptions.RetryError: Deadline of 10.0s
exceeded while calling target function, last exception:
../../build/gradleenv/-1734967050/lib/python3.9/site-packages/google/api_core/retry.py:205:
RetryError
During handling of the above exception, another exception occurred:
self = <apache_beam.examples.cookbook.bigtableio_it_test.BigtableIOWriteTest
testMethod=test_bigtable_write>
def setUp(self):
try:
from google.cloud.bigtable import enums
self.STORAGE_TYPE = enums.StorageType.HDD
self.INSTANCE_TYPE = enums.Instance.Type.DEVELOPMENT
except ImportError:
self.STORAGE_TYPE = 2
self.INSTANCE_TYPE = 2
self.test_pipeline = TestPipeline(is_integration_test=True)
self.runner_name = type(self.test_pipeline.runner).__name__
self.project = self.test_pipeline.get_option('project')
self.client = Client(project=self.project, admin=True)
self._delete_old_instances()
self.instance = self.client.instance(
self.instance_id, instance_type=self.INSTANCE_TYPE, labels=LABELS)
if not self.instance.exists():
cluster = self.instance.cluster(
self.cluster_id,
self.LOCATION_ID,
default_storage_type=self.STORAGE_TYPE)
operation = self.instance.create(clusters=[cluster])
> operation.result(timeout=10)
apache_beam/examples/cookbook/bigtableio_it_test.py:144:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
_ _
../../build/gradleenv/-1734967050/lib/python3.9/site-packages/google/api_core/future/polling.py:132:
in result
self._blocking_poll(timeout=timeout, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
_ _
self = <google.api_core.operation.Operation object at 0x7f01330a1f10>
timeout = 10, retry = <google.api_core.retry.Retry object at 0x7f010e280280>
def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY):
"""Poll and wait for the Future to be resolved.
Args:
timeout (int):
How long (in seconds) to wait for the operation to complete.
If None, wait indefinitely.
"""
if self._result_set:
return
retry_ = self._retry.with_deadline(timeout)
try:
kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
retry_(self._done_or_raise)(**kwargs)
except exceptions.RetryError:
> raise concurrent.futures.TimeoutError(
"Operation did not complete within the designated "
"timeout."
)
E concurrent.futures._base.TimeoutError: Operation did not
complete within the designated timeout.
../../build/gradleenv/-1734967050/lib/python3.9/site-packages/google/api_core/future/polling.py:112:
TimeoutError
```
### Issue Priority
Priority: 1
### Issue Component
Component: io-py-gcp
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
