[
https://issues.apache.org/jira/browse/BEAM-13025?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17444037#comment-17444037
]
Brian Hulette edited comment on BEAM-13025 at 11/15/21, 6:54 PM:
-----------------------------------------------------------------
Looks like this test is passing sometimes on Dataflow V2, but it's flaky now.
Example failure in
https://ci-beam.apache.org/job/beam_PostCommit_Java_DataflowV2/908.
>From Dataflow logs for job 2021-11-15_04_55_22-3569164692454078716:
{code}
Error message from worker: generic::unknown:
org.apache.beam.sdk.util.UserCodeException:
com.google.api.gax.rpc.NotFoundException: io.grpc.StatusRuntimeException:
NOT_FOUND: Resource
projects/844138762903/locations/us-central1-b/subscriptions/beam_it_resource_1932162271320884807
of type SUBSCRIPTION does not exist.
org.apache.beam.sdk.util.UserCodeException.wrap(UserCodeException.java:39)
org.apache.beam.sdk.io.gcp.pubsublite.internal.PerSubscriptionPartitionSdf$DoFnInvoker.invokeNewTracker(Unknown
Source)
org.apache.beam.fn.harness.FnApiDoFnRunner.processElementForWindowObservingSizedElementAndRestriction(FnApiDoFnRunner.java:1041)
org.apache.beam.fn.harness.FnApiDoFnRunner.access$1200(FnApiDoFnRunner.java:142)
org.apache.beam.fn.harness.FnApiDoFnRunner$4.accept(FnApiDoFnRunner.java:640)
org.apache.beam.fn.harness.FnApiDoFnRunner$4.accept(FnApiDoFnRunner.java:635)
org.apache.beam.fn.harness.data.PCollectionConsumerRegistry$MetricTrackingFnDataReceiver.accept(PCollectionConsumerRegistry.java:257)
org.apache.beam.fn.harness.data.PCollectionConsumerRegistry$MetricTrackingFnDataReceiver.accept(PCollectionConsumerRegistry.java:209)
org.apache.beam.fn.harness.BeamFnDataReadRunner.forwardElementToConsumer(BeamFnDataReadRunner.java:172)
org.apache.beam.sdk.fn.data.BeamFnDataInboundObserver2.awaitCompletion(BeamFnDataInboundObserver2.java:126)
org.apache.beam.fn.harness.control.ProcessBundleHandler.processBundle(ProcessBundleHandler.java:467)
org.apache.beam.fn.harness.control.BeamFnControlClient.delegateOnInstructionRequestType(BeamFnControlClient.java:151)
org.apache.beam.fn.harness.control.BeamFnControlClient$InboundObserver.lambda$onNext$0(BeamFnControlClient.java:116)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
Caused by: com.google.api.gax.rpc.NotFoundException:
io.grpc.StatusRuntimeException: NOT_FOUND: Resource
projects/844138762903/locations/us-central1-b/subscriptions/beam_it_resource_1932162271320884807
of type SUBSCRIPTION does not exist.
com.google.api.gax.rpc.ApiExceptionFactory.createException(ApiExceptionFactory.java:45)
com.google.api.gax.grpc.GrpcApiExceptionFactory.create(GrpcApiExceptionFactory.java:72)
com.google.api.gax.grpc.GrpcApiExceptionFactory.create(GrpcApiExceptionFactory.java:60)
com.google.api.gax.grpc.GrpcExceptionCallable$ExceptionTransformingFuture.onFailure(GrpcExceptionCallable.java:97)
com.google.api.core.ApiFutures$1.onFailure(ApiFutures.java:68)
com.google.common.util.concurrent.Futures$CallbackListener.run(Futures.java:1133)
com.google.common.util.concurrent.DirectExecutor.execute(DirectExecutor.java:31)
com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:1277)
com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:1038)
com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:808)
io.grpc.stub.ClientCalls$GrpcFuture.setException(ClientCalls.java:563)
io.grpc.stub.ClientCalls$UnaryStreamToFuture.onClose(ClientCalls.java:533)
io.grpc.internal.DelayedClientCall$DelayedListener$3.run(DelayedClientCall.java:463)
io.grpc.internal.DelayedClientCall$DelayedListener.delayOrExecute(DelayedClientCall.java:427)
io.grpc.internal.DelayedClientCall$DelayedListener.onClose(DelayedClientCall.java:460)
io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:557)
io.grpc.internal.ClientCallImpl.access$300(ClientCallImpl.java:69)
io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInternal(ClientCallImpl.java:738)
io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:717)
io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)
io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:133)
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
java.util.concurrent.FutureTask.run(FutureTask.java:266)
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
Caused by: io.grpc.StatusRuntimeException: NOT_FOUND: Resource
projects/844138762903/locations/us-central1-b/subscriptions/beam_it_resource_1932162271320884807
of type SUBSCRIPTION does not exist.
io.grpc.Status.asRuntimeException(Status.java:535)
io.grpc.stub.ClientCalls$UnaryStreamToFuture.onClose(ClientCalls.java:533)
io.grpc.internal.DelayedClientCall$DelayedListener$3.run(DelayedClientCall.java:463)
io.grpc.internal.DelayedClientCall$DelayedListener.delayOrExecute(DelayedClientCall.java:427)
io.grpc.internal.DelayedClientCall$DelayedListener.onClose(DelayedClientCall.java:460)
io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:557)
io.grpc.internal.ClientCallImpl.access$300(ClientCallImpl.java:69)
io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInternal(ClientCallImpl.java:738)
io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:717)
io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)
io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:133)
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
java.util.concurrent.FutureTask.run(FutureTask.java:266)
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
{code}
Is there some kind of race here? Could you take a look [~dpcollins-google]?
was (Author: bhulette):
Looks like this test is passing sometimes on Dataflow V2, but it's flaky now.
Example failure in
https://ci-beam.apache.org/job/beam_PostCommit_Java_DataflowV2/908.
>From Dataflow logs for job 2021-11-15_04_55_22-3569164692454078716:
{code}
Error message from worker: generic::unknown:
org.apache.beam.sdk.util.UserCodeException:
com.google.api.gax.rpc.NotFoundException: io.grpc.StatusRuntimeException:
NOT_FOUND: Resource
projects/844138762903/locations/us-central1-b/subscriptions/beam_it_resource_1932162271320884807
of type SUBSCRIPTION does not exist.
org.apache.beam.sdk.util.UserCodeException.wrap(UserCodeException.java:39)
org.apache.beam.sdk.io.gcp.pubsublite.internal.PerSubscriptionPartitionSdf$DoFnInvoker.invokeNewTracker(Unknown
Source)
org.apache.beam.fn.harness.FnApiDoFnRunner.processElementForWindowObservingSizedElementAndRestriction(FnApiDoFnRunner.java:1041)
org.apache.beam.fn.harness.FnApiDoFnRunner.access$1200(FnApiDoFnRunner.java:142)
org.apache.beam.fn.harness.FnApiDoFnRunner$4.accept(FnApiDoFnRunner.java:640)
org.apache.beam.fn.harness.FnApiDoFnRunner$4.accept(FnApiDoFnRunner.java:635)
org.apache.beam.fn.harness.data.PCollectionConsumerRegistry$MetricTrackingFnDataReceiver.accept(PCollectionConsumerRegistry.java:257)
org.apache.beam.fn.harness.data.PCollectionConsumerRegistry$MetricTrackingFnDataReceiver.accept(PCollectionConsumerRegistry.java:209)
org.apache.beam.fn.harness.BeamFnDataReadRunner.forwardElementToConsumer(BeamFnDataReadRunner.java:172)
org.apache.beam.sdk.fn.data.BeamFnDataInboundObserver2.awaitCompletion(BeamFnDataInboundObserver2.java:126)
org.apache.beam.fn.harness.control.ProcessBundleHandler.processBundle(ProcessBundleHandler.java:467)
org.apache.beam.fn.harness.control.BeamFnControlClient.delegateOnInstructionRequestType(BeamFnControlClient.java:151)
org.apache.beam.fn.harness.control.BeamFnControlClient$InboundObserver.lambda$onNext$0(BeamFnControlClient.java:116)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
Caused by: com.google.api.gax.rpc.NotFoundException:
io.grpc.StatusRuntimeException: NOT_FOUND: Resource
projects/844138762903/locations/us-central1-b/subscriptions/beam_it_resource_1932162271320884807
of type SUBSCRIPTION does not exist.
com.google.api.gax.rpc.ApiExceptionFactory.createException(ApiExceptionFactory.java:45)
com.google.api.gax.grpc.GrpcApiExceptionFactory.create(GrpcApiExceptionFactory.java:72)
com.google.api.gax.grpc.GrpcApiExceptionFactory.create(GrpcApiExceptionFactory.java:60)
com.google.api.gax.grpc.GrpcExceptionCallable$ExceptionTransformingFuture.onFailure(GrpcExceptionCallable.java:97)
com.google.api.core.ApiFutures$1.onFailure(ApiFutures.java:68)
com.google.common.util.concurrent.Futures$CallbackListener.run(Futures.java:1133)
com.google.common.util.concurrent.DirectExecutor.execute(DirectExecutor.java:31)
com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:1277)
com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:1038)
com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:808)
io.grpc.stub.ClientCalls$GrpcFuture.setException(ClientCalls.java:563)
io.grpc.stub.ClientCalls$UnaryStreamToFuture.onClose(ClientCalls.java:533)
io.grpc.internal.DelayedClientCall$DelayedListener$3.run(DelayedClientCall.java:463)
io.grpc.internal.DelayedClientCall$DelayedListener.delayOrExecute(DelayedClientCall.java:427)
io.grpc.internal.DelayedClientCall$DelayedListener.onClose(DelayedClientCall.java:460)
io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:557)
io.grpc.internal.ClientCallImpl.access$300(ClientCallImpl.java:69)
io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInternal(ClientCallImpl.java:738)
io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:717)
io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)
io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:133)
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
java.util.concurrent.FutureTask.run(FutureTask.java:266)
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
Caused by: io.grpc.StatusRuntimeException: NOT_FOUND: Resource
projects/844138762903/locations/us-central1-b/subscriptions/beam_it_resource_1932162271320884807
of type SUBSCRIPTION does not exist.
io.grpc.Status.asRuntimeException(Status.java:535)
io.grpc.stub.ClientCalls$UnaryStreamToFuture.onClose(ClientCalls.java:533)
io.grpc.internal.DelayedClientCall$DelayedListener$3.run(DelayedClientCall.java:463)
io.grpc.internal.DelayedClientCall$DelayedListener.delayOrExecute(DelayedClientCall.java:427)
io.grpc.internal.DelayedClientCall$DelayedListener.onClose(DelayedClientCall.java:460)
io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:557)
io.grpc.internal.ClientCallImpl.access$300(ClientCallImpl.java:69)
io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInternal(ClientCallImpl.java:738)
io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:717)
io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)
io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:133)
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
java.util.concurrent.FutureTask.run(FutureTask.java:266)
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
{code}
Is there some kind of race here?
> beam_PostCommit_Java_DataflowV2 failing pubsublite.ReadWriteIT
> --------------------------------------------------------------
>
> Key: BEAM-13025
> URL: https://issues.apache.org/jira/browse/BEAM-13025
> Project: Beam
> Issue Type: Bug
> Components: test-failures
> Reporter: Kyle Weaver
> Assignee: Daniel Collins
> Priority: P1
> Labels: currently-failing
> Fix For: Not applicable
>
> Time Spent: 10m
> Remaining Estimate: 0h
>
> [https://ci-beam.apache.org/job/beam_PostCommit_Java_DataflowV2/758/testReport/org.apache.beam.sdk.io.gcp.pubsublite/ReadWriteIT/testReadWrite/]
> java.lang.AssertionError: Did not receive signal on
> projects/apache-beam-testing/subscriptions/result-subscription--5335365384640437489
> in 300s
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
