See
<https://ci-beam.apache.org/job/beam_PerformanceTests_xlang_KafkaIO_Python/169/display/redirect?page=changes>
Changes:
[noreply] Structured logging support - Java SDK (#26848)
------------------------------------------
[...truncated 124.02 KB...]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
Caused by: java.io.IOException: KafkaWriter : failed to send 5772 records
(since last report)
at
org.apache.beam.sdk.io.kafka.KafkaWriter.checkForFailures(KafkaWriter.java:149)
at
org.apache.beam.sdk.io.kafka.KafkaWriter.finishBundle(KafkaWriter.java:97)
Caused by: org.apache.kafka.common.errors.UnknownTopicOrPartitionException:
This server does not host this topic-partition.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:17:14.503Z:
JOB_MESSAGE_ERROR: org.apache.beam.sdk.util.UserCodeException:
java.io.IOException: KafkaWriter : failed to send 5180 records (since last
report)
at
org.apache.beam.sdk.util.UserCodeException.wrap(UserCodeException.java:39)
at
org.apache.beam.sdk.io.kafka.KafkaWriter$DoFnInvoker.invokeFinishBundle(Unknown
Source)
at
org.apache.beam.fn.harness.FnApiDoFnRunner.finishBundle(FnApiDoFnRunner.java:1772)
at
org.apache.beam.fn.harness.data.PTransformFunctionRegistry.lambda$register$0(PTransformFunctionRegistry.java:116)
at
org.apache.beam.fn.harness.control.ProcessBundleHandler.processBundle(ProcessBundleHandler.java:560)
at
org.apache.beam.fn.harness.control.BeamFnControlClient.delegateOnInstructionRequestType(BeamFnControlClient.java:151)
at
org.apache.beam.fn.harness.control.BeamFnControlClient$InboundObserver.lambda$onNext$0(BeamFnControlClient.java:116)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
org.apache.beam.sdk.util.UnboundedScheduledExecutorService$ScheduledFutureTask.run(UnboundedScheduledExecutorService.java:163)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
Caused by: java.io.IOException: KafkaWriter : failed to send 5180 records
(since last report)
at
org.apache.beam.sdk.io.kafka.KafkaWriter.checkForFailures(KafkaWriter.java:149)
at
org.apache.beam.sdk.io.kafka.KafkaWriter.finishBundle(KafkaWriter.java:97)
Caused by: org.apache.kafka.common.errors.UnknownTopicOrPartitionException:
This server does not host this topic-partition.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:17:15.222Z:
JOB_MESSAGE_ERROR: org.apache.beam.sdk.util.UserCodeException:
java.io.IOException: KafkaWriter : failed to send 5180 records (since last
report)
at
org.apache.beam.sdk.util.UserCodeException.wrap(UserCodeException.java:39)
at
org.apache.beam.sdk.io.kafka.KafkaWriter$DoFnInvoker.invokeFinishBundle(Unknown
Source)
at
org.apache.beam.fn.harness.FnApiDoFnRunner.finishBundle(FnApiDoFnRunner.java:1772)
at
org.apache.beam.fn.harness.data.PTransformFunctionRegistry.lambda$register$0(PTransformFunctionRegistry.java:116)
at
org.apache.beam.fn.harness.control.ProcessBundleHandler.processBundle(ProcessBundleHandler.java:560)
at
org.apache.beam.fn.harness.control.BeamFnControlClient.delegateOnInstructionRequestType(BeamFnControlClient.java:151)
at
org.apache.beam.fn.harness.control.BeamFnControlClient$InboundObserver.lambda$onNext$0(BeamFnControlClient.java:116)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
org.apache.beam.sdk.util.UnboundedScheduledExecutorService$ScheduledFutureTask.run(UnboundedScheduledExecutorService.java:163)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
Caused by: java.io.IOException: KafkaWriter : failed to send 5180 records
(since last report)
at
org.apache.beam.sdk.io.kafka.KafkaWriter.checkForFailures(KafkaWriter.java:149)
at
org.apache.beam.sdk.io.kafka.KafkaWriter.finishBundle(KafkaWriter.java:97)
Caused by: org.apache.kafka.common.errors.UnknownTopicOrPartitionException:
This server does not host this topic-partition.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:17:15.476Z:
JOB_MESSAGE_ERROR: org.apache.beam.sdk.util.UserCodeException:
java.io.IOException: KafkaWriter : failed to send 5180 records (since last
report)
at
org.apache.beam.sdk.util.UserCodeException.wrap(UserCodeException.java:39)
at
org.apache.beam.sdk.io.kafka.KafkaWriter$DoFnInvoker.invokeFinishBundle(Unknown
Source)
at
org.apache.beam.fn.harness.FnApiDoFnRunner.finishBundle(FnApiDoFnRunner.java:1772)
at
org.apache.beam.fn.harness.data.PTransformFunctionRegistry.lambda$register$0(PTransformFunctionRegistry.java:116)
at
org.apache.beam.fn.harness.control.ProcessBundleHandler.processBundle(ProcessBundleHandler.java:560)
at
org.apache.beam.fn.harness.control.BeamFnControlClient.delegateOnInstructionRequestType(BeamFnControlClient.java:151)
at
org.apache.beam.fn.harness.control.BeamFnControlClient$InboundObserver.lambda$onNext$0(BeamFnControlClient.java:116)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
org.apache.beam.sdk.util.UnboundedScheduledExecutorService$ScheduledFutureTask.run(UnboundedScheduledExecutorService.java:163)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
Caused by: java.io.IOException: KafkaWriter : failed to send 5180 records
(since last report)
at
org.apache.beam.sdk.io.kafka.KafkaWriter.checkForFailures(KafkaWriter.java:149)
at
org.apache.beam.sdk.io.kafka.KafkaWriter.finishBundle(KafkaWriter.java:97)
Caused by: org.apache.kafka.common.errors.UnknownTopicOrPartitionException:
This server does not host this topic-partition.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:17:15.568Z:
JOB_MESSAGE_ERROR: org.apache.beam.sdk.util.UserCodeException:
java.io.IOException: KafkaWriter : failed to send 5180 records (since last
report)
at
org.apache.beam.sdk.util.UserCodeException.wrap(UserCodeException.java:39)
at
org.apache.beam.sdk.io.kafka.KafkaWriter$DoFnInvoker.invokeFinishBundle(Unknown
Source)
at
org.apache.beam.fn.harness.FnApiDoFnRunner.finishBundle(FnApiDoFnRunner.java:1772)
at
org.apache.beam.fn.harness.data.PTransformFunctionRegistry.lambda$register$0(PTransformFunctionRegistry.java:116)
at
org.apache.beam.fn.harness.control.ProcessBundleHandler.processBundle(ProcessBundleHandler.java:560)
at
org.apache.beam.fn.harness.control.BeamFnControlClient.delegateOnInstructionRequestType(BeamFnControlClient.java:151)
at
org.apache.beam.fn.harness.control.BeamFnControlClient$InboundObserver.lambda$onNext$0(BeamFnControlClient.java:116)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
org.apache.beam.sdk.util.UnboundedScheduledExecutorService$ScheduledFutureTask.run(UnboundedScheduledExecutorService.java:163)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
Caused by: java.io.IOException: KafkaWriter : failed to send 5180 records
(since last report)
at
org.apache.beam.sdk.io.kafka.KafkaWriter.checkForFailures(KafkaWriter.java:149)
at
org.apache.beam.sdk.io.kafka.KafkaWriter.finishBundle(KafkaWriter.java:97)
Caused by: org.apache.kafka.common.errors.UnknownTopicOrPartitionException:
This server does not host this topic-partition.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:17:15.641Z:
JOB_MESSAGE_ERROR: org.apache.beam.sdk.util.UserCodeException:
java.io.IOException: KafkaWriter : failed to send 5180 records (since last
report)
at
org.apache.beam.sdk.util.UserCodeException.wrap(UserCodeException.java:39)
at
org.apache.beam.sdk.io.kafka.KafkaWriter$DoFnInvoker.invokeFinishBundle(Unknown
Source)
at
org.apache.beam.fn.harness.FnApiDoFnRunner.finishBundle(FnApiDoFnRunner.java:1772)
at
org.apache.beam.fn.harness.data.PTransformFunctionRegistry.lambda$register$0(PTransformFunctionRegistry.java:116)
at
org.apache.beam.fn.harness.control.ProcessBundleHandler.processBundle(ProcessBundleHandler.java:560)
at
org.apache.beam.fn.harness.control.BeamFnControlClient.delegateOnInstructionRequestType(BeamFnControlClient.java:151)
at
org.apache.beam.fn.harness.control.BeamFnControlClient$InboundObserver.lambda$onNext$0(BeamFnControlClient.java:116)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
org.apache.beam.sdk.util.UnboundedScheduledExecutorService$ScheduledFutureTask.run(UnboundedScheduledExecutorService.java:163)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
Caused by: java.io.IOException: KafkaWriter : failed to send 5180 records
(since last report)
at
org.apache.beam.sdk.io.kafka.KafkaWriter.checkForFailures(KafkaWriter.java:149)
at
org.apache.beam.sdk.io.kafka.KafkaWriter.finishBundle(KafkaWriter.java:97)
Caused by: org.apache.kafka.common.errors.UnknownTopicOrPartitionException:
This server does not host this topic-partition.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:17:40.061Z:
JOB_MESSAGE_BASIC: Finished operation Avoid
Fusion/ReshufflePerKey/GroupByKey/Read+Avoid
Fusion/ReshufflePerKey/GroupByKey/GroupByWindow+Avoid
Fusion/ReshufflePerKey/FlatMap(restore_timestamps)+Avoid
Fusion/RemoveRandomKeys+Measure time+WriteToKafka/Kafka
ProducerRecord/Map/ParMultiDo(Anonymous)+WriteToKafka/KafkaIO.WriteRecords/ParDo(KafkaWriter)/ParMultiDo(KafkaWriter)
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:17:40.101Z:
JOB_MESSAGE_DEBUG: Executing success step success18
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:17:40.158Z:
JOB_MESSAGE_DETAILED: Cleaning up.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:17:40.194Z:
JOB_MESSAGE_DEBUG: Starting **** pool teardown.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:17:40.220Z:
JOB_MESSAGE_BASIC: Stopping **** pool...
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:19:50.908Z:
JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 5 to 0.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:19:50.937Z:
JOB_MESSAGE_BASIC: Worker pool stopped.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:19:50.957Z:
JOB_MESSAGE_DEBUG: Tearing down pending resources...
INFO:apache_beam.runners.dataflow.dataflow_runner:Job
2023-05-26_04_55_07-5367566744610050088 is in state JOB_STATE_DONE
INFO:apache_beam.testing.load_tests.load_test_metrics_utils:Load test results
for test: 75aa80bd23ea4486a1e9257c482d1750 and timestamp: 1685103599.4099026:
INFO:apache_beam.testing.load_tests.load_test_metrics_utils:Metric:
write_count_records_total_messages Value: 100000000
INFO:apache_beam.testing.load_tests.load_test_metrics_utils:Metric:
write_runtime Value: 446
INFO:apache_beam.utils.subprocess_server:Using pre-built snapshot at
<https://ci-beam.apache.org/job/beam_PerformanceTests_xlang_KafkaIO_Python/ws/src/sdks/java/io/expansion-service/build/libs/beam-sdks-java-io-expansion-service-2.49.0-SNAPSHOT.jar>
INFO:root:Starting a JAR-based expansion service from JAR
<https://ci-beam.apache.org/job/beam_PerformanceTests_xlang_KafkaIO_Python/ws/src/sdks/java/io/expansion-service/build/libs/beam-sdks-java-io-expansion-service-2.49.0-SNAPSHOT.jar>
INFO:apache_beam.utils.subprocess_server:Starting service with ['java' '-jar'
'<https://ci-beam.apache.org/job/beam_PerformanceTests_xlang_KafkaIO_Python/ws/src/sdks/java/io/expansion-service/build/libs/beam-sdks-java-io-expansion-service-2.49.0-SNAPSHOT.jar'>
'59541'
'--filesToStage=<https://ci-beam.apache.org/job/beam_PerformanceTests_xlang_KafkaIO_Python/ws/src/sdks/java/io/expansion-service/build/libs/beam-sdks-java-io-expansion-service-2.49.0-SNAPSHOT.jar']>
INFO:apache_beam.utils.subprocess_server:Starting expansion service at
localhost:59541
INFO:apache_beam.utils.subprocess_server:May 26, 2023 12:20:00 PM
org.apache.beam.sdk.expansion.service.ExpansionService loadRegisteredTransforms
INFO:apache_beam.utils.subprocess_server:INFO: Registering external transforms:
[beam:transform:org.apache.beam:kafka_read_with_metadata:v1,
beam:transform:org.apache.beam:kafka_read_without_metadata:v1,
beam:transform:org.apache.beam:kafka_write:v1,
beam:external:java:generate_sequence:v1]
INFO:apache_beam.utils.subprocess_server:
INFO:apache_beam.utils.subprocess_server:Registered transforms:
INFO:apache_beam.utils.subprocess_server:
beam:transform:org.apache.beam:kafka_read_with_metadata:v1:
org.apache.beam.sdk.expansion.service.ExpansionService$ExternalTransformRegistrarLoader$1@4b168fa9
INFO:apache_beam.utils.subprocess_server:
beam:transform:org.apache.beam:kafka_read_without_metadata:v1:
org.apache.beam.sdk.expansion.service.ExpansionService$ExternalTransformRegistrarLoader$1@1a84f40f
INFO:apache_beam.utils.subprocess_server:
beam:transform:org.apache.beam:kafka_write:v1:
org.apache.beam.sdk.expansion.service.ExpansionService$ExternalTransformRegistrarLoader$1@23282c25
INFO:apache_beam.utils.subprocess_server:
beam:external:java:generate_sequence:v1:
org.apache.beam.sdk.expansion.service.ExpansionService$ExternalTransformRegistrarLoader$1@7920ba90
INFO:apache_beam.utils.subprocess_server:
INFO:apache_beam.utils.subprocess_server:Registered SchemaTransformProviders:
INFO:apache_beam.utils.subprocess_server:
beam:schematransform:org.apache.beam:kafka_read:v1
INFO:apache_beam.utils.subprocess_server:
beam:schematransform:org.apache.beam:kafka_write:v1
INFO:apache_beam.utils.subprocess_server:May 26, 2023 12:20:02 PM
org.apache.beam.sdk.expansion.service.ExpansionService expand
INFO:apache_beam.utils.subprocess_server:INFO: Expanding 'ReadFromKafka' with
URN 'beam:transform:org.apache.beam:kafka_read_without_metadata:v1'
INFO:apache_beam.utils.subprocess_server:Dependencies list: {}
INFO:apache_beam.utils.subprocess_server:May 26, 2023 12:20:02 PM
org.apache.beam.sdk.expansion.service.ExpansionService$ExternalTransformRegistrarLoader
payloadToConfig
INFO:apache_beam.utils.subprocess_server:WARNING: Configuration class
'org.apache.beam.sdk.io.kafka.KafkaIO$Read$External$Configuration' has no
schema registered. Attempting to construct with setter approach.
INFO:apache_beam.utils.subprocess_server:May 26, 2023 12:20:03 PM
org.apache.beam.sdk.expansion.service.ExpansionService$ExternalTransformRegistrarLoader
payloadToConfig
INFO:apache_beam.utils.subprocess_server:WARNING: Configuration class
'org.apache.beam.sdk.io.kafka.KafkaIO$Read$External$Configuration' has no
schema registered. Attempting to construct with setter approach.
INFO:apache_beam.runners.dataflow.dataflow_runner:Automatically enabling
Dataflow Runner V2 since the pipeline used cross-language transforms.
INFO:apache_beam.runners.portability.stager:Copying Beam SDK
"<https://ci-beam.apache.org/job/beam_PerformanceTests_xlang_KafkaIO_Python/ws/src/sdks/python/build/apache-beam.tar.gz";>
to staging location.
INFO:apache_beam.runners.dataflow.dataflow_runner:Pipeline has additional
dependencies to be installed in SDK **** container, consider using the SDK
container image pre-building workflow to avoid repetitive installations. Learn
more on
https://cloud.google.com/dataflow/docs/guides/using-custom-containers#prebuild
INFO:root:Default Python SDK image for environment is
apache/beam_python3.7_sdk:2.49.0.dev
INFO:root:Using provided Python SDK container image:
gcr.io/cloud-dataflow/v1beta3/beam_python3.7_sdk:beam-master-20230422
INFO:root:Python SDK container image set to
"gcr.io/cloud-dataflow/v1beta3/beam_python3.7_sdk:beam-master-20230422" for
Docker environment
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function pack_combiners at 0x7fcdc445f7a0> ====================
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function sort_stages at 0x7fcdc445ff80> ====================
INFO:apache_beam.runners.dataflow.internal.apiclient:Defaulting to the
temp_location as staging_location: gs://temp-storage-for-perf-tests/
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to
gs://temp-storage-for-perf-tests/beamapp-jenkins-0526122006-462712-sqao7fda.1685103606.463214/beam-sdks-java-io-expansion-service-2.49.0-SNAPSHOT-Hefrfye3E2TGt6_Mhc1Ir9oafeZNhCuZPQZUFr0ONeA.jar...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to
gs://temp-storage-for-perf-tests/beamapp-jenkins-0526122006-462712-sqao7fda.1685103606.463214/beam-sdks-java-io-expansion-service-2.49.0-SNAPSHOT-Hefrfye3E2TGt6_Mhc1Ir9oafeZNhCuZPQZUFr0ONeA.jar
in 2 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to
gs://temp-storage-for-perf-tests/beamapp-jenkins-0526122006-462712-sqao7fda.1685103606.463214/dataflow_python_sdk.tar...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to
gs://temp-storage-for-perf-tests/beamapp-jenkins-0526122006-462712-sqao7fda.1685103606.463214/dataflow_python_sdk.tar
in 0 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to
gs://temp-storage-for-perf-tests/beamapp-jenkins-0526122006-462712-sqao7fda.1685103606.463214/pipeline.pb...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to
gs://temp-storage-for-perf-tests/beamapp-jenkins-0526122006-462712-sqao7fda.1685103606.463214/pipeline.pb
in 0 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Create job: <Job
clientRequestId: '20230526122006464667-8394'
createTime: '2023-05-26T12:20:10.194467Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2023-05-26_05_20_09-9916593403019600062'
location: 'us-central1'
name: 'beamapp-jenkins-0526122006-462712-sqao7fda'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2023-05-26T12:20:10.194467Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
INFO:apache_beam.runners.dataflow.internal.apiclient:Created job with id:
[2023-05-26_05_20_09-9916593403019600062]
INFO:apache_beam.runners.dataflow.internal.apiclient:Submitted job:
2023-05-26_05_20_09-9916593403019600062
INFO:apache_beam.runners.dataflow.internal.apiclient:To access the Dataflow
monitoring console, please navigate to
https://console.cloud.google.com/dataflow/jobs/us-central1/2023-05-26_05_20_09-9916593403019600062?project=apache-beam-testing
INFO:apache_beam.runners.dataflow.dataflow_runner:Job
2023-05-26_05_20_09-9916593403019600062 is in state JOB_STATE_PENDING
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:20:12.412Z:
JOB_MESSAGE_BASIC: Worker configuration: e2-standard-2 in us-central1-f.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:20:13.543Z:
JOB_MESSAGE_DETAILED: Expanding SplittableParDo operations into optimizable
parts.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:20:13.574Z:
JOB_MESSAGE_DETAILED: Expanding CollectionToSingleton operations into
optimizable parts.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:20:13.637Z:
JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:20:13.677Z:
JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:20:13.705Z:
JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:20:13.742Z:
JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:20:13.786Z:
JOB_MESSAGE_DETAILED: Fusing consumer
ReadFromKafka/KafkaIO.Read/KafkaIO.Read.ReadFromKafkaViaSDF/ParDo(GenerateKafkaSourceDescriptor)/ParMultiDo(GenerateKafkaSourceDescriptor)
into ReadFromKafka/KafkaIO.Read/KafkaIO.Read.ReadFromKafkaViaSDF/Impulse
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:20:13.816Z:
JOB_MESSAGE_DETAILED: Fusing consumer
external_2ReadFromKafka-KafkaIO-Read-KafkaIO-Read-ReadFromKafkaViaSDF-KafkaIO-ReadSourceDescriptors-ParDo-Unbo/PairWithRestriction
into
ReadFromKafka/KafkaIO.Read/KafkaIO.Read.ReadFromKafkaViaSDF/ParDo(GenerateKafkaSourceDescriptor)/ParMultiDo(GenerateKafkaSourceDescriptor)
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:20:13.848Z:
JOB_MESSAGE_DETAILED: Fusing consumer
external_2ReadFromKafka-KafkaIO-Read-KafkaIO-Read-ReadFromKafkaViaSDF-KafkaIO-ReadSourceDescriptors-ParDo-Unbo/SplitWithSizing
into
external_2ReadFromKafka-KafkaIO-Read-KafkaIO-Read-ReadFromKafkaViaSDF-KafkaIO-ReadSourceDescriptors-ParDo-Unbo/PairWithRestriction
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:20:13.880Z:
JOB_MESSAGE_DETAILED: Fusing consumer
ReadFromKafka/KafkaIO.Read/KafkaIO.Read.ReadFromKafkaViaSDF/KafkaIO.ReadSourceDescriptors/MapElements/Map/ParMultiDo(Anonymous)
into
external_2ReadFromKafka-KafkaIO-Read-KafkaIO-Read-ReadFromKafkaViaSDF-KafkaIO-ReadSourceDescriptors-ParDo-Unbo/ProcessElementAndRestrictionWithSizing
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:20:13.913Z:
JOB_MESSAGE_DETAILED: Fusing consumer ReadFromKafka/Remove Kafka
Metadata/ParMultiDo(Anonymous) into
ReadFromKafka/KafkaIO.Read/KafkaIO.Read.ReadFromKafkaViaSDF/KafkaIO.ReadSourceDescriptors/MapElements/Map/ParMultiDo(Anonymous)
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:20:13.944Z:
JOB_MESSAGE_DETAILED: Fusing consumer Count records into ReadFromKafka/Remove
Kafka Metadata/ParMultiDo(Anonymous)
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:20:13.979Z:
JOB_MESSAGE_DETAILED: Fusing consumer Measure time into Count records
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:20:14.022Z:
JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:20:14.057Z:
JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:20:14.090Z:
JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:20:14.122Z:
JOB_MESSAGE_DEBUG: Assigning stage ids.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:20:14.249Z:
JOB_MESSAGE_DEBUG: Executing wait step start10
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:20:14.315Z:
JOB_MESSAGE_BASIC: Executing operation
ReadFromKafka/KafkaIO.Read/KafkaIO.Read.ReadFromKafkaViaSDF/Impulse+ReadFromKafka/KafkaIO.Read/KafkaIO.Read.ReadFromKafkaViaSDF/ParDo(GenerateKafkaSourceDescriptor)/ParMultiDo(GenerateKafkaSourceDescriptor)+external_2ReadFromKafka-KafkaIO-Read-KafkaIO-Read-ReadFromKafkaViaSDF-KafkaIO-ReadSourceDescriptors-ParDo-Unbo/PairWithRestriction+external_2ReadFromKafka-KafkaIO-Read-KafkaIO-Read-ReadFromKafkaViaSDF-KafkaIO-ReadSourceDescriptors-ParDo-Unbo/SplitWithSizing
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:20:14.359Z:
JOB_MESSAGE_DEBUG: Starting **** pool setup.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:20:14.390Z:
JOB_MESSAGE_BASIC: Starting 5 ****s in us-central1-f...
INFO:apache_beam.runners.dataflow.dataflow_runner:Job
2023-05-26_05_20_09-9916593403019600062 is in state JOB_STATE_RUNNING
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:20:23.172Z:
JOB_MESSAGE_BASIC: Your project already contains 100 Dataflow-created metric
descriptors, so new user metrics of the form custom.googleapis.com/* will not
be created. However, all user metrics are also available in the metric
dataflow.googleapis.com/job/user_counter. If you rely on the custom metrics,
you can delete old / unused metric descriptors. See
https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.list
and
https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.delete
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:20:53.314Z:
JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 5 based on the
rate of progress in the currently running stage(s).
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:22:42.370Z:
JOB_MESSAGE_DETAILED: Workers have started successfully.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:23:43.955Z:
JOB_MESSAGE_DETAILED: All ****s have finished the startup processes and began
to receive work requests.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:23:47.130Z:
JOB_MESSAGE_BASIC: Finished operation
ReadFromKafka/KafkaIO.Read/KafkaIO.Read.ReadFromKafkaViaSDF/Impulse+ReadFromKafka/KafkaIO.Read/KafkaIO.Read.ReadFromKafkaViaSDF/ParDo(GenerateKafkaSourceDescriptor)/ParMultiDo(GenerateKafkaSourceDescriptor)+external_2ReadFromKafka-KafkaIO-Read-KafkaIO-Read-ReadFromKafkaViaSDF-KafkaIO-ReadSourceDescriptors-ParDo-Unbo/PairWithRestriction+external_2ReadFromKafka-KafkaIO-Read-KafkaIO-Read-ReadFromKafkaViaSDF-KafkaIO-ReadSourceDescriptors-ParDo-Unbo/SplitWithSizing
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:23:47.180Z:
JOB_MESSAGE_DEBUG: Value
"external_2ReadFromKafka-KafkaIO-Read-KafkaIO-Read-ReadFromKafkaViaSDF-KafkaIO-ReadSourceDescriptors-ParDo-Unbo-split-with-sizing-out3"
materialized.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-05-26T12:23:47.225Z:
JOB_MESSAGE_BASIC: Executing operation
external_2ReadFromKafka-KafkaIO-Read-KafkaIO-Read-ReadFromKafkaViaSDF-KafkaIO-ReadSourceDescriptors-ParDo-Unbo/ProcessElementAndRestrictionWithSizing+ReadFromKafka/KafkaIO.Read/KafkaIO.Read.ReadFromKafkaViaSDF/KafkaIO.ReadSourceDescriptors/MapElements/Map/ParMultiDo(Anonymous)+ReadFromKafka/Remove
Kafka Metadata/ParMultiDo(Anonymous)+Count records+Measure time
WARNING:apache_beam.runners.dataflow.dataflow_runner:Timing out on waiting for
job 2023-05-26_05_20_09-9916593403019600062 after 1500 seconds
INFO:apache_beam.testing.load_tests.load_test_metrics_utils:Load test results
for test: 179b569f37fe445e99d08f767f4bafe6 and timestamp: 1685105111.7392817:
INFO:apache_beam.testing.load_tests.load_test_metrics_utils:Metric:
read_count_records_total_messages Value: 103697774
INFO:apache_beam.testing.load_tests.load_test_metrics_utils:Metric:
read_runtime Value: 249
Traceback (most recent call last):
File "/usr/lib/python3.7/runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "/usr/lib/python3.7/runpy.py", line 85, in _run_code
exec(code, run_globals)
File
"<https://ci-beam.apache.org/job/beam_PerformanceTests_xlang_KafkaIO_Python/ws/src/sdks/python/apache_beam/io/external/xlang_kafkaio_perf_test.py";,>
line 139, in <module>
getattr(sys.modules[__name__], test_options.test_class)().run()
File
"<https://ci-beam.apache.org/job/beam_PerformanceTests_xlang_KafkaIO_Python/ws/src/sdks/python/apache_beam/io/external/xlang_kafkaio_perf_test.py";,>
line 64, in run
read_test.run()
File
"<https://ci-beam.apache.org/job/beam_PerformanceTests_xlang_KafkaIO_Python/ws/src/sdks/python/apache_beam/testing/load_tests/load_test.py";,>
line 156, in run
self.cleanup()
File
"<https://ci-beam.apache.org/job/beam_PerformanceTests_xlang_KafkaIO_Python/ws/src/sdks/python/apache_beam/io/external/xlang_kafkaio_perf_test.py";,>
line 121, in cleanup
assert total_messages == self.input_options['num_records']
AssertionError
> Task :sdks:python:apache_beam:testing:load_tests:run FAILED
FAILURE: Build failed with an exception.
* Where:
Build file
'<https://ci-beam.apache.org/job/beam_PerformanceTests_xlang_KafkaIO_Python/ws/src/sdks/python/apache_beam/testing/load_tests/build.gradle'>
line: 63
* What went wrong:
Execution failed for task ':sdks:python:apache_beam:testing:load_tests:run'.
> error occurred
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with
Gradle 8.0.
You can use '--warning-mode all' to show the individual deprecation warnings
and determine if they come from your own scripts or plugins.
See
https://docs.gradle.org/7.5.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 51m 50s
15 actionable tasks: 5 executed, 10 up-to-date
Publishing build scan...
https://ge.apache.org/s/osszsfgmtexbw
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
