See
<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/1398/display/redirect?page=changes>
Changes:
[nielm] Remove spurious error message.
[alxavier] [BEAM-9911]Replace SpannerIO.write latency counter to distribution
[kcweaver] [BEAM-4782] Remove workaround in Python multimap tests.
[kamil.wasilewski] [BEAM-8132, BEAM-8133] Apply InfluxDB pipeline options in
Load Tests and
[kamil.wasilewski] [BEAM-8132, BEAM-8133] Assume no trailing slash after a
hostname
[iemejia] [BEAM-2530] Compile and run tests on java 11 for Precommit portability
[github] [BEAM-9883] Refactor SDF test restrictions. (#11605)
------------------------------------------
[...truncated 152.41 KB...]
},
"python_side_input2-write/Write/WriteImpl/FinalizeWrite": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s20"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type":
"FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": [
{
"@type":
"FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": [],
"pipeline_proto_coder_id":
"ref_Coder_FastPrimitivesCoder_5"
},
{
"@type":
"FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": [],
"pipeline_proto_coder_id":
"ref_Coder_FastPrimitivesCoder_5"
}
],
"is_pair_like": true,
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "None",
"user_name": "write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn":
"eNrNWHd8HEcVvjvJbe04LsRgUriYGE4hd2c7sWKbkJCcLcccPisrJV6K2eztzt2ste3NzEpW0CZxHJU4AQKE0EwvoXcIvfeS0DEdQif0Gqp5M3sn+YQs/J/5Sdrdezvv23nve+99e7qxq2BbkWVTYtaJ5ZcEswLeCJnPS3bIiFaxPM+qe2Qvs6KIsO1hX6BBpucgZBPIFYyuTCZjNgLo6gBxQ/ytW5xoZsMNLM+9jpgjzBVEg25jmXKJEZY1eURsWFAzFqEtYqFNOIeFtNtYKteI0YiY1A0Eh0W243peyZRHzbQZsQQxA8snjogjD2EXK5ddewbR50rlskRZ3CCKhULioKlHh7GYMS01FkoTc5tuAMsm4TQjj5878oHr1B5KDsGEWCJkHJZPwuk6rKjV9sHKSVilw+q4vg8eVJidBEEOCDfUzEE8D7jBkAZnYObWJPDggrECH9RwPWJGlqBmxEjDPQAP6UAII/QOeGnY8mJcx8Jh1yFMGxCWcO1rpLG/bYO1CPzQBM4sGBoCpx5y73BWR+q80HKUXYOzFXlcMDhnHB6mQ95Y0PaEc409eN3k28plQfyoyDFsq0mKWBhFEjhFEaYnwgUvR6NFVxRtL4ydcprd8sbNW7Zs7d20dePFF/VuKTPCYw8pWRfXZ8KW7Jk8bsiwH07l9s9LYH2BnkXz9FwDV2XgEbheRhPEvsmpxRwOj6xm1DbtEKOGzoQrG2+dtMHwilHcXyVU6elB/PMTeFTdWCmDlmDpDmSpWwIuMM5De3F9oXUr9ns2bHaKYQNNM4+XNigaazohml5Yb+OU1M3i+XN6llX0dugj25wjtSlDG4xuWXSxCGGjsQQvfddvkbdJxS/LqBx5FtbohcZa2RqyFwMkklmu5wZNMyAjeMaCvmhK9ZJJiSUTtLkW1zuLMhCEYUeWYuF62uWsGfskEP2eZRMaeipVvZiqi6k8bqlm1XkrSWDbPnh0weiVfToqKG6dY92Zqr02FFVvl/fOHHf5kVfua7W+ssAlHfuIVJ1p5tVB5NpDHnEGEG6XRNPgMQlcWlA5cSxhwWVzOU4v345LNHgs7vLyBK4oGMtlKm05SGRjyWChYpyJRum9bWbT23BPTA62bcMbYbuqihE3cMIR08fkypziVNtxotEocRQM1xT9lmf+l7cGfYqLYZeMSLCdc82wRhzYssk1uLJAzzYW43pZsXKMwa5xeJwO1UI1W83gX1d1dUWbyoxlprKHcgMZeHxtHHb3KJd2LICmPQZDS5mGPinvJ8GQG/D2ucg9a5iUR0I2xDEuUpZhmf2EqeINbDIo+9ncGzKnEsaB2DVo9o9e2FvmzC5zZ0h2uqS+fFxSyikfpWgU+tVWLvEsv+5Yl8JVuw9nKxnQjTNkx7PQNxlCysqe3u2AGsgqP61RB4MTcHWPgGt02NuRriYRJtKJ5Wmox9Rj1xMYEjxB5Rhvy7vwxAl4kg5P7nB1/ShkwvRDJ5ZKsc9YNWvEpyHAU8bB1OFaBW+iry1ME6wJqOtg06tqc7FnE/wADkXukKYc0tRVXVStVK4VGZHdnxvLOEjVoWySG8vxdWPZ/V1OTnQfykj6xIIxvON0DeVY71jO6V6bQdtCZ0FqF4tSW+uqS141utbg8aasgyuBFGrVnArdIQ0Lpys0ZJkY16OlvxJ6HlFllQ8beY4tmF/v5EdcQfM+anpeUAvvBCRPPCLbP582DHHyFs9b6BA0PSLQW1JTyve5jIu8GAnb63meBLYsEMKkDyKuW8/XXaCOJWgKoGn7ei4X4KqRJjkXYehx2K8k1+WeaxMYUhqE7IGnOneHH4nR6d4GX932SACBGoRK83YwFjII6TkCIiOnsAFUKtpFxNTT5QgEPgmC9quhbM5dhPHuB7KVFZncimx3dnl2WXZxNpfN5WC4B8twRIcD1KZxrZbAaE3AdTo8dRzGdEiok8D1hROMjRto33EideM8IrVX+RJHBdaSqoM4y25K4FDBWC3R1XuXY6aKniLePA9in8VFP3N9V7jD0+o3jpATCUwap6s84Ctd3bKHWmhT86D1q+HcQrkFUQ4ncGud1xWQzCMXlh+1gG6bT43ba1tYT0OspyfwjLoaAa0UpjC3zwOzU83aNGstpGci0rMSeLYqqzRNcMeJ5nb6oA4YDZ6DCHcm8FxVNSbqKjyvw19KJS9NRzsTiwbPR9cXJPBC9XDftVnI4cjhxUePFO67/9ixG3hd/sT1eBxepMOLJ+AlCbwU6+hlOrw8rtMdtI8WqOT7FQm8skBvppKoVyXwajpFZbbvSuA1CEBvozJfr03gdXV6O5URvz6BN9A76J0KfRLeOLcybzwpZX4TRcV9c4FeRqWMviWBtxbo/9bMt6nt72zNvlSistWBSveYnE9vRyV6Rw/F4ztPlR69q1OP7t69Lkd12dXv1uE92NV3y65+L7LxPh3e38HGBxL4YJuNDyXwYWTjrmkWPpLAR9ssfCyBjx/HwifmZmHTSbHwScnCp9osfDqBz5wMC5+dg4XZLwqfQxY+r7j4wqni4p5OLu7FdwOq0wGKcv9FZORLOnwZGbm3Rk+Fln5Fain9v9HPrwr4WoG6dD8doh71aUCV1n2dAmUU9ewb9J759Ozo3Hr2TVn539Lh25jno7Lyv4OV/10dvjcO39fhB1LPfngiPbuvQ89+pHrkxwn8pN0jP03gZ62J9fMEfjEzsX6ZwP3tXvlVAr+e7pVx+I0Ov52A3yXwe9zHH3T4Y0cH/imBP7fR/5LAXzs68IEE/tZG/XsC/ziuA/9J5v1nRjrxMSoN/oXO/07gWEFVZhpzI2CZLIZ58uLBNZZFj56DLIcn9aYjmNtsYtsErGserNYqbXv69jbY+si6U7QFEm1l+lUm9mPPksUoX2EJW4i3qln1TRJ1XL48mHib4Jc/zhbNfuJs6doeMwWlscXpg5bgKWEaHumRagaFfdUsYffr+L2SsaXpU9X/ZVxutl462TI0T8V1wU7Di9J/AI0OYIc=",
"user_name": "write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
INFO:apache_beam.runners.dataflow.internal.apiclient:Create job: <Job
createTime: '2020-05-08T18:35:21.756720Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2020-05-08_11_35_20-17841672732010029672'
location: 'us-central1'
name: 'beamapp-jenkins-0508183518-877218'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2020-05-08T18:35:21.756720Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
INFO:apache_beam.runners.dataflow.internal.apiclient:Created job with id:
[2020-05-08_11_35_20-17841672732010029672]
INFO:apache_beam.runners.dataflow.internal.apiclient:Submitted job:
2020-05-08_11_35_20-17841672732010029672
INFO:apache_beam.runners.dataflow.internal.apiclient:To access the Dataflow
monitoring console, please navigate to
https://console.cloud.google.com/dataflow/jobs/us-central1/2020-05-08_11_35_20-17841672732010029672?project=apache-beam-testing
INFO:apache_beam.runners.dataflow.dataflow_runner:Job
2020-05-08_11_35_20-17841672732010029672 is in state JOB_STATE_RUNNING
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:24.027Z:
JOB_MESSAGE_DETAILED: Checking permissions granted to controller Service
Account.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:24.555Z:
JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-f.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:25.267Z:
JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:25.302Z:
JOB_MESSAGE_DEBUG: Combiner lifting skipped for step
write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:25.398Z:
JOB_MESSAGE_DEBUG: Combiner lifting skipped for step group: GroupByKey not
followed by a combiner.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:25.445Z:
JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:25.487Z:
JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into
MergeBucketsMappingFns
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:25.590Z:
JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:25.650Z:
JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:25.683Z:
JOB_MESSAGE_DETAILED: Fusing consumer split into read/Read
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:25.721Z:
JOB_MESSAGE_DETAILED: Fusing consumer pair_with_one into split
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:25.759Z:
JOB_MESSAGE_DETAILED: Fusing consumer group/Reify into pair_with_one
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:25.787Z:
JOB_MESSAGE_DETAILED: Fusing consumer group/Write into group/Reify
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:25.833Z:
JOB_MESSAGE_DETAILED: Fusing consumer group/GroupByWindow into group/Read
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:25.896Z:
JOB_MESSAGE_DETAILED: Fusing consumer count into group/GroupByWindow
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:25.988Z:
JOB_MESSAGE_DETAILED: Fusing consumer format into count
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:26.073Z:
JOB_MESSAGE_DETAILED: Fusing consumer
write/Write/WriteImpl/WindowInto(WindowIntoFn) into format
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:26.149Z:
JOB_MESSAGE_DETAILED: Fusing consumer
write/Write/WriteImpl/WriteBundles/WriteBundles into
write/Write/WriteImpl/WindowInto(WindowIntoFn)
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:26.287Z:
JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/Pair into
write/Write/WriteImpl/WriteBundles/WriteBundles
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:26.441Z:
JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/Reify
into write/Write/WriteImpl/Pair
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:26.477Z:
JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/Write
into write/Write/WriteImpl/GroupByKey/Reify
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:26.516Z:
JOB_MESSAGE_DETAILED: Fusing consumer
write/Write/WriteImpl/GroupByKey/GroupByWindow into
write/Write/WriteImpl/GroupByKey/Read
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:26.552Z:
JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/Extract into
write/Write/WriteImpl/GroupByKey/GroupByWindow
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:26.595Z:
JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/InitializeWrite
into write/Write/WriteImpl/DoOnce/Read
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:26.628Z:
JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:26.663Z:
JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:26.695Z:
JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:26.729Z:
JOB_MESSAGE_DEBUG: Assigning stage ids.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:26.891Z:
JOB_MESSAGE_DEBUG: Executing wait step start26
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:26.955Z:
JOB_MESSAGE_BASIC: Executing operation
write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:26.988Z:
JOB_MESSAGE_BASIC: Executing operation write/Write/WriteImpl/GroupByKey/Create
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:27Z:
JOB_MESSAGE_DEBUG: Starting worker pool setup.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:27.023Z:
JOB_MESSAGE_BASIC: Starting 10 workers in us-central1-f...
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:27.023Z:
JOB_MESSAGE_BASIC: Executing operation group/Create
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:27.085Z:
JOB_MESSAGE_BASIC: Finished operation group/Create
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:27.085Z:
JOB_MESSAGE_BASIC: Finished operation write/Write/WriteImpl/GroupByKey/Create
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:27.159Z:
JOB_MESSAGE_DEBUG: Value "write/Write/WriteImpl/GroupByKey/Session"
materialized.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:27.192Z:
JOB_MESSAGE_DEBUG: Value "group/Session" materialized.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:27.269Z:
JOB_MESSAGE_BASIC: Executing operation
read/Read+split+pair_with_one+group/Reify+group/Write
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:54.810Z:
JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of workers to 6 based on
the rate of progress in the currently running stage(s).
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:35:54.853Z:
JOB_MESSAGE_DETAILED: Resized worker pool to 6, though goal was 10. This could
be a quota issue.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:36:00.294Z:
JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of workers to 10 based on
the rate of progress in the currently running stage(s).
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:36:02.181Z:
JOB_MESSAGE_WARNING: Your project already contains 100 Dataflow-created metric
descriptors and Stackdriver will not create new Dataflow custom metrics for
this job. Each unique user-defined metric name (independent of the DoFn in
which it is defined) produces a new metric descriptor. To delete old / unused
metric descriptors see
https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.list
and
https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.delete
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:37:25.022Z:
JOB_MESSAGE_DETAILED: Workers have started successfully.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:37:25.059Z:
JOB_MESSAGE_DETAILED: Workers have started successfully.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:40:41.954Z:
JOB_MESSAGE_BASIC: Finished operation
write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:40:42.033Z:
JOB_MESSAGE_DEBUG: Value "write/Write/WriteImpl/DoOnce/Read.out" materialized.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:40:42.068Z:
JOB_MESSAGE_DEBUG: Value "write/Write/WriteImpl/InitializeWrite.out"
materialized.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:40:42.152Z:
JOB_MESSAGE_BASIC: Executing operation
write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:40:42.179Z:
JOB_MESSAGE_BASIC: Executing operation
write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:40:42.199Z:
JOB_MESSAGE_BASIC: Finished operation
write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:40:42.215Z:
JOB_MESSAGE_BASIC: Executing operation
write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:40:42.249Z:
JOB_MESSAGE_BASIC: Finished operation
write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:40:42.262Z:
JOB_MESSAGE_DEBUG: Value
"write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0).output"
materialized.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:40:42.276Z:
JOB_MESSAGE_BASIC: Finished operation
write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:40:42.335Z:
JOB_MESSAGE_DEBUG: Value
"write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0).output"
materialized.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:40:42.376Z:
JOB_MESSAGE_DEBUG: Value
"write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0).output"
materialized.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:41:26.859Z:
JOB_MESSAGE_DETAILED: Checking permissions granted to controller Service
Account.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:42:46.228Z:
JOB_MESSAGE_BASIC: Finished operation
read/Read+split+pair_with_one+group/Reify+group/Write
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:42:46.307Z:
JOB_MESSAGE_BASIC: Executing operation group/Close
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:42:46.364Z:
JOB_MESSAGE_BASIC: Finished operation group/Close
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:42:46.435Z:
JOB_MESSAGE_BASIC: Executing operation
group/Read+group/GroupByWindow+count+format+write/Write/WriteImpl/WindowInto(WindowIntoFn)+write/Write/WriteImpl/WriteBundles/WriteBundles+write/Write/WriteImpl/Pair+write/Write/WriteImpl/GroupByKey/Reify+write/Write/WriteImpl/GroupByKey/Write
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:45:43.728Z:
JOB_MESSAGE_BASIC: Finished operation
group/Read+group/GroupByWindow+count+format+write/Write/WriteImpl/WindowInto(WindowIntoFn)+write/Write/WriteImpl/WriteBundles/WriteBundles+write/Write/WriteImpl/Pair+write/Write/WriteImpl/GroupByKey/Reify+write/Write/WriteImpl/GroupByKey/Write
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:45:43.811Z:
JOB_MESSAGE_BASIC: Executing operation write/Write/WriteImpl/GroupByKey/Close
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:45:43.868Z:
JOB_MESSAGE_BASIC: Finished operation write/Write/WriteImpl/GroupByKey/Close
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:45:43.939Z:
JOB_MESSAGE_BASIC: Executing operation
write/Write/WriteImpl/GroupByKey/Read+write/Write/WriteImpl/GroupByKey/GroupByWindow+write/Write/WriteImpl/Extract
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:45:46.692Z:
JOB_MESSAGE_BASIC: Finished operation
write/Write/WriteImpl/GroupByKey/Read+write/Write/WriteImpl/GroupByKey/GroupByWindow+write/Write/WriteImpl/Extract
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:45:46.769Z:
JOB_MESSAGE_DEBUG: Value "write/Write/WriteImpl/Extract.out" materialized.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:45:46.848Z:
JOB_MESSAGE_BASIC: Executing operation
write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:45:46.890Z:
JOB_MESSAGE_BASIC: Executing operation
write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:45:46.909Z:
JOB_MESSAGE_BASIC: Finished operation
write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:45:46.947Z:
JOB_MESSAGE_BASIC: Finished operation
write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:45:46.982Z:
JOB_MESSAGE_DEBUG: Value
"write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0).output"
materialized.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:45:47.040Z:
JOB_MESSAGE_DEBUG: Value
"write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0).output"
materialized.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:45:47.127Z:
JOB_MESSAGE_BASIC: Executing operation
write/Write/WriteImpl/PreFinalize/PreFinalize
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:45:49.615Z:
JOB_MESSAGE_BASIC: Finished operation
write/Write/WriteImpl/PreFinalize/PreFinalize
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:45:49.689Z:
JOB_MESSAGE_DEBUG: Value "write/Write/WriteImpl/PreFinalize.out" materialized.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:45:49.759Z:
JOB_MESSAGE_BASIC: Executing operation
write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:45:49.810Z:
JOB_MESSAGE_BASIC: Finished operation
write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:45:49.890Z:
JOB_MESSAGE_DEBUG: Value
"write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0).output"
materialized.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:45:49.979Z:
JOB_MESSAGE_BASIC: Executing operation
write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:45:52.907Z:
JOB_MESSAGE_BASIC: Finished operation
write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:45:52.973Z:
JOB_MESSAGE_DEBUG: Executing success step success24
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:45:53.124Z:
JOB_MESSAGE_DETAILED: Cleaning up.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:45:53.172Z:
JOB_MESSAGE_DEBUG: Starting worker pool teardown.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:45:53.202Z:
JOB_MESSAGE_BASIC: Stopping worker pool...
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:48:10.447Z:
JOB_MESSAGE_DETAILED: Autoscaling: Resized worker pool from 10 to 0.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:48:10.494Z:
JOB_MESSAGE_BASIC: Worker pool stopped.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-08T18:48:10.528Z:
JOB_MESSAGE_DEBUG: Tearing down pending resources...
INFO:apache_beam.runners.dataflow.dataflow_runner:Job
2020-05-08_11_35_20-17841672732010029672 is in state JOB_STATE_DONE
DEBUG:apache_beam.io.filesystem:Listing files in
'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results'
DEBUG:apache_beam.io.filesystem:translate_pattern:
'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results*-of-*'
->
'gs\\:\\/\\/temp\\-storage\\-for\\-end\\-to\\-end\\-tests\\/py\\-it\\-cloud\\/output\\/1588962917468\\/results[^/\\\\]*\\-of\\-[^/\\\\]*'
INFO:apache_beam.io.gcp.gcsio:Starting the size estimation of the input
INFO:apache_beam.io.gcp.gcsio:Finished listing 30 files in 0.06673240661621094
seconds.
INFO:apache_beam.testing.pipeline_verifiers:Find 30 files in
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results*-of-*:
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00000-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00001-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00002-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00003-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00004-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00005-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00006-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00007-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00008-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00009-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00010-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00011-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00012-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00013-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00014-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00015-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00016-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00017-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00018-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00019-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00020-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00021-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00022-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00023-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00024-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00025-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00026-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00027-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00028-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results-00029-of-00030
INFO:apache_beam.testing.pipeline_verifiers:Read from given path
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results*-of-*,
26186927 lines, checksum: ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710.
INFO:root:average word length: 19
DEBUG:apache_beam.io.filesystem:Listing files in
'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results'
DEBUG:apache_beam.io.filesystem:translate_pattern:
'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1588962917468/results*'
->
'gs\\:\\/\\/temp\\-storage\\-for\\-end\\-to\\-end\\-tests\\/py\\-it\\-cloud\\/output\\/1588962917468\\/results[^/\\\\]*'
INFO:apache_beam.io.gcp.gcsio:Starting the size estimation of the input
INFO:apache_beam.io.gcp.gcsio:Finished listing 30 files in 0.05718588829040527
seconds.
error: [Errno 111] Connection refused
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task
':sdks:python:test-suites:dataflow:py36:integrationTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to
get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
BUILD FAILED in 16m 15s
2020-05-08 18:50:21,511 d7c60f72 MainThread beam_integration_benchmark(1/1)
ERROR Error during benchmark beam_integration_benchmark
Traceback (most recent call last):
File
"<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py";,>
line 846, in RunBenchmark
DoRunPhase(spec, collector, detailed_timer)
File
"<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py";,>
line 689, in DoRunPhase
samples = spec.BenchmarkRun(spec)
File
"<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/linux_benchmarks/beam_integration_benchmark.py";,>
line 161, in Run
job_type=job_type)
File
"<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/providers/gcp/gcp_dpb_dataflow.py";,>
line 96, in SubmitJob
assert retcode == 0, "Integration Test Failed."
AssertionError: Integration Test Failed.
2020-05-08 18:50:21,512 d7c60f72 MainThread beam_integration_benchmark(1/1)
INFO Cleaning up benchmark beam_integration_benchmark
2020-05-08 18:50:21,514 d7c60f72 MainThread beam_integration_benchmark(1/1)
ERROR Exception running benchmark
Traceback (most recent call last):
File
"<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py";,>
line 995, in RunBenchmarkTask
RunBenchmark(spec, collector)
File
"<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py";,>
line 846, in RunBenchmark
DoRunPhase(spec, collector, detailed_timer)
File
"<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py";,>
line 689, in DoRunPhase
samples = spec.BenchmarkRun(spec)
File
"<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/linux_benchmarks/beam_integration_benchmark.py";,>
line 161, in Run
job_type=job_type)
File
"<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/providers/gcp/gcp_dpb_dataflow.py";,>
line 96, in SubmitJob
assert retcode == 0, "Integration Test Failed."
AssertionError: Integration Test Failed.
2020-05-08 18:50:21,515 d7c60f72 MainThread beam_integration_benchmark(1/1)
ERROR Benchmark 1/1 beam_integration_benchmark (UID:
beam_integration_benchmark0) failed. Execution will continue.
2020-05-08 18:50:21,515 d7c60f72 MainThread beam_integration_benchmark(1/1)
INFO Benchmark run statuses:
---------------------------------------------------------------------------------
Name UID Status Failed
Substatus
---------------------------------------------------------------------------------
beam_integration_benchmark beam_integration_benchmark0 FAILED
---------------------------------------------------------------------------------
Success rate: 0.00% (0/1)
2020-05-08 18:50:21,515 d7c60f72 MainThread beam_integration_benchmark(1/1)
INFO Complete logs can be found at:
<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/d7c60f72/pkb.log>
2020-05-08 18:50:21,516 d7c60f72 MainThread beam_integration_benchmark(1/1)
INFO Completion statuses can be found at:
<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/d7c60f72/completion_statuses.json>
Build step 'Execute shell' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
