See
<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/1438/display/redirect?page=changes>
Changes:
[mxm] [BEAM-6733] Add pipeline option to flush bundle data before
[github] Python3 fix - convert dict.keys() to list before indexing (#11733)
[github] Updates google-apitools and httplib2 (#11726)
------------------------------------------
[...truncated 150.31 KB...]
},
"output_name": "out",
"user_name":
"write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0).output"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "None",
"step_name": "s17"
},
"user_name":
"write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)",
"windowing_strategy":
"%0AB%22%40%0A%1Dref_Coder_GlobalWindowCoder_1%12%1F%0A%1D%0A%1Bbeam%3Acoder%3Aglobal_window%3Av1jQ%0A%22%0A%20beam%3Awindow_fn%3Aglobal_windows%3Av1%10%01%1A%1Dref_Coder_GlobalWindowCoder_1%22%02%3A%00%28%010%018%01H%01"
}
},
{
"kind": "ParallelDo",
"name": "s21",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_finalize_write"
}
],
"non_parallel_inputs": {
"python_side_input0-write/Write/WriteImpl/FinalizeWrite": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s18"
},
"python_side_input1-write/Write/WriteImpl/FinalizeWrite": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s19"
},
"python_side_input2-write/Write/WriteImpl/FinalizeWrite": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s20"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type":
"FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": [
{
"@type":
"FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": [],
"pipeline_proto_coder_id":
"ref_Coder_FastPrimitivesCoder_5"
},
{
"@type":
"FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": [],
"pipeline_proto_coder_id":
"ref_Coder_FastPrimitivesCoder_5"
}
],
"is_pair_like": true,
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "None",
"user_name": "write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn":
"eNrNWHd8HEcVvjvJbe0kLsRgUriYGFYhd+eCHceEhORsOeawrKyUeClms7c7d7PWtjczK1lBm8RxVDABAoTQTC+hdwi995LQMR1CJ/QaqvNm9k7yCVn4P/OTtLv3dt6389733vv2dGOX7tix7VBi1YkdlAWzQ96IWMDLTsSIVrV93677ZC+z45iw7VFvqEGu5yDkUyjoZlcul7MaIXR1gHgR/tZtTjSr4YW2711HrBHmCaJBt7lMuSQIy5o8Jg4s6DMXoS1mkUM4h4W021wq14jRmFjUCwWHRY7r+X7ZkkfNchixBbFCOyCuSGIfYRcrl117BtHnSuWyRFm8ME6EQuKgqUdHiZgxLTUXShPzml4IyybhNLOInzvygevUHsouwYTYImIcTp+EMwxY3te3D1ZMwkoDViX1ffAAfXYSBDkgvEizBvE84IVDGpyJmVudwgN1czk+qOH5xIptQa2YkYZ3AB7UgRDF6B3y8rDtJ7iORcOeS5g2IGzhOddIY3/bBmsQ+MEpnKWbGgJnHnLvcHZH6vzIdpVdg3MUeVwwOHccHmJA0VzQ9oTzzD143eTbKhVBgrjEMWy7SUpYGCUSuiURZSfCBa/EoyVPlBw/StxKlt3Khs1bL966ceum9VsvWr+pwghPfKRkbVKfCVuyZ/GkIcN+KJXbPz+FdTo9mxbpeSauysHDcL2MJkwCi1ObuRweXsupbToRRg2dCVc23jppg9EVo7i/aqTS04P4F6TwiLq5QgYtwbIdyFK3BVxono/20jq9dSsJetZvdktRA00zj5c2KJmrOyGaflRv45TVzdIFc3pWVPROFCDbnCO1GUPrzW5ZdImIYIO5BC8DL2iRt1HFL8uoEvs21ugmc41sDdmLIRLJbM/3wqYVkhE8Y0E/ckr1kkWJLRO0uS+pdxZlKAjDjiwnwvO1y1kzCUgo+n3bITTyVaq2YKouovK4tZZX54tJCtv2waN0c4vs01FBcesc685S7bW+pHq7snfmuCuI/Upvq/WVBS7p2Ees6kyzrg5jzxnyiTuAcLskmgaPTuFSXeXEtYUNl83lOL18Oy7R4DG4y8tTuEI3T5epdOQgkY0lg4WqeRYapfe2mU1vwz0xOdi2DW+A7aoqRrzQjUasAJMrc4pTbceJRqPEUTBcU/TbvvVf3hr0Ki6GPTIiwXbONcMaSejIJtfgSp2eYy7G9bJi5RiDXePwWANqei1fy+FfV21VVZvKjeWm8ocKAzl4XN847O5RLu1YAE17TIaWCo0CUtlPwiEv5O1zifv2MKmMRGyIY1ykIsOy+glTxRs6ZFD2s7U3Ym41SkKxa9DqH920pcKZU+HukOx0SX3luKRUMj7K8Sj0q61c4ttB3bUvhat2H85Xc2CYZ8qOZ1FgMYSUlT292wE1kFV+WqMOBifg6h4B1xiwtyNdTSIspBPL01SPqSeeLzAkeLzKMd6Wd+EJE/BEA57U4eoFccSEFURuIpVin7ly1ojPQoAnj4NlwLUK3kJfR1gW2BNQN8ChV/XNxZ5D8AO4FLlDmgpIU1dtUa1avVbkRH5/YSznIlWH8mlhrMDXjuX3d7kF0X0oJ+kTC8bwjts1VGBbxgpu95oc2ha6CzK7WJTZWldd8qrRtRqPN+VdXAlE76sVVOguadg4XaEhy8S8Hi391cj3iSqrYtQocmzB4jq3OOIJWgxQ04uC2ngnJEXiE9n+xaxhiFu0edFGh7DpE4HekppysddjXBTFSNRez4skdGSBECZ9EHHtOr72QnUsQ1MAzdrX97gAT400ybmIIp/DfiW5Hvc9h8CQ0iBkD3zVuTuCWIxO9zYE6rZPQgjVIFSat4OxiEFEzxUQmwWFDaBS0S4ipp4uRyDwSRC0Xw1la+4iTHbfl68uzxWW57vzp+eX5RfnC/lCAYZ7sAxHDDhAHZr09aUw2ifgOgOeMg5jBqTUTeF6/QRj4wbae5xI3TiPSO1VvsRVgbWk6iDOsptSOKSbqyS6eu9yrUzRM8Sb50HstbnoZ17gCW94Wv3GEXIihUnzDJUHfKWr285QC21qHrR+NZxbKE9FlMMpPK3O6wpI5pELO4hbQLfMp8bttS2spyPWM1J4Zl2NgFYKM5hb54HZqWZtlrUW0rMQ6dkpPEeVVZYmuO1Eczt7UAeMBs9FhNtTeJ6qGgt1FZ7f4S+lkpeno52JRYMXoOsLU3iRenjgOSzicOTw4qNH9HvuPXbsBl6XP0k9GYcXG/CSCXhpCi/DOnq5Aa9I6nQH7aU6lXy/MoVX6fRmKol6dQqvoVNUZvuOFF6LAPQWKvP1uhReX6e3UhnxG1J4I72N3q7QJ+FNcyvzhpNS5jdTVNy36PQyKmX0rSm8Taf/WzPfrra/szX7MonK1waq3WNyPr0DleidPRSP7zpVevTuTj26c/faAjVkV7/HgPdiV98pu/p9yMb7DfhABxsfTOFDbTY+nMJHkI07pln4aAofa7Pw8RQ+cRwLn5ybhY0nxcKnJAufbrPwmRQ+ezIsfG4OFma/KHweWfiC4uKLp4qLuzq5uBvfDahBByjK/ZeQkS8b8BVk5O4+eiq09KtSS+n/jX5+TcDXderR/XSI+jSgIVVa9w0KlFHUs2/Su+bTs6Nz69m3ZOV/24DvYJ6Pysr/Llb+9wz4/jj8wIAfSj370Yn07J4OPfux6pGfpPDTdo/8LIWftybWL1L45czE+lUK97Z75dcp/Ga6V8bhtwb8bgJ+n8IfcB9/NOBPHR345xT+0kb/awp/6+jA+1L4exv1Hyn887gO/BeZ958Z2cTHqDT4Nzr/J4VjuqrMLOZGyHJ5DPPkxYNrLI8ePQdZAU/qTUcwr9nEtglZ1zxYrVXa9uztbbD1kXVnaAsk2orsq0wSJL4ti1G+whK2EG/V8uqbJOq4fHmw8DbBL3+cLZr9xNnStT1hCkpji7MHLcFTyjQ80iO1HAr7ylnCHtTxeyVjS7Onqv/LeNxqvXSyZWieSuqCnYYX5fsBzfFgew==",
"user_name": "write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
INFO:apache_beam.runners.dataflow.internal.apiclient:Create job: <Job
createTime: '2020-05-18T18:58:32.757546Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2020-05-18_11_58_31-15077564107790020772'
location: 'us-central1'
name: 'beamapp-jenkins-0518185829-950539'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2020-05-18T18:58:32.757546Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
INFO:apache_beam.runners.dataflow.internal.apiclient:Created job with id:
[2020-05-18_11_58_31-15077564107790020772]
INFO:apache_beam.runners.dataflow.internal.apiclient:Submitted job:
2020-05-18_11_58_31-15077564107790020772
INFO:apache_beam.runners.dataflow.internal.apiclient:To access the Dataflow
monitoring console, please navigate to
https://console.cloud.google.com/dataflow/jobs/us-central1/2020-05-18_11_58_31-15077564107790020772?project=apache-beam-testing
INFO:apache_beam.runners.dataflow.dataflow_runner:Job
2020-05-18_11_58_31-15077564107790020772 is in state JOB_STATE_RUNNING
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:34.832Z:
JOB_MESSAGE_DETAILED: Checking permissions granted to controller Service
Account.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:35.315Z:
JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-a.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:36.195Z:
JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:36.234Z:
JOB_MESSAGE_DEBUG: Combiner lifting skipped for step
write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:36.272Z:
JOB_MESSAGE_DEBUG: Combiner lifting skipped for step group: GroupByKey not
followed by a combiner.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:36.300Z:
JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:36.341Z:
JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into
MergeBucketsMappingFns
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:36.443Z:
JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:36.518Z:
JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:36.553Z:
JOB_MESSAGE_DETAILED: Fusing consumer split into read/Read
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:36.579Z:
JOB_MESSAGE_DETAILED: Fusing consumer pair_with_one into split
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:36.617Z:
JOB_MESSAGE_DETAILED: Fusing consumer group/Reify into pair_with_one
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:36.657Z:
JOB_MESSAGE_DETAILED: Fusing consumer group/Write into group/Reify
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:36.693Z:
JOB_MESSAGE_DETAILED: Fusing consumer group/GroupByWindow into group/Read
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:36.739Z:
JOB_MESSAGE_DETAILED: Fusing consumer count into group/GroupByWindow
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:36.765Z:
JOB_MESSAGE_DETAILED: Fusing consumer format into count
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:36.792Z:
JOB_MESSAGE_DETAILED: Fusing consumer
write/Write/WriteImpl/WindowInto(WindowIntoFn) into format
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:36.829Z:
JOB_MESSAGE_DETAILED: Fusing consumer
write/Write/WriteImpl/WriteBundles/WriteBundles into
write/Write/WriteImpl/WindowInto(WindowIntoFn)
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:36.864Z:
JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/Pair into
write/Write/WriteImpl/WriteBundles/WriteBundles
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:36.897Z:
JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/Reify
into write/Write/WriteImpl/Pair
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:36.935Z:
JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/Write
into write/Write/WriteImpl/GroupByKey/Reify
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:36.970Z:
JOB_MESSAGE_DETAILED: Fusing consumer
write/Write/WriteImpl/GroupByKey/GroupByWindow into
write/Write/WriteImpl/GroupByKey/Read
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:37.011Z:
JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/Extract into
write/Write/WriteImpl/GroupByKey/GroupByWindow
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:37.051Z:
JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/InitializeWrite
into write/Write/WriteImpl/DoOnce/Read
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:37.089Z:
JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:37.133Z:
JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:37.159Z:
JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:37.191Z:
JOB_MESSAGE_DEBUG: Assigning stage ids.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:37.336Z:
JOB_MESSAGE_DEBUG: Executing wait step start26
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:37.480Z:
JOB_MESSAGE_BASIC: Executing operation
write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:37.513Z:
JOB_MESSAGE_BASIC: Executing operation write/Write/WriteImpl/GroupByKey/Create
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:37.525Z:
JOB_MESSAGE_DEBUG: Starting worker pool setup.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:37.543Z:
JOB_MESSAGE_BASIC: Executing operation group/Create
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:37.559Z:
JOB_MESSAGE_BASIC: Starting 10 workers in us-central1-a...
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:37.618Z:
JOB_MESSAGE_BASIC: Finished operation group/Create
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:37.618Z:
JOB_MESSAGE_BASIC: Finished operation write/Write/WriteImpl/GroupByKey/Create
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:37.688Z:
JOB_MESSAGE_DEBUG: Value "write/Write/WriteImpl/GroupByKey/Session"
materialized.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:37.716Z:
JOB_MESSAGE_DEBUG: Value "group/Session" materialized.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:58:37.834Z:
JOB_MESSAGE_BASIC: Executing operation
read/Read+split+pair_with_one+group/Reify+group/Write
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:59:04.982Z:
JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of workers to 6 based on
the rate of progress in the currently running stage(s).
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:59:05.017Z:
JOB_MESSAGE_DETAILED: Resized worker pool to 6, though goal was 10. This could
be a quota issue.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:59:06.790Z:
JOB_MESSAGE_WARNING: Your project already contains 100 Dataflow-created metric
descriptors and Stackdriver will not create new Dataflow custom metrics for
this job. Each unique user-defined metric name (independent of the DoFn in
which it is defined) produces a new metric descriptor. To delete old / unused
metric descriptors see
https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.list
and
https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.delete
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T18:59:10.384Z:
JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of workers to 10 based on
the rate of progress in the currently running stage(s).
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:00:38.532Z:
JOB_MESSAGE_DETAILED: Workers have started successfully.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:00:38.566Z:
JOB_MESSAGE_DETAILED: Workers have started successfully.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:04:37.309Z:
JOB_MESSAGE_DETAILED: Checking permissions granted to controller Service
Account.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:09.267Z:
JOB_MESSAGE_BASIC: Finished operation
write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:09.350Z:
JOB_MESSAGE_DEBUG: Value "write/Write/WriteImpl/DoOnce/Read.out" materialized.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:09.395Z:
JOB_MESSAGE_DEBUG: Value "write/Write/WriteImpl/InitializeWrite.out"
materialized.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:09.470Z:
JOB_MESSAGE_BASIC: Executing operation
write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:09.503Z:
JOB_MESSAGE_BASIC: Executing operation
write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:09.527Z:
JOB_MESSAGE_BASIC: Finished operation
write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:09.544Z:
JOB_MESSAGE_BASIC: Executing operation
write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:09.551Z:
JOB_MESSAGE_BASIC: Finished operation
write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:09.604Z:
JOB_MESSAGE_DEBUG: Value
"write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0).output"
materialized.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:09.606Z:
JOB_MESSAGE_BASIC: Finished operation
write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:09.640Z:
JOB_MESSAGE_DEBUG: Value
"write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0).output"
materialized.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:09.842Z:
JOB_MESSAGE_DEBUG: Value
"write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0).output"
materialized.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:54.719Z:
JOB_MESSAGE_BASIC: Finished operation
read/Read+split+pair_with_one+group/Reify+group/Write
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:54.786Z:
JOB_MESSAGE_BASIC: Executing operation group/Close
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:54.831Z:
JOB_MESSAGE_BASIC: Finished operation group/Close
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:54.892Z:
JOB_MESSAGE_BASIC: Executing operation
group/Read+group/GroupByWindow+count+format+write/Write/WriteImpl/WindowInto(WindowIntoFn)+write/Write/WriteImpl/WriteBundles/WriteBundles+write/Write/WriteImpl/Pair+write/Write/WriteImpl/GroupByKey/Reify+write/Write/WriteImpl/GroupByKey/Write
The message received from the daemon indicates that the daemon has disappeared.
Build request sent: Build{id=66c6ed10-390d-4977-909f-3760c84eb232,
currentDir=<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src}>
Attempting to read last messages from the daemon log...
Daemon pid: 27577
log file: /home/jenkins/.gradle/daemon/5.2.1/daemon-27577.out.log
----- Last 20 lines from daemon log file - daemon-27577.out.log -----
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:00:38.566Z:
JOB_MESSAGE_DETAILED: Workers have started successfully.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:04:37.309Z:
JOB_MESSAGE_DETAILED: Checking permissions granted to controller Service
Account.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:09.267Z:
JOB_MESSAGE_BASIC: Finished operation
write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:09.350Z:
JOB_MESSAGE_DEBUG: Value "write/Write/WriteImpl/DoOnce/Read.out" materialized.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:09.395Z:
JOB_MESSAGE_DEBUG: Value "write/Write/WriteImpl/InitializeWrite.out"
materialized.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:09.470Z:
JOB_MESSAGE_BASIC: Executing operation
write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:09.503Z:
JOB_MESSAGE_BASIC: Executing operation
write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:09.527Z:
JOB_MESSAGE_BASIC: Finished operation
write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:09.544Z:
JOB_MESSAGE_BASIC: Executing operation
write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:09.551Z:
JOB_MESSAGE_BASIC: Finished operation
write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:09.604Z:
JOB_MESSAGE_DEBUG: Value
"write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0).output"
materialized.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:09.606Z:
JOB_MESSAGE_BASIC: Finished operation
write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:09.640Z:
JOB_MESSAGE_DEBUG: Value
"write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0).output"
materialized.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:09.842Z:
JOB_MESSAGE_DEBUG: Value
"write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0).output"
materialized.
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:54.719Z:
JOB_MESSAGE_BASIC: Finished operation
read/Read+split+pair_with_one+group/Reify+group/Write
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:54.786Z:
JOB_MESSAGE_BASIC: Executing operation group/Close
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:54.831Z:
JOB_MESSAGE_BASIC: Finished operation group/Close
INFO:apache_beam.runners.dataflow.dataflow_runner:2020-05-18T19:05:54.892Z:
JOB_MESSAGE_BASIC: Executing operation
group/Read+group/GroupByWindow+count+format+write/Write/WriteImpl/WindowInto(WindowIntoFn)+write/Write/WriteImpl/WriteBundles/WriteBundles+write/Write/WriteImpl/Pair+write/Write/WriteImpl/GroupByKey/Reify+write/Write/WriteImpl/GroupByKey/Write
<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/scripts/run_integration_test.sh>:
line 278: 2215 Terminated python setup.py nosetests
--test-pipeline-options="$PIPELINE_OPTS" --with-xunitmp
--xunitmp-file=$XUNIT_FILE --ignore-files '.*py3\d?\.py$' $TEST_OPTS
Daemon vm is shutting down... The daemon has exited normally or was terminated
in response to a user interrupt.
----- End of the daemon log -----
FAILURE: Build failed with an exception.
* What went wrong:
Gradle build daemon disappeared unexpectedly (it may have been killed or may
have crashed)
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to
get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
2020-05-18 19:08:42,246 9a65839a MainThread beam_integration_benchmark(1/1)
ERROR Error during benchmark beam_integration_benchmark
Traceback (most recent call last):
File
"<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py";,>
line 846, in RunBenchmark
DoRunPhase(spec, collector, detailed_timer)
File
"<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py";,>
line 689, in DoRunPhase
samples = spec.BenchmarkRun(spec)
File
"<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/linux_benchmarks/beam_integration_benchmark.py";,>
line 161, in Run
job_type=job_type)
File
"<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/providers/gcp/gcp_dpb_dataflow.py";,>
line 96, in SubmitJob
assert retcode == 0, "Integration Test Failed."
AssertionError: Integration Test Failed.
2020-05-18 19:08:42,247 9a65839a MainThread beam_integration_benchmark(1/1)
INFO Cleaning up benchmark beam_integration_benchmark
2020-05-18 19:08:42,249 9a65839a MainThread beam_integration_benchmark(1/1)
ERROR Exception running benchmark
Traceback (most recent call last):
File
"<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py";,>
line 995, in RunBenchmarkTask
RunBenchmark(spec, collector)
File
"<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py";,>
line 846, in RunBenchmark
DoRunPhase(spec, collector, detailed_timer)
File
"<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py";,>
line 689, in DoRunPhase
samples = spec.BenchmarkRun(spec)
File
"<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/linux_benchmarks/beam_integration_benchmark.py";,>
line 161, in Run
job_type=job_type)
File
"<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/providers/gcp/gcp_dpb_dataflow.py";,>
line 96, in SubmitJob
assert retcode == 0, "Integration Test Failed."
AssertionError: Integration Test Failed.
2020-05-18 19:08:42,249 9a65839a MainThread beam_integration_benchmark(1/1)
ERROR Benchmark 1/1 beam_integration_benchmark (UID:
beam_integration_benchmark0) failed. Execution will continue.
2020-05-18 19:08:42,250 9a65839a MainThread beam_integration_benchmark(1/1)
INFO Benchmark run statuses:
---------------------------------------------------------------------------------
Name UID Status Failed
Substatus
---------------------------------------------------------------------------------
beam_integration_benchmark beam_integration_benchmark0 FAILED
---------------------------------------------------------------------------------
Success rate: 0.00% (0/1)
2020-05-18 19:08:42,250 9a65839a MainThread beam_integration_benchmark(1/1)
INFO Complete logs can be found at:
<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/9a65839a/pkb.log>
2020-05-18 19:08:42,250 9a65839a MainThread beam_integration_benchmark(1/1)
INFO Completion statuses can be found at:
<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/9a65839a/completion_statuses.json>
Build step 'Execute shell' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
