See <https://builds.apache.org/job/beam_PostCommit_PythonVerify/136/changes>
Changes:
[robertwb] Add tests for WindowedValue.
------------------------------------------
[...truncated 3961 lines...]
"component_encodings": [
{
"@type": "WindowedValueCoder$<string of 736 bytes>",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$<string of 172 bytes>",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$<string of 172 bytes>",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$<string of 172 bytes>",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type":
"TimestampCoder$gAJjYXBhY2hlX2JlYW0uY29kZXJzLmNvZGVycwpUaW1lc3RhbXBDb2RlcgpxACmBcQF9cQJiLg==",
"component_encodings": []
},
{
"@type": "SingletonCoder$<string of 344 bytes>",
"component_encodings": []
}
],
"is_wrapper": true
}
]
},
"output_name": "out",
"user_name":
"write/WriteImpl/ViewAsIterable(write|WriteImpl|write_bundles.None)/CreatePCollectionView.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s10"
},
"user_name":
"write/WriteImpl/ViewAsIterable(write|WriteImpl|write_bundles.None)/CreatePCollectionView"
}
},
{
"kind": "ParallelDo",
"name": "s12",
"properties": {
"non_parallel_inputs": {
"s11": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s11"
},
"s9": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s9"
}
},
"output_info": [
{
"encoding": {
"@type": "WindowedValueCoder$<string of 736 bytes>",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$<string of 172 bytes>",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$<string of 172 bytes>",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$<string of 172 bytes>",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type":
"TimestampCoder$gAJjYXBhY2hlX2JlYW0uY29kZXJzLmNvZGVycwpUaW1lc3RhbXBDb2RlcgpxACmBcQF9cQJiLg==",
"component_encodings": []
},
{
"@type": "SingletonCoder$<string of 344 bytes>",
"component_encodings": []
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/WriteImpl/finalize_write.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "<string of 1496 bytes>",
"user_name": "write/WriteImpl/finalize_write"
}
}
],
"type": "JOB_TYPE_BATCH"
}
INFO:root:Create job: <Job
id: u'2016-07-21_17_51_37-13201584638255343868'
projectId: u'apache-beam-testing'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
INFO:root:Created job with id: [2016-07-21_17_51_37-13201584638255343868]
INFO:root:To access the Dataflow monitoring console, please navigate to
https://console.developers.google.com/project/apache-beam-testing/dataflow/job/2016-07-21_17_51_37-13201584638255343868
INFO:root:2016-07-21_17_51_37-13201584638255343868_00000156101618ca:
2016-07-22T00:51:38.826Z: JOB_MESSAGE_DETAILED: (4f3def46854313b4): Checking
required Cloud APIs are enabled.
INFO:root:Job 2016-07-21_17_51_37-13201584638255343868 is in state
JOB_STATE_RUNNING
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610161a26:
2016-07-22T00:51:39.174Z: JOB_MESSAGE_DEBUG: (4f3def46854317f9): Combiner
lifting skipped for step group: GroupByKey not followed by a combiner.
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610161a29:
2016-07-22T00:51:39.177Z: JOB_MESSAGE_DETAILED: (4f3def4685431ab3): Expanding
GroupByKey operations into optimizable parts.
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610161a2c:
2016-07-22T00:51:39.180Z: JOB_MESSAGE_DETAILED: (4f3def4685431d6d): Lifting
ValueCombiningMappingFns into MergeBucketsMappingFns
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610161a31:
2016-07-22T00:51:39.185Z: JOB_MESSAGE_DETAILED: (4f3def46854312e1): Annotating
graph with Autotuner information.
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610161a6c:
2016-07-22T00:51:39.244Z: JOB_MESSAGE_DETAILED: (4f3def4685431dc9): Fusing
adjacent ParDo, Read, Write, and Flatten operations
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610161a6f:
2016-07-22T00:51:39.247Z: JOB_MESSAGE_DETAILED: (4f3def4685431083): Fusing
consumer split into read
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610161a73:
2016-07-22T00:51:39.251Z: JOB_MESSAGE_DETAILED: (4f3def468543133d): Fusing
consumer group/Reify into pair_with_one
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610161a75:
2016-07-22T00:51:39.253Z: JOB_MESSAGE_DETAILED: (4f3def46854315f7): Fusing
consumer format into count
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610161a77:
2016-07-22T00:51:39.255Z: JOB_MESSAGE_DETAILED: (4f3def46854318b1): Fusing
consumer count into group/GroupByWindow
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610161a7d:
2016-07-22T00:51:39.261Z: JOB_MESSAGE_DETAILED: (4f3def4685431e25): Fusing
consumer pair_with_one into split
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610161a7f:
2016-07-22T00:51:39.263Z: JOB_MESSAGE_DETAILED: (4f3def46854310df): Fusing
consumer group/GroupByWindow into group/Read
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610161a82:
2016-07-22T00:51:39.266Z: JOB_MESSAGE_DETAILED: (4f3def4685431399): Fusing
consumer write/WriteImpl/write_bundles into format
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610161a85:
2016-07-22T00:51:39.269Z: JOB_MESSAGE_DETAILED: (4f3def4685431653): Fusing
consumer group/Write into group/Reify
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610161ac2:
2016-07-22T00:51:39.330Z: JOB_MESSAGE_DEBUG: (4f3def468543124f): Workflow
config is missing a default resource spec.
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610161ac6:
2016-07-22T00:51:39.334Z: JOB_MESSAGE_DETAILED: (4f3def4685431509): Adding
StepResource setup and teardown to workflow graph.
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610161ae9:
2016-07-22T00:51:39.369Z: JOB_MESSAGE_DEBUG: (b991d539699b3dad): Adding
workflow start and stop steps.
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610161b1b:
2016-07-22T00:51:39.419Z: JOB_MESSAGE_DEBUG: (1138960ce236da4): Assigning stage
ids.
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610161b8c:
2016-07-22T00:51:39.532Z: JOB_MESSAGE_DEBUG: (4f3def46854313da): Executing wait
step start2
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610161b9a:
2016-07-22T00:51:39.546Z: JOB_MESSAGE_BASIC: S02: (aac97674c28c4c7f): Executing
operation group/Create
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610161bb1:
2016-07-22T00:51:39.569Z: JOB_MESSAGE_DEBUG: (dc6a842c89d6f8fd): Executing
operation write/WriteImpl/DoOnce
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610161bc3:
2016-07-22T00:51:39.587Z: JOB_MESSAGE_DEBUG: (48fa59a3a6e0b9ef): Value
"write/WriteImpl/DoOnce.out" materialized.
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610161bd0:
2016-07-22T00:51:39.600Z: JOB_MESSAGE_BASIC: S01: (b991d539699b3004): Executing
operation write/WriteImpl/initialize_write
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610161c66:
2016-07-22T00:51:39.750Z: JOB_MESSAGE_DEBUG: (b7e867830e35ddfb): Starting
worker pool setup.
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610161c69:
2016-07-22T00:51:39.753Z: JOB_MESSAGE_BASIC: (b7e867830e35d465): Starting 1
workers...
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610161c81:
2016-07-22T00:51:39.777Z: JOB_MESSAGE_DEBUG: (a993b3962c2158b6): Value
"group/Session" materialized.
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610161c8f:
2016-07-22T00:51:39.791Z: JOB_MESSAGE_BASIC: S03: (4f3def4685431c08): Executing
operation read+split+pair_with_one+group/Reify+group/Write
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610178da7:
2016-07-22T00:53:14.279Z: JOB_MESSAGE_DETAILED: (40d5c1bf39052c56): Workers
have started successfully.
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610184aaf:
2016-07-22T00:54:02.671Z: JOB_MESSAGE_ERROR: (f9aaa5909718f9a8): Traceback
(most recent call last):
File "/usr/local/lib/python2.7/dist-packages/dataflow_worker/batchworker.py",
line 462, in do_work
work_executor.execute()
File "dataflow_worker/executor.py", line 891, in
dataflow_worker.executor.MapTaskExecutor.execute
(dataflow_worker/executor.c:24041)
op.start()
File "dataflow_worker/executor.py", line 477, in
dataflow_worker.executor.DoOperation.start (dataflow_worker/executor.c:14398)
def start(self):
File "dataflow_worker/executor.py", line 508, in
dataflow_worker.executor.DoOperation.start (dataflow_worker/executor.c:14333)
self.dofn_runner.start()
File "apache_beam/runners/common.py", line 92, in
apache_beam.runners.common.DoFnRunner.start (apache_beam/runners/common.c:3281)
self.logging_context.exit()
AttributeError: 'PerThreadLoggingContext' object has no attribute 'exit'
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610184ee9:
2016-07-22T00:54:03.753Z: JOB_MESSAGE_ERROR: (d6ec76c2e7617e0e): Traceback
(most recent call last):
File "/usr/local/lib/python2.7/dist-packages/dataflow_worker/batchworker.py",
line 462, in do_work
work_executor.execute()
File "dataflow_worker/executor.py", line 891, in
dataflow_worker.executor.MapTaskExecutor.execute
(dataflow_worker/executor.c:24041)
op.start()
File "dataflow_worker/executor.py", line 477, in
dataflow_worker.executor.DoOperation.start (dataflow_worker/executor.c:14398)
def start(self):
File "dataflow_worker/executor.py", line 508, in
dataflow_worker.executor.DoOperation.start (dataflow_worker/executor.c:14333)
self.dofn_runner.start()
File "apache_beam/runners/common.py", line 92, in
apache_beam.runners.common.DoFnRunner.start (apache_beam/runners/common.c:3281)
self.logging_context.exit()
AttributeError: 'PerThreadLoggingContext' object has no attribute 'exit'
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610185353:
2016-07-22T00:54:04.883Z: JOB_MESSAGE_ERROR: (7966faa632df5b4b): Traceback
(most recent call last):
File "/usr/local/lib/python2.7/dist-packages/dataflow_worker/batchworker.py",
line 462, in do_work
work_executor.execute()
File "dataflow_worker/executor.py", line 891, in
dataflow_worker.executor.MapTaskExecutor.execute
(dataflow_worker/executor.c:24041)
op.start()
File "dataflow_worker/executor.py", line 477, in
dataflow_worker.executor.DoOperation.start (dataflow_worker/executor.c:14398)
def start(self):
File "dataflow_worker/executor.py", line 508, in
dataflow_worker.executor.DoOperation.start (dataflow_worker/executor.c:14333)
self.dofn_runner.start()
File "apache_beam/runners/common.py", line 92, in
apache_beam.runners.common.DoFnRunner.start (apache_beam/runners/common.c:3281)
self.logging_context.exit()
AttributeError: 'PerThreadLoggingContext' object has no attribute 'exit'
INFO:root:2016-07-21_17_51_37-13201584638255343868_00000156101857ea:
2016-07-22T00:54:06.058Z: JOB_MESSAGE_ERROR: (47fa3864172b6bee): Traceback
(most recent call last):
File "/usr/local/lib/python2.7/dist-packages/dataflow_worker/batchworker.py",
line 462, in do_work
work_executor.execute()
File "dataflow_worker/executor.py", line 891, in
dataflow_worker.executor.MapTaskExecutor.execute
(dataflow_worker/executor.c:24041)
op.start()
File "dataflow_worker/executor.py", line 477, in
dataflow_worker.executor.DoOperation.start (dataflow_worker/executor.c:14398)
def start(self):
File "dataflow_worker/executor.py", line 508, in
dataflow_worker.executor.DoOperation.start (dataflow_worker/executor.c:14333)
self.dofn_runner.start()
File "apache_beam/runners/common.py", line 92, in
apache_beam.runners.common.DoFnRunner.start (apache_beam/runners/common.c:3281)
self.logging_context.exit()
AttributeError: 'PerThreadLoggingContext' object has no attribute 'exit'
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610185c21:
2016-07-22T00:54:07.137Z: JOB_MESSAGE_ERROR: (3219b09f27891943): Traceback
(most recent call last):
File "/usr/local/lib/python2.7/dist-packages/dataflow_worker/batchworker.py",
line 462, in do_work
work_executor.execute()
File "dataflow_worker/executor.py", line 891, in
dataflow_worker.executor.MapTaskExecutor.execute
(dataflow_worker/executor.c:24041)
op.start()
File "dataflow_worker/executor.py", line 477, in
dataflow_worker.executor.DoOperation.start (dataflow_worker/executor.c:14398)
def start(self):
File "dataflow_worker/executor.py", line 508, in
dataflow_worker.executor.DoOperation.start (dataflow_worker/executor.c:14333)
self.dofn_runner.start()
File "apache_beam/runners/common.py", line 92, in
apache_beam.runners.common.DoFnRunner.start (apache_beam/runners/common.c:3281)
self.logging_context.exit()
AttributeError: 'PerThreadLoggingContext' object has no attribute 'exit'
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610185c7a:
2016-07-22T00:54:07.226Z: JOB_MESSAGE_DEBUG: (4c14a458e3ae44bc): Executing
failure step failure1
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610185c7d:
2016-07-22T00:54:07.229Z: JOB_MESSAGE_ERROR: (4c14a458e3ae4a06): Workflow
failed. Causes: (4f3def46854315c1):
S03:read+split+pair_with_one+group/Reify+group/Write failed.
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610185cba:
2016-07-22T00:54:07.290Z: JOB_MESSAGE_DETAILED: (8c71a48a1f91b1f7): Cleaning up.
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610185d0d:
2016-07-22T00:54:07.373Z: JOB_MESSAGE_DEBUG: (8c71a48a1f91be40): Starting
worker pool teardown.
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610185d10:
2016-07-22T00:54:07.376Z: JOB_MESSAGE_BASIC: (8c71a48a1f91bbc6): Stopping
worker pool...
INFO:root:2016-07-21_17_51_37-13201584638255343868_000001561018604f:
2016-07-22T00:54:08.207Z: JOB_MESSAGE_DETAILED: (3fd3f1a5f1e66cbd): Failed to
publish the result of the work update. Causes: (3fd3f1a5f1e664b8): Failed to
update work status. Causes: (f8d716acac58cc73): Work "8707453335425830429" not
leased (or the lease was lost).
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610199598:
2016-07-22T00:55:27.384Z: JOB_MESSAGE_BASIC: (8c71a48a1f91b80f): Worker pool
stopped.
INFO:root:2016-07-21_17_51_37-13201584638255343868_0000015610199a1e:
2016-07-22T00:55:28.542Z: JOB_MESSAGE_DEBUG: (8c71a48a1f91b302): Tearing down
pending resources...
INFO:root:Job 2016-07-21_17_51_37-13201584638255343868 is in state
JOB_STATE_FAILED
Traceback (most recent call last):
File "/usr/lib/python2.7/runpy.py", line 162, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/usr/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File
"<https://builds.apache.org/job/beam_PostCommit_PythonVerify/ws/sdks/python/apache_beam/examples/wordcount.py";,>
line 102, in <module>
run()
File
"<https://builds.apache.org/job/beam_PostCommit_PythonVerify/ws/sdks/python/apache_beam/examples/wordcount.py";,>
line 93, in run
result = p.run()
File "apache_beam/pipeline.py", line 159, in run
return self.runner.run(self)
File "apache_beam/runners/dataflow_runner.py", line 188, in run
% getattr(self, 'last_error_msg', None), self.result)
apache_beam.runners.dataflow_runner.DataflowRuntimeException: Dataflow pipeline
failed:
(3219b09f27891943): Traceback (most recent call last):
File "/usr/local/lib/python2.7/dist-packages/dataflow_worker/batchworker.py",
line 462, in do_work
work_executor.execute()
File "dataflow_worker/executor.py", line 891, in
dataflow_worker.executor.MapTaskExecutor.execute
(dataflow_worker/executor.c:24041)
op.start()
File "dataflow_worker/executor.py", line 477, in
dataflow_worker.executor.DoOperation.start (dataflow_worker/executor.c:14398)
def start(self):
File "dataflow_worker/executor.py", line 508, in
dataflow_worker.executor.DoOperation.start (dataflow_worker/executor.c:14333)
self.dofn_runner.start()
File "apache_beam/runners/common.py", line 92, in
apache_beam.runners.common.DoFnRunner.start (apache_beam/runners/common.c:3281)
self.logging_context.exit()
AttributeError: 'PerThreadLoggingContext' object has no attribute 'exit'
# Grep will exit with status 1 if success message was not found.
echo ">>> CHECKING JOB SUCCESS"
>>> CHECKING JOB SUCCESS
grep JOB_STATE_DONE job_output
Build step 'Execute shell' marked build as failure
