See <https://builds.apache.org/job/beam_PostCommit_PythonVerify/440/>
------------------------------------------
[...truncated 3624 lines...]
"component_encodings": []
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/WriteImpl/GroupByKey.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s12"
},
"serialized_fn": "<string of 356 bytes>",
"user_name": "write/WriteImpl/GroupByKey"
}
},
{
"kind": "ParallelDo",
"name": "s14",
"properties": {
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "WindowedValueCoder$<string of 736 bytes>",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$<string of 172 bytes>",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$<string of 172 bytes>",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$<string of 172 bytes>",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type":
"TimestampCoder$gAJjYXBhY2hlX2JlYW0uY29kZXJzLmNvZGVycwpUaW1lc3RhbXBDb2RlcgpxACmBcQF9cQJiLg==",
"component_encodings": []
},
{
"@type": "SingletonCoder$<string of 344 bytes>",
"component_encodings": []
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/WriteImpl/FlatMap(<lambda at
iobase.py:1037>).out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s13"
},
"serialized_fn": "<string of 1316 bytes>",
"user_name": "write/WriteImpl/FlatMap(<lambda at iobase.py:1037>)"
}
},
{
"kind": "CollectionToSingleton",
"name": "s15",
"properties": {
"output_info": [
{
"encoding": {
"@type": "WindowedValueCoder$<string of 736 bytes>",
"component_encodings": [
{
"@type": "WindowedValueCoder$<string of 736 bytes>",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$<string of 172 bytes>",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$<string of 172 bytes>",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$<string of 172 bytes>",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type":
"TimestampCoder$gAJjYXBhY2hlX2JlYW0uY29kZXJzLmNvZGVycwpUaW1lc3RhbXBDb2RlcgpxACmBcQF9cQJiLg==",
"component_encodings": []
},
{
"@type": "SingletonCoder$<string of 344 bytes>",
"component_encodings": []
}
],
"is_wrapper": true
}
]
},
"output_name": "out",
"user_name":
"write/WriteImpl/ViewAsIterable(write|WriteImpl|FlatMap(<lambda at
iobase.py:1037>).None)/CreatePCollectionView.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s14"
},
"user_name":
"write/WriteImpl/ViewAsIterable(write|WriteImpl|FlatMap(<lambda at
iobase.py:1037>).None)/CreatePCollectionView"
}
},
{
"kind": "ParallelDo",
"name": "s16",
"properties": {
"non_parallel_inputs": {
"s15": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s15"
},
"s9": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s9"
}
},
"output_info": [
{
"encoding": {
"@type": "WindowedValueCoder$<string of 736 bytes>",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$<string of 172 bytes>",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$<string of 172 bytes>",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$<string of 172 bytes>",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type":
"TimestampCoder$gAJjYXBhY2hlX2JlYW0uY29kZXJzLmNvZGVycwpUaW1lc3RhbXBDb2RlcgpxACmBcQF9cQJiLg==",
"component_encodings": []
},
{
"@type": "SingletonCoder$<string of 344 bytes>",
"component_encodings": []
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/WriteImpl/finalize_write.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "<string of 2028 bytes>",
"user_name": "write/WriteImpl/finalize_write"
}
}
],
"type": "JOB_TYPE_BATCH"
}
INFO:root:Create job: <Job
id: u'2016-09-23_02_02_56-15049378165243260057'
projectId: u'apache-beam-testing'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
INFO:root:Created job with id: [2016-09-23_02_02_56-15049378165243260057]
INFO:root:To access the Dataflow monitoring console, please navigate to
https://console.developers.google.com/project/apache-beam-testing/dataflow/job/2016-09-23_02_02_56-15049378165243260057
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756488e92:
2016-09-23T09:02:58.194Z: JOB_MESSAGE_DETAILED: (789ec911a87c299): Checking
required Cloud APIs are enabled.
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756488fa8:
2016-09-23T09:02:58.472Z: JOB_MESSAGE_DEBUG: (789ec911a87c6d1): Combiner
lifting skipped for step write/WriteImpl/GroupByKey: GroupByKey not followed by
a combiner.
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756488faa:
2016-09-23T09:02:58.474Z: JOB_MESSAGE_DEBUG: (789ec911a87cbdf): Combiner
lifting skipped for step group: GroupByKey not followed by a combiner.
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756488fad:
2016-09-23T09:02:58.477Z: JOB_MESSAGE_DETAILED: (789ec911a87c0ed): Expanding
GroupByKey operations into optimizable parts.
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756488fb0:
2016-09-23T09:02:58.480Z: JOB_MESSAGE_DETAILED: (789ec911a87c5fb): Lifting
ValueCombiningMappingFns into MergeBucketsMappingFns
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756488fb8:
2016-09-23T09:02:58.488Z: JOB_MESSAGE_DETAILED: (789ec911a87c525): Annotating
graph with Autotuner information.
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756488fdb:
2016-09-23T09:02:58.523Z: JOB_MESSAGE_DETAILED: (789ec911a87c95d): Fusing
adjacent ParDo, Read, Write, and Flatten operations
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756488fdd:
2016-09-23T09:02:58.525Z: JOB_MESSAGE_DETAILED: (789ec911a87ce6b): Fusing
consumer split into read
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756488fdf:
2016-09-23T09:02:58.527Z: JOB_MESSAGE_DETAILED: (789ec911a87c379): Fusing
consumer group/Reify into pair_with_one
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756488fe1:
2016-09-23T09:02:58.529Z: JOB_MESSAGE_DETAILED: (789ec911a87c887): Fusing
consumer format into count
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756488fe4:
2016-09-23T09:02:58.532Z: JOB_MESSAGE_DETAILED: (789ec911a87cd95): Fusing
consumer write/WriteImpl/GroupByKey/GroupByWindow into
write/WriteImpl/GroupByKey/Read
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756488fe6:
2016-09-23T09:02:58.534Z: JOB_MESSAGE_DETAILED: (789ec911a87c2a3): Fusing
consumer write/WriteImpl/GroupByKey/Write into write/WriteImpl/GroupByKey/Reify
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756488fea:
2016-09-23T09:02:58.538Z: JOB_MESSAGE_DETAILED: (789ec911a87ccbf): Fusing
consumer write/WriteImpl/FlatMap(<lambda at iobase.py:1037>) into
write/WriteImpl/GroupByKey/GroupByWindow
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756488fec:
2016-09-23T09:02:58.540Z: JOB_MESSAGE_DETAILED: (789ec911a87c1cd): Fusing
consumer count into group/GroupByWindow
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756488fef:
2016-09-23T09:02:58.543Z: JOB_MESSAGE_DETAILED: (789ec911a87c6db): Fusing
consumer write/WriteImpl/WindowInto into write/WriteImpl/Map(<lambda at
iobase.py:1034>)
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756488ff1:
2016-09-23T09:02:58.545Z: JOB_MESSAGE_DETAILED: (789ec911a87cbe9): Fusing
consumer write/WriteImpl/GroupByKey/Reify into write/WriteImpl/WindowInto
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756488ff3:
2016-09-23T09:02:58.547Z: JOB_MESSAGE_DETAILED: (789ec911a87c0f7): Fusing
consumer write/WriteImpl/Map(<lambda at iobase.py:1034>) into
write/WriteImpl/write_bundles
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756488ff5:
2016-09-23T09:02:58.549Z: JOB_MESSAGE_DETAILED: (789ec911a87c605): Fusing
consumer pair_with_one into split
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756488ff8:
2016-09-23T09:02:58.552Z: JOB_MESSAGE_DETAILED: (789ec911a87cb13): Fusing
consumer group/GroupByWindow into group/Read
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756488ffa:
2016-09-23T09:02:58.554Z: JOB_MESSAGE_DETAILED: (789ec911a87c021): Fusing
consumer write/WriteImpl/write_bundles into format
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756488ffc:
2016-09-23T09:02:58.556Z: JOB_MESSAGE_DETAILED: (789ec911a87c52f): Fusing
consumer group/Write into group/Reify
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756489047:
2016-09-23T09:02:58.631Z: JOB_MESSAGE_DEBUG: (789ec911a87c6ef): Workflow config
is missing a default resource spec.
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756489049:
2016-09-23T09:02:58.633Z: JOB_MESSAGE_DETAILED: (789ec911a87cbfd): Adding
StepResource setup and teardown to workflow graph.
INFO:root:2016-09-23_02_02_56-15049378165243260057_000001575648907b:
2016-09-23T09:02:58.683Z: JOB_MESSAGE_DEBUG: (8d2c01aa5592b141): Adding
workflow start and stop steps.
INFO:root:2016-09-23_02_02_56-15049378165243260057_00000157564890ad:
2016-09-23T09:02:58.733Z: JOB_MESSAGE_DEBUG: (9a148e268a0d638d): Assigning
stage ids.
INFO:root:Job 2016-09-23_02_02_56-15049378165243260057 is in state
JOB_STATE_RUNNING
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756489119:
2016-09-23T09:02:58.841Z: JOB_MESSAGE_DEBUG: (153332145fbf97bf): Executing wait
step start2
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756489124:
2016-09-23T09:02:58.852Z: JOB_MESSAGE_BASIC: S01: (b7ad4ce0f09353b2): Executing
operation group/Create
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756489143:
2016-09-23T09:02:58.883Z: JOB_MESSAGE_DEBUG: (cbfbb2ae84185369): Executing
operation write/WriteImpl/DoOnce
INFO:root:2016-09-23_02_02_56-15049378165243260057_000001575648914f:
2016-09-23T09:02:58.895Z: JOB_MESSAGE_DEBUG: (d4ec96d284c3fbf8): Value
"write/WriteImpl/DoOnce.out" materialized.
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756489159:
2016-09-23T09:02:58.905Z: JOB_MESSAGE_BASIC: S04: (cbfbb2ae84185fd3): Executing
operation write/WriteImpl/initialize_write
INFO:root:2016-09-23_02_02_56-15049378165243260057_00000157564891ef:
2016-09-23T09:02:59.055Z: JOB_MESSAGE_DEBUG: (9b7d43f9eea6c6f0): Starting
worker pool setup.
INFO:root:2016-09-23_02_02_56-15049378165243260057_00000157564891f1:
2016-09-23T09:02:59.057Z: JOB_MESSAGE_BASIC: (9b7d43f9eea6c54a): Starting 1
workers...
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756489203:
2016-09-23T09:02:59.075Z: JOB_MESSAGE_DEBUG: (9d4594415f6b6be7): Value
"group/Session" materialized.
INFO:root:2016-09-23_02_02_56-15049378165243260057_000001575648920d:
2016-09-23T09:02:59.085Z: JOB_MESSAGE_BASIC: S02: (d4ec96d284c3fb10): Executing
operation read+split+pair_with_one+group/Reify+group/Write
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756494bd0:
2016-09-23T09:03:46.640Z: JOB_MESSAGE_ERROR: (bd06a59e2b64a314): Workflow
failed. Causes: (e7ef9da8cd66c7cc): Unable to bring up enough workers: minimum
1, actual 0.
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756494c0f:
2016-09-23T09:03:46.703Z: JOB_MESSAGE_DETAILED: (7277ab71f6c87c6d): Cleaning up.
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756494c2c:
2016-09-23T09:03:46.732Z: JOB_MESSAGE_DEBUG: (7277ab71f6c8768a): Starting
worker pool teardown.
INFO:root:2016-09-23_02_02_56-15049378165243260057_0000015756494c2f:
2016-09-23T09:03:46.735Z: JOB_MESSAGE_BASIC: (7277ab71f6c87d48): Stopping
worker pool...
INFO:root:2016-09-23_02_02_56-15049378165243260057_000001575649502f:
2016-09-23T09:03:47.759Z: JOB_MESSAGE_DEBUG: (7277ab71f6c87182): Tearing down
pending resources...
INFO:root:Job 2016-09-23_02_02_56-15049378165243260057 is in state
JOB_STATE_FAILED
Traceback (most recent call last):
File "/usr/lib/python2.7/runpy.py", line 162, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/usr/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File
"<https://builds.apache.org/job/beam_PostCommit_PythonVerify/ws/sdks/python/apache_beam/examples/wordcount.py";,>
line 107, in <module>
run()
File
"<https://builds.apache.org/job/beam_PostCommit_PythonVerify/ws/sdks/python/apache_beam/examples/wordcount.py";,>
line 98, in run
result = p.run()
File "apache_beam/pipeline.py", line 159, in run
return self.runner.run(self)
File "apache_beam/runners/dataflow_runner.py", line 188, in run
% getattr(self, 'last_error_msg', None), self.result)
apache_beam.runners.dataflow_runner.DataflowRuntimeException: Dataflow pipeline
failed:
(bd06a59e2b64a314): Workflow failed. Causes: (e7ef9da8cd66c7cc): Unable to
bring up enough workers: minimum 1, actual 0.
# Grep will exit with status 1 if success message was not found.
echo ">>> CHECKING JOB SUCCESS"
>>> CHECKING JOB SUCCESS
grep JOB_STATE_DONE job_output
Build step 'Execute shell' marked build as failure
