See
<https://builds.apache.org/job/beam_PostCommit_Python_Verify/1566/display/redirect>
------------------------------------------
[...truncated 673.14 KB...]
"value": "<lambda>"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type":
"FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type":
"FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type":
"FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/Extract.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s13"
},
"serialized_fn": "<string of 744 bytes>",
"user_name": "write/Write/WriteImpl/Extract"
}
},
{
"kind": "CollectionToSingleton",
"name": "s15",
"properties": {
"display_data": [],
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type":
"FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type":
"FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type":
"FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
}
]
},
"output_name": "out",
"user_name":
"write/Write/WriteImpl/ViewAsIterable(write|Write|WriteImpl|Extract.None)/CreatePCollectionView.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s14"
},
"user_name":
"write/Write/WriteImpl/ViewAsIterable(write|Write|WriteImpl|Extract.None)/CreatePCollectionView"
}
},
{
"kind": "ParallelDo",
"name": "s16",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_finalize_write"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {
"s15": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s15"
},
"s9": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s9"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type":
"FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type":
"FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type":
"FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/finalize_write.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "<string of 996 bytes>",
"user_name": "write/Write/WriteImpl/finalize_write"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: u'2017-03-21T09:34:05.514525Z'
currentStateTime: u'1970-01-01T00:00:00Z'
id: u'2017-03-21_02_34_05-16756345099870542426'
location: u'global'
name: u'py-wordcount-1490087238'
projectId: u'apache-beam-testing'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2017-03-21_02_34_05-16756345099870542426]
root: INFO: To access the Dataflow monitoring console, please navigate to
https://console.developers.google.com/project/apache-beam-testing/dataflow/job/2017-03-21_02_34_05-16756345099870542426
root: INFO: Job 2017-03-21_02_34_05-16756345099870542426 is in state
JOB_STATE_RUNNING
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af0375f58:
2017-03-21T09:34:05.144Z: JOB_MESSAGE_WARNING: (e88a7f16e3a41426): Setting the
number of workers (1) disables autoscaling for this job. If you are trying to
cap autoscaling, consider only setting max_num_workers. If you want to disable
autoscaling altogether, the documented way is to explicitly use
autoscalingAlgorithm=NONE.
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af0376373:
2017-03-21T09:34:06.195Z: JOB_MESSAGE_DETAILED: (122a718d203c694): Checking
required Cloud APIs are enabled.
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af0376942:
2017-03-21T09:34:07.682Z: JOB_MESSAGE_DEBUG: (122a718d203c43a): Combiner
lifting skipped for step write/Write/WriteImpl/GroupByKey: GroupByKey not
followed by a combiner.
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af0376945:
2017-03-21T09:34:07.685Z: JOB_MESSAGE_DEBUG: (122a718d203ca28): Combiner
lifting skipped for step group: GroupByKey not followed by a combiner.
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af0376947:
2017-03-21T09:34:07.687Z: JOB_MESSAGE_DETAILED: (122a718d203c016): Expanding
GroupByKey operations into optimizable parts.
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af037694a:
2017-03-21T09:34:07.690Z: JOB_MESSAGE_DETAILED: (122a718d203c604): Lifting
ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af0376951:
2017-03-21T09:34:07.697Z: JOB_MESSAGE_DETAILED: (122a718d203c7ce): Annotating
graph with Autotuner information.
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af037695c:
2017-03-21T09:34:07.708Z: JOB_MESSAGE_DETAILED: (122a718d203c6a1): Fusing
adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af037695f:
2017-03-21T09:34:07.711Z: JOB_MESSAGE_DETAILED: (122a718d203cc8f): Fusing
consumer split into read/Read
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af0376961:
2017-03-21T09:34:07.713Z: JOB_MESSAGE_DETAILED: (122a718d203c27d): Fusing
consumer group/Reify into pair_with_one
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af0376963:
2017-03-21T09:34:07.715Z: JOB_MESSAGE_DETAILED: (122a718d203c86b): Fusing
consumer format into count
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af0376965:
2017-03-21T09:34:07.717Z: JOB_MESSAGE_DETAILED: (122a718d203ce59): Fusing
consumer write/Write/WriteImpl/GroupByKey/GroupByWindow into
write/Write/WriteImpl/GroupByKey/Read
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af0376968:
2017-03-21T09:34:07.720Z: JOB_MESSAGE_DETAILED: (122a718d203c447): Fusing
consumer write/Write/WriteImpl/GroupByKey/Write into
write/Write/WriteImpl/GroupByKey/Reify
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af037696c:
2017-03-21T09:34:07.724Z: JOB_MESSAGE_DETAILED: (122a718d203c023): Fusing
consumer write/Write/WriteImpl/Extract into
write/Write/WriteImpl/GroupByKey/GroupByWindow
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af037696e:
2017-03-21T09:34:07.726Z: JOB_MESSAGE_DETAILED: (122a718d203c611): Fusing
consumer count into group/GroupByWindow
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af0376970:
2017-03-21T09:34:07.728Z: JOB_MESSAGE_DETAILED: (122a718d203cbff): Fusing
consumer write/Write/WriteImpl/WindowInto(WindowIntoFn) into
write/Write/WriteImpl/Pair
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af0376972:
2017-03-21T09:34:07.730Z: JOB_MESSAGE_DETAILED: (122a718d203c1ed): Fusing
consumer write/Write/WriteImpl/GroupByKey/Reify into
write/Write/WriteImpl/WindowInto(WindowIntoFn)
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af0376974:
2017-03-21T09:34:07.732Z: JOB_MESSAGE_DETAILED: (122a718d203c7db): Fusing
consumer write/Write/WriteImpl/Pair into write/Write/WriteImpl/WriteBundles
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af0376976:
2017-03-21T09:34:07.734Z: JOB_MESSAGE_DETAILED: (122a718d203cdc9): Fusing
consumer pair_with_one into split
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af0376978:
2017-03-21T09:34:07.736Z: JOB_MESSAGE_DETAILED: (122a718d203c3b7): Fusing
consumer group/GroupByWindow into group/Read
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af037697a:
2017-03-21T09:34:07.738Z: JOB_MESSAGE_DETAILED: (122a718d203c9a5): Fusing
consumer write/Write/WriteImpl/WriteBundles into format
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af037697d:
2017-03-21T09:34:07.741Z: JOB_MESSAGE_DETAILED: (122a718d203cf93): Fusing
consumer group/Write into group/Reify
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af03769c2:
2017-03-21T09:34:07.810Z: JOB_MESSAGE_DEBUG: (122a718d203cd53): Workflow config
is missing a default resource spec.
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af03769c5:
2017-03-21T09:34:07.813Z: JOB_MESSAGE_DETAILED: (122a718d203c341): Adding
StepResource setup and teardown to workflow graph.
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af03769c7:
2017-03-21T09:34:07.815Z: JOB_MESSAGE_DEBUG: (122a718d203c92f): Adding workflow
start and stop steps.
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af03769c9:
2017-03-21T09:34:07.817Z: JOB_MESSAGE_DEBUG: (122a718d203cf1d): Assigning stage
ids.
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af03769ec:
2017-03-21T09:34:07.852Z: JOB_MESSAGE_DEBUG: (f7505cd4fdf14a1e): Executing wait
step start25
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af03769f3:
2017-03-21T09:34:07.859Z: JOB_MESSAGE_BASIC: S01: (f7505cd4fdf14c00): Executing
operation group/Create
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af03769f6:
2017-03-21T09:34:07.862Z: JOB_MESSAGE_DEBUG: (3cf8a2e23e213aaf): Executing
operation write/Write/WriteImpl/DoOnce
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af0376a00:
2017-03-21T09:34:07.872Z: JOB_MESSAGE_DEBUG: (3cf8a2e23e2139b8): Value
"write/Write/WriteImpl/DoOnce.out" materialized.
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af0376a08:
2017-03-21T09:34:07.880Z: JOB_MESSAGE_BASIC: S04: (3cf8a2e23e213d30): Executing
operation write/Write/WriteImpl/InitializeWrite
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af0376ac0:
2017-03-21T09:34:08.064Z: JOB_MESSAGE_DEBUG: (e24ae0e724ed7675): Starting
worker pool setup.
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af0376ac2:
2017-03-21T09:34:08.066Z: JOB_MESSAGE_BASIC: (e24ae0e724ed7d4f): Starting 1
workers...
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af0376ace:
2017-03-21T09:34:08.078Z: JOB_MESSAGE_DEBUG: (f7505cd4fdf146d3): Value
"group/Session" materialized.
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af0376ad6:
2017-03-21T09:34:08.086Z: JOB_MESSAGE_BASIC: S02: (f7505cd4fdf14eab): Executing
operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af0385044:
2017-03-21T09:35:06.820Z: JOB_MESSAGE_DETAILED: (cc6e741ab231f7e6): Workers
have started successfully.
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af03a7e5f:
2017-03-21T09:37:29.695Z: JOB_MESSAGE_DEBUG: (3cf8a2e23e2132f7): Value
"write/Write/WriteImpl/InitializeWrite.out" materialized.
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af03a7e66:
2017-03-21T09:37:29.702Z: JOB_MESSAGE_BASIC: S05: (3cf8a2e23e213200): Executing
operation
write/Write/WriteImpl/ViewAsSingleton(write|Write|WriteImpl|InitializeWrite.None)/CreatePCollectionView
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af03a7eba:
2017-03-21T09:37:29.786Z: JOB_MESSAGE_DEBUG: (3cf8a2e23e2132c5): Value
"write/Write/WriteImpl/ViewAsSingleton(write|Write|WriteImpl|InitializeWrite.None)/CreatePCollectionView.out"
materialized.
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af03a89ba:
2017-03-21T09:37:32.602Z: JOB_MESSAGE_BASIC: S03: (f7505cd4fdf1426f): Executing
operation group/Close
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af03a89cf:
2017-03-21T09:37:32.623Z: JOB_MESSAGE_BASIC: S06: (f7505cd4fdf14d42): Executing
operation write/Write/WriteImpl/GroupByKey/Create
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af03a8a7f:
2017-03-21T09:37:32.799Z: JOB_MESSAGE_DEBUG: (f7505cd4fdf14815): Value
"write/Write/WriteImpl/GroupByKey/Session" materialized.
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af03a8a88:
2017-03-21T09:37:32.808Z: JOB_MESSAGE_BASIC: S07: (f7505cd4fdf14cf2): Executing
operation
group/Read+group/GroupByWindow+count+format+write/Write/WriteImpl/WriteBundles+write/Write/WriteImpl/Pair+write/Write/WriteImpl/WindowInto(WindowIntoFn)+write/Write/WriteImpl/GroupByKey/Reify+write/Write/WriteImpl/GroupByKey/Write
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af03ac137:
2017-03-21T09:37:46.807Z: JOB_MESSAGE_BASIC: S08: (f7505cd4fdf14e84): Executing
operation write/Write/WriteImpl/GroupByKey/Close
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af03ac14a:
2017-03-21T09:37:46.826Z: JOB_MESSAGE_BASIC: S09: (f7505cd4fdf1465c): Executing
operation
write/Write/WriteImpl/GroupByKey/Read+write/Write/WriteImpl/GroupByKey/GroupByWindow+write/Write/WriteImpl/Extract
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af03af135:
2017-03-21T09:37:59.093Z: JOB_MESSAGE_DEBUG: (f7505cd4fdf142c1): Value
"write/Write/WriteImpl/Extract.out" materialized.
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af03af13c:
2017-03-21T09:37:59.100Z: JOB_MESSAGE_BASIC: S10: (f7505cd4fdf1479e): Executing
operation
write/Write/WriteImpl/ViewAsIterable(write|Write|WriteImpl|Extract.None)/CreatePCollectionView
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af03af37a:
2017-03-21T09:37:59.674Z: JOB_MESSAGE_DEBUG: (f7505cd4fdf1456c): Value
"write/Write/WriteImpl/ViewAsIterable(write|Write|WriteImpl|Extract.None)/CreatePCollectionView.out"
materialized.
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af03af382:
2017-03-21T09:37:59.682Z: JOB_MESSAGE_BASIC: S11: (f7505cd4fdf14a49): Executing
operation write/Write/WriteImpl/finalize_write
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af03b0f69:
2017-03-21T09:38:06.825Z: JOB_MESSAGE_DEBUG: (f7505cd4fdf14ed6): Executing
success step success23
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af03b0fad:
2017-03-21T09:38:06.893Z: JOB_MESSAGE_DETAILED: (122a718d203cfba): Cleaning up.
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af03b102a:
2017-03-21T09:38:07.018Z: JOB_MESSAGE_DEBUG: (122a718d203c5a8): Starting worker
pool teardown.
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af03b102c:
2017-03-21T09:38:07.020Z: JOB_MESSAGE_BASIC: (122a718d203cb96): Stopping worker
pool...
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af03c352b:
2017-03-21T09:39:22.027Z: JOB_MESSAGE_BASIC: (122a718d203c47b): Worker pool
stopped.
root: INFO: 2017-03-21_02_34_05-16756345099870542426_0000015af03c3549:
2017-03-21T09:39:22.057Z: JOB_MESSAGE_DEBUG: (122a718d203c645): Tearing down
pending resources...
root: INFO: Job 2017-03-21_02_34_05-16756345099870542426 is in state
JOB_STATE_DONE
root: INFO: Read from given path
gs://temp-storage-for-end-to-end-tests/py-wordcount-cloud/output/py-wordcount-1490087238/results*-of-*,
3179 lines, checksum: a9bcb4acd65daf8f6a9ac5e026de7803cc09f662.
root: INFO: Read from given path
gs://temp-storage-for-end-to-end-tests/py-wordcount-cloud/output/py-wordcount-1490087238/results*-of-*,
4784 lines, checksum: 33535a832b7db6d78389759577d4ff495980b9c0.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 2 tests in 668.588s
FAILED (failures=1)
Build step 'Execute shell' marked build as failure
