See
<https://builds.apache.org/job/beam_PostCommit_Python_Verify/2491/display/redirect?page=changes>
Changes:
[tgroh] Do not produce Unprocessed Inputs if all inputs were Processed
------------------------------------------
[...truncated 622.03 KB...]
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type":
"FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type":
"FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type":
"FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name":
"assert_that/Group/Map(_merge_tagged_vals_under_key).out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s11"
},
"serialized_fn": "<string of 1344 bytes>",
"user_name": "assert_that/Group/Map(_merge_tagged_vals_under_key)"
}
},
{
"kind": "ParallelDo",
"name": "s13",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "<lambda>"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type":
"FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type":
"FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type":
"FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Unkey.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s12"
},
"serialized_fn": "<string of 980 bytes>",
"user_name": "assert_that/Unkey"
}
},
{
"kind": "ParallelDo",
"name": "s14",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_equal"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type":
"FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type":
"FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type":
"FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Match.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s13"
},
"serialized_fn": "<string of 1128 bytes>",
"user_name": "assert_that/Match"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: u'2017-06-14T17:50:19.234856Z'
currentStateTime: u'1970-01-01T00:00:00Z'
id: u'2017-06-14_10_50_18-17653045414281085121'
location: u'global'
name: u'beamapp-jenkins-0614175004-728167'
projectId: u'apache-beam-testing'
stageStates: []
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2017-06-14_10_50_18-17653045414281085121]
root: INFO: To access the Dataflow monitoring console, please navigate to
https://console.developers.google.com/project/apache-beam-testing/dataflow/job/2017-06-14_10_50_18-17653045414281085121
root: INFO: Job 2017-06-14_10_50_18-17653045414281085121 is in state
JOB_STATE_RUNNING
root: INFO: 2017-06-14T17:50:18.542Z: JOB_MESSAGE_WARNING: (f4fc374642100bd3):
Setting the number of workers (1) disables autoscaling for this job. If you are
trying to cap autoscaling, consider only setting max_num_workers. If you want
to disable autoscaling altogether, the documented way is to explicitly use
autoscalingAlgorithm=NONE.
root: INFO: 2017-06-14T17:50:21.087Z: JOB_MESSAGE_DETAILED: (4ea7ac02ecbdb7c7):
Checking required Cloud APIs are enabled.
root: INFO: 2017-06-14T17:50:24.380Z: JOB_MESSAGE_DEBUG: (4ea7ac02ecbdb13e):
Combiner lifting skipped for step assert_that/Group/GroupByKey: GroupByKey not
followed by a combiner.
root: INFO: 2017-06-14T17:50:24.382Z: JOB_MESSAGE_DETAILED: (4ea7ac02ecbdb58c):
Expanding GroupByKey operations into optimizable parts.
root: INFO: 2017-06-14T17:50:24.385Z: JOB_MESSAGE_DETAILED: (4ea7ac02ecbdb9da):
Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2017-06-14T17:50:24.389Z: JOB_MESSAGE_DEBUG: (4ea7ac02ecbdb276):
Annotating graph with Autotuner information.
root: INFO: 2017-06-14T17:50:24.399Z: JOB_MESSAGE_DETAILED: (4ea7ac02ecbdbb12):
Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2017-06-14T17:50:24.401Z: JOB_MESSAGE_DETAILED: (4ea7ac02ecbdbf60):
Unzipping flatten s10 for input s8.out
root: INFO: 2017-06-14T17:50:24.403Z: JOB_MESSAGE_DETAILED: (4ea7ac02ecbdb3ae):
Fusing unzipped copy of assert_that/Group/GroupByKey/Reify, through flatten ,
into producer assert_that/Group/pair_with_0
root: INFO: 2017-06-14T17:50:24.406Z: JOB_MESSAGE_DETAILED: (4ea7ac02ecbdb7fc):
Fusing consumer assert_that/Group/Map(_merge_tagged_vals_under_key) into
assert_that/Group/GroupByKey/GroupByWindow
root: INFO: 2017-06-14T17:50:24.408Z: JOB_MESSAGE_DETAILED: (4ea7ac02ecbdbc4a):
Fusing consumer assert_that/Match into assert_that/Unkey
root: INFO: 2017-06-14T17:50:24.410Z: JOB_MESSAGE_DETAILED: (4ea7ac02ecbdb098):
Fusing consumer assert_that/Unkey into
assert_that/Group/Map(_merge_tagged_vals_under_key)
root: INFO: 2017-06-14T17:50:24.414Z: JOB_MESSAGE_DETAILED: (4ea7ac02ecbdb934):
Fusing consumer assert_that/Group/GroupByKey/GroupByWindow into
assert_that/Group/GroupByKey/Read
root: INFO: 2017-06-14T17:50:24.416Z: JOB_MESSAGE_DETAILED: (4ea7ac02ecbdbd82):
Unzipping flatten s10-u13 for input s11-reify-value0-c11
root: INFO: 2017-06-14T17:50:24.420Z: JOB_MESSAGE_DETAILED: (4ea7ac02ecbdb1d0):
Fusing unzipped copy of assert_that/Group/GroupByKey/Write, through flatten ,
into producer assert_that/Group/GroupByKey/Reify
root: INFO: 2017-06-14T17:50:24.422Z: JOB_MESSAGE_DETAILED: (4ea7ac02ecbdb61e):
Fusing consumer assert_that/Group/GroupByKey/Reify into
assert_that/Group/pair_with_1
root: INFO: 2017-06-14T17:50:24.424Z: JOB_MESSAGE_DETAILED: (4ea7ac02ecbdba6c):
Fusing consumer assert_that/Group/GroupByKey/Write into
assert_that/Group/GroupByKey/Reify
root: INFO: 2017-06-14T17:50:24.427Z: JOB_MESSAGE_DETAILED: (4ea7ac02ecbdbeba):
Fusing consumer assert_that/Group/pair_with_0 into assert_that/Create/Read
root: INFO: 2017-06-14T17:50:24.429Z: JOB_MESSAGE_DETAILED: (4ea7ac02ecbdb308):
Fusing consumer assert_that/ToVoidKey into assert_that/WindowInto(WindowIntoFn)
root: INFO: 2017-06-14T17:50:24.431Z: JOB_MESSAGE_DETAILED: (4ea7ac02ecbdb756):
Fusing consumer assert_that/Group/pair_with_1 into assert_that/ToVoidKey
root: INFO: 2017-06-14T17:50:24.433Z: JOB_MESSAGE_DETAILED: (4ea7ac02ecbdbba4):
Fusing consumer FlatMap(<lambda at sideinputs_test.py:164>)/Do into start/Read
root: INFO: 2017-06-14T17:50:24.435Z: JOB_MESSAGE_DETAILED: (4ea7ac02ecbdbff2):
Fusing consumer assert_that/WindowInto(WindowIntoFn) into FlatMap(<lambda at
sideinputs_test.py:164>)/Do
root: INFO: 2017-06-14T17:50:24.491Z: JOB_MESSAGE_DEBUG: (4ea7ac02ecbdb742):
Workflow config is missing a default resource spec.
root: INFO: 2017-06-14T17:50:24.493Z: JOB_MESSAGE_DETAILED: (4ea7ac02ecbdbb90):
Adding StepResource setup and teardown to workflow graph.
root: INFO: 2017-06-14T17:50:24.495Z: JOB_MESSAGE_DEBUG: (4ea7ac02ecbdbfde):
Adding workflow start and stop steps.
root: INFO: 2017-06-14T17:50:24.498Z: JOB_MESSAGE_DEBUG: (4ea7ac02ecbdb42c):
Assigning stage ids.
root: INFO: 2017-06-14T17:50:24.534Z: JOB_MESSAGE_DEBUG: (1156b8c88bcdb493):
Executing wait step start21
root: INFO: 2017-06-14T17:50:24.542Z: JOB_MESSAGE_BASIC: (1156b8c88bcdbaa8):
Executing operation side/Read
root: INFO: 2017-06-14T17:50:24.552Z: JOB_MESSAGE_DEBUG: (1156b8c88bcdbb8e):
Value "side/Read.out" materialized.
root: INFO: 2017-06-14T17:50:24.559Z: JOB_MESSAGE_BASIC: (1156b8c88bcdb1a3):
Executing operation FlatMap(<lambda at sideinputs_test.py:164>)/SideInput-s3
root: INFO: 2017-06-14T17:50:24.569Z: JOB_MESSAGE_DEBUG: (1156b8c88bcdb289):
Value "FlatMap(<lambda at sideinputs_test.py:164>)/SideInput-s3.output"
materialized.
root: INFO: 2017-06-14T17:50:24.577Z: JOB_MESSAGE_BASIC: (1156b8c88bcdb36f):
Executing operation assert_that/Group/GroupByKey/Create
root: INFO: 2017-06-14T17:50:24.781Z: JOB_MESSAGE_DEBUG: (28c88755d68df20b):
Starting worker pool setup.
root: INFO: 2017-06-14T17:50:24.783Z: JOB_MESSAGE_BASIC: (28c88755d68dfba5):
Starting 1 workers in us-central1-f...
root: INFO: 2017-06-14T17:50:24.798Z: JOB_MESSAGE_DEBUG: (1156b8c88bcdb4c8):
Value "assert_that/Group/GroupByKey/Session" materialized.
root: INFO: 2017-06-14T17:50:24.809Z: JOB_MESSAGE_BASIC: (1156b8c88bcdbb50):
Executing operation start/Read+FlatMap(<lambda at
sideinputs_test.py:164>)/Do+assert_that/WindowInto(WindowIntoFn)+assert_that/ToVoidKey+assert_that/Group/pair_with_1+assert_that/Group/GroupByKey/Reify+assert_that/Group/GroupByKey/Write
root: INFO: 2017-06-14T17:50:24.813Z: JOB_MESSAGE_BASIC: (6249d3d4d2ac3361):
Executing operation
assert_that/Create/Read+assert_that/Group/pair_with_0+assert_that/Group/GroupByKey/Reify+assert_that/Group/GroupByKey/Write
root: INFO: 2017-06-14T17:51:41.338Z: JOB_MESSAGE_ERROR: (93a1b3ac8779545a):
Workflow failed. Causes: (cddb28e57cb5e5f8): Unable to bring up enough workers:
minimum 1, actual 0. Please check your quota and retry later, or please try in
a different zone.
root: INFO: 2017-06-14T17:51:41.408Z: JOB_MESSAGE_DETAILED: (4ea7ac02ecbdb9b2):
Cleaning up.
root: INFO: 2017-06-14T17:51:41.413Z: JOB_MESSAGE_DEBUG: (4ea7ac02ecbdbe00):
Starting worker pool teardown.
root: INFO: 2017-06-14T17:51:41.416Z: JOB_MESSAGE_BASIC: (4ea7ac02ecbdb24e):
Stopping worker pool...
root: INFO: 2017-06-14T17:51:41.467Z: JOB_MESSAGE_DEBUG: (4ea7ac02ecbdb5ad):
Tearing down pending resources...
root: INFO: Job 2017-06-14_10_50_18-17653045414281085121 is in state
JOB_STATE_FAILED
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 15 tests in 1423.154s
FAILED (errors=1)
Found:
https://console.cloud.google.com/dataflow/job/2017-06-14_10_38_42-11582084412979565074?project=apache-beam-testing
Found:
https://console.cloud.google.com/dataflow/job/2017-06-14_10_43_59-7870595605773836859?project=apache-beam-testing
Found:
https://console.cloud.google.com/dataflow/job/2017-06-14_10_50_10-12017267651331563226?project=apache-beam-testing
Found:
https://console.cloud.google.com/dataflow/job/2017-06-14_10_56_29-2116107037414547436?project=apache-beam-testing
Found:
https://console.cloud.google.com/dataflow/job/2017-06-14_10_38_44-11344921718106488256?project=apache-beam-testing
Found:
https://console.cloud.google.com/dataflow/job/2017-06-14_10_44_59-12437360482989441282?project=apache-beam-testing
Found:
https://console.cloud.google.com/dataflow/job/2017-06-14_10_50_54-8903325815103365168?project=apache-beam-testing
Found:
https://console.cloud.google.com/dataflow/job/2017-06-14_10_38_44-3199157406047746005?project=apache-beam-testing
Found:
https://console.cloud.google.com/dataflow/job/2017-06-14_10_45_14-6917862492667887344?project=apache-beam-testing
Found:
https://console.cloud.google.com/dataflow/job/2017-06-14_10_50_14-13502837550833143976?project=apache-beam-testing
Found:
https://console.cloud.google.com/dataflow/job/2017-06-14_10_55_59-11865453306001573614?project=apache-beam-testing
Found:
https://console.cloud.google.com/dataflow/job/2017-06-14_10_38_44-13634196707725776842?project=apache-beam-testing
Found:
https://console.cloud.google.com/dataflow/job/2017-06-14_10_44_48-15249063041001838132?project=apache-beam-testing
Found:
https://console.cloud.google.com/dataflow/job/2017-06-14_10_50_18-17653045414281085121?project=apache-beam-testing
Found:
https://console.cloud.google.com/dataflow/job/2017-06-14_10_52_02-5370887557230411542?project=apache-beam-testing
Build step 'Execute shell' marked build as failure
