See
<https://builds.apache.org/job/beam_PostCommit_Python36/1089/display/redirect>
Changes:
------------------------------------------
[...truncated 107.39 KB...]
response = self.worker.do_instruction(request)
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/sdks/python/apache_beam/runners/worker/sdk_worker.py";,>
line 286, in do_instruction
request.instruction_id)
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/sdks/python/apache_beam/runners/worker/sdk_worker.py";,>
line 312, in process_bundle
bundle_processor.process_bundle(instruction_id))
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/sdks/python/apache_beam/runners/worker/bundle_processor.py";,>
line 680, in process_bundle
op.finish()
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/sdks/python/apache_beam/runners/worker/operations.py";,>
line 616, in finish
self.dofn_runner.finish()
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/sdks/python/apache_beam/runners/common.py";,>
line 856, in finish
self._invoke_bundle_method(self.do_fn_invoker.invoke_finish_bundle)
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/sdks/python/apache_beam/runners/common.py";,>
line 840, in _invoke_bundle_method
self._reraise_augmented(exn)
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/sdks/python/apache_beam/runners/common.py";,>
line 881, in _reraise_augmented
raise_with_traceback(new_exn)
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/build/gradleenv/1398941890/lib/python3.6/site-packages/future/utils/__init__.py";,>
line 446, in raise_with_traceback
raise exc.with_traceback(traceback)
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/sdks/python/apache_beam/runners/common.py";,>
line 838, in _invoke_bundle_method
bundle_method()
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/sdks/python/apache_beam/runners/common.py";,>
line 409, in invoke_finish_bundle
self.signature.finish_bundle_method.method_value())
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/sdks/python/apache_beam/runners/common.py";,>
line 976, in finish_bundle_outputs
for result in results:
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/sdks/python/apache_beam/io/iobase.py";,>
line 1051, in finish_bundle
yield WindowedValue(self.writer.close(),
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/sdks/python/apache_beam/io/filebasedsink.py";,>
line 399, in close
self.sink.close(self.temp_handle)
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/sdks/python/apache_beam/io/filebasedsink.py";,>
line 155, in close
file_handle.close()
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/sdks/python/apache_beam/io/filesystemio.py";,>
line 217, in close
self._uploader.finish()
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/sdks/python/apache_beam/io/gcp/gcsio.py";,>
line 591, in finish
raise self._upload_thread.last_error # pylint: disable=raising-bad-type
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/sdks/python/apache_beam/io/gcp/gcsio.py";,>
line 568, in _start_upload
self._client.objects.Insert(self._insert_request, upload=self._upload)
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py";,>
line 1156, in Insert
upload=upload, upload_config=upload_config)
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/build/gradleenv/1398941890/lib/python3.6/site-packages/apitools/base/py/base_api.py";,>
line 715, in _RunMethod
http_request, client=self.client)
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/build/gradleenv/1398941890/lib/python3.6/site-packages/apitools/base/py/transfer.py";,>
line 908, in InitializeUpload
return self.StreamInChunks()
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/build/gradleenv/1398941890/lib/python3.6/site-packages/apitools/base/py/transfer.py";,>
line 1020, in StreamInChunks
additional_headers=additional_headers)
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/build/gradleenv/1398941890/lib/python3.6/site-packages/apitools/base/py/transfer.py";,>
line 971, in __StreamMedia
self.RefreshResumableUploadState()
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/build/gradleenv/1398941890/lib/python3.6/site-packages/apitools/base/py/transfer.py";,>
line 875, in RefreshResumableUploadState
raise exceptions.HttpError.FromResponse(refresh_response)
RuntimeError: apitools.base.py.exceptions.HttpError: HttpError accessing
<https://www.googleapis.com/resumable/upload/storage/v1/b/temp-storage-for-end-to-end-tests/o?alt=json&name=py-it-cloud%2Foutput%2F1574813463545%2Fbeam-temp-results-678bb78610aa11ea8fd542010a8000a5%2F71448830-ba96-4de2-b6e5-2b72f388cceb.results&uploadType=resumable&upload_id=AEnB2UqduGnn-MrJq8b8sZJU8blP4yaFHPmrZ-jm07AuVU_i1LyCtHBekeTUonMNpYl-tCsWldmMC-FSYD84VVkUByb7JF7jbg>:
response: <{'x-guploader-uploadid':
'AEnB2UqduGnn-MrJq8b8sZJU8blP4yaFHPmrZ-jm07AuVU_i1LyCtHBekeTUonMNpYl-tCsWldmMC-FSYD84VVkUByb7JF7jbg',
'vary': 'Origin, X-Origin', 'content-type': 'application/json; charset=UTF-8',
'content-length': '177', 'date': 'Wed, 27 Nov 2019 00:11:16 GMT', 'server':
'UploadServer', 'status': '410'}>, content <{
"error": {
"errors": [
{
"domain": "global",
"reason": "backendError",
"message": "Backend Error"
}
],
"code": 503,
"message": "Backend Error"
}
}
> [while running 'write/Write/WriteImpl/WriteBundles']
-------------------- >> begin captured logging << --------------------
apache_beam.internal.gcp.auth: INFO: Setting socket default timeout to 60
seconds.
apache_beam.internal.gcp.auth: INFO: socket default timeout is 60.0 seconds.
root: DEBUG: Connecting using Google Application Default Credentials.
oauth2client.transport: INFO: Attempting refresh to obtain initial access_token
apache_beam.io.filesystem: DEBUG: translate_pattern:
'gs://dataflow-samples/shakespeare/kinglear.txt' ->
'gs\\:\\/\\/dataflow\\-samples\\/shakespeare\\/kinglear\\.txt'
root: DEBUG: Unhandled type_constraint: Union[]
apache_beam.runners.portability.fn_api_runner_transforms: INFO:
==================== <function annotate_downstream_side_inputs at
0x7fe8b41aa268> ====================
apache_beam.runners.portability.fn_api_runner_transforms: DEBUG: 15 [1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
apache_beam.runners.portability.fn_api_runner_transforms: DEBUG: Stages:
['ref_AppliedPTransform_read/Read_3\n read/Read:beam:transform:read:v1\n must
follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18', 'ref_AppliedPTransform_split_4\n
split:beam:transform:pardo:v1\n must follow: \n downstream_side_inputs:
ref_PCollection_PCollection_17, ref_PCollection_PCollection_18',
'ref_AppliedPTransform_pair_with_one_5\n
pair_with_one:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18', 'ref_AppliedPTransform_group_6\n
group:beam:transform:group_by_key:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18', 'ref_AppliedPTransform_count_10\n
count:beam:transform:pardo:v1\n must follow: \n downstream_side_inputs:
ref_PCollection_PCollection_17, ref_PCollection_PCollection_18',
'ref_AppliedPTransform_format_11\n format:beam:transform:pardo:v1\n must
follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16\n
write/Write/WriteImpl/DoOnce/Read:beam:transform:read:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18, ref_PCollection_PCollection_10',
'ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17\n
write/Write/WriteImpl/InitializeWrite:beam:transform:pardo:v1\n must follow:
\n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18, ref_PCollection_PCollection_10',
'ref_AppliedPTransform_write/Write/WriteImpl/WriteBundles_18\n
write/Write/WriteImpl/WriteBundles:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/Pair_19\n
write/Write/WriteImpl/Pair:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/WindowInto(WindowIntoFn)_20\n
write/Write/WriteImpl/WindowInto(WindowIntoFn):beam:transform:window_into:v1\n
must follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/GroupByKey_21\n
write/Write/WriteImpl/GroupByKey:beam:transform:group_by_key:v1\n must follow:
\n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/Extract_25\n
write/Write/WriteImpl/Extract:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/PreFinalize_26\n
write/Write/WriteImpl/PreFinalize:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/FinalizeWrite_27\n
write/Write/WriteImpl/FinalizeWrite:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ']
apache_beam.runners.portability.fn_api_runner_transforms: INFO:
==================== <function fix_side_input_pcoll_coders at 0x7fe8b41aa378>
====================
apache_beam.runners.portability.fn_api_runner_transforms: DEBUG: 15 [1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
apache_beam.runners.portability.fn_api_runner_transforms: DEBUG: Stages:
['ref_AppliedPTransform_read/Read_3\n read/Read:beam:transform:read:v1\n must
follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18', 'ref_AppliedPTransform_split_4\n
split:beam:transform:pardo:v1\n must follow: \n downstream_side_inputs:
ref_PCollection_PCollection_17, ref_PCollection_PCollection_18',
'ref_AppliedPTransform_pair_with_one_5\n
pair_with_one:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18', 'ref_AppliedPTransform_group_6\n
group:beam:transform:group_by_key:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18', 'ref_AppliedPTransform_count_10\n
count:beam:transform:pardo:v1\n must follow: \n downstream_side_inputs:
ref_PCollection_PCollection_17, ref_PCollection_PCollection_18',
'ref_AppliedPTransform_format_11\n format:beam:transform:pardo:v1\n must
follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16\n
write/Write/WriteImpl/DoOnce/Read:beam:transform:read:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18, ref_PCollection_PCollection_10',
'ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17\n
write/Write/WriteImpl/InitializeWrite:beam:transform:pardo:v1\n must follow:
\n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18, ref_PCollection_PCollection_10',
'ref_AppliedPTransform_write/Write/WriteImpl/WriteBundles_18\n
write/Write/WriteImpl/WriteBundles:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/Pair_19\n
write/Write/WriteImpl/Pair:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/WindowInto(WindowIntoFn)_20\n
write/Write/WriteImpl/WindowInto(WindowIntoFn):beam:transform:window_into:v1\n
must follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/GroupByKey_21\n
write/Write/WriteImpl/GroupByKey:beam:transform:group_by_key:v1\n must follow:
\n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/Extract_25\n
write/Write/WriteImpl/Extract:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/PreFinalize_26\n
write/Write/WriteImpl/PreFinalize:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/FinalizeWrite_27\n
write/Write/WriteImpl/FinalizeWrite:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ']
apache_beam.runners.portability.fn_api_runner_transforms: INFO:
==================== <function lift_combiners at 0x7fe8b41aa400>
====================
apache_beam.runners.portability.fn_api_runner_transforms: DEBUG: 15 [1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
apache_beam.runners.portability.fn_api_runner_transforms: DEBUG: Stages:
['ref_AppliedPTransform_read/Read_3\n read/Read:beam:transform:read:v1\n must
follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18', 'ref_AppliedPTransform_split_4\n
split:beam:transform:pardo:v1\n must follow: \n downstream_side_inputs:
ref_PCollection_PCollection_17, ref_PCollection_PCollection_18',
'ref_AppliedPTransform_pair_with_one_5\n
pair_with_one:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18', 'ref_AppliedPTransform_group_6\n
group:beam:transform:group_by_key:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18', 'ref_AppliedPTransform_count_10\n
count:beam:transform:pardo:v1\n must follow: \n downstream_side_inputs:
ref_PCollection_PCollection_17, ref_PCollection_PCollection_18',
'ref_AppliedPTransform_format_11\n format:beam:transform:pardo:v1\n must
follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16\n
write/Write/WriteImpl/DoOnce/Read:beam:transform:read:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18, ref_PCollection_PCollection_10',
'ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17\n
write/Write/WriteImpl/InitializeWrite:beam:transform:pardo:v1\n must follow:
\n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18, ref_PCollection_PCollection_10',
'ref_AppliedPTransform_write/Write/WriteImpl/WriteBundles_18\n
write/Write/WriteImpl/WriteBundles:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/Pair_19\n
write/Write/WriteImpl/Pair:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/WindowInto(WindowIntoFn)_20\n
write/Write/WriteImpl/WindowInto(WindowIntoFn):beam:transform:window_into:v1\n
must follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/GroupByKey_21\n
write/Write/WriteImpl/GroupByKey:beam:transform:group_by_key:v1\n must follow:
\n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/Extract_25\n
write/Write/WriteImpl/Extract:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/PreFinalize_26\n
write/Write/WriteImpl/PreFinalize:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/FinalizeWrite_27\n
write/Write/WriteImpl/FinalizeWrite:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ']
apache_beam.runners.portability.fn_api_runner_transforms: INFO:
==================== <function expand_sdf at 0x7fe8b41aa488>
====================
apache_beam.runners.portability.fn_api_runner_transforms: DEBUG: 15 [1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
apache_beam.runners.portability.fn_api_runner_transforms: DEBUG: Stages:
['ref_AppliedPTransform_read/Read_3\n read/Read:beam:transform:read:v1\n must
follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18', 'ref_AppliedPTransform_split_4\n
split:beam:transform:pardo:v1\n must follow: \n downstream_side_inputs:
ref_PCollection_PCollection_17, ref_PCollection_PCollection_18',
'ref_AppliedPTransform_pair_with_one_5\n
pair_with_one:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18', 'ref_AppliedPTransform_group_6\n
group:beam:transform:group_by_key:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18', 'ref_AppliedPTransform_count_10\n
count:beam:transform:pardo:v1\n must follow: \n downstream_side_inputs:
ref_PCollection_PCollection_17, ref_PCollection_PCollection_18',
'ref_AppliedPTransform_format_11\n format:beam:transform:pardo:v1\n must
follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16\n
write/Write/WriteImpl/DoOnce/Read:beam:transform:read:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18, ref_PCollection_PCollection_10',
'ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17\n
write/Write/WriteImpl/InitializeWrite:beam:transform:pardo:v1\n must follow:
\n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18, ref_PCollection_PCollection_10',
'ref_AppliedPTransform_write/Write/WriteImpl/WriteBundles_18\n
write/Write/WriteImpl/WriteBundles:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/Pair_19\n
write/Write/WriteImpl/Pair:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/WindowInto(WindowIntoFn)_20\n
write/Write/WriteImpl/WindowInto(WindowIntoFn):beam:transform:window_into:v1\n
must follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/GroupByKey_21\n
write/Write/WriteImpl/GroupByKey:beam:transform:group_by_key:v1\n must follow:
\n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/Extract_25\n
write/Write/WriteImpl/Extract:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/PreFinalize_26\n
write/Write/WriteImpl/PreFinalize:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/FinalizeWrite_27\n
write/Write/WriteImpl/FinalizeWrite:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ']
apache_beam.runners.portability.fn_api_runner_transforms: INFO:
==================== <function expand_gbk at 0x7fe8b41aa510>
====================
apache_beam.runners.portability.fn_api_runner_transforms: DEBUG: 17 [1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
apache_beam.runners.portability.fn_api_runner_transforms: DEBUG: Stages:
['ref_AppliedPTransform_read/Read_3\n read/Read:beam:transform:read:v1\n must
follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18', 'ref_AppliedPTransform_split_4\n
split:beam:transform:pardo:v1\n must follow: \n downstream_side_inputs:
ref_PCollection_PCollection_17, ref_PCollection_PCollection_18',
'ref_AppliedPTransform_pair_with_one_5\n
pair_with_one:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18', 'group/Write\n
group/Write:beam:sink:runner:0.1\n must follow: \n downstream_side_inputs: ',
'group/Read\n group/Read:beam:source:runner:0.1\n must follow: group/Write\n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18', 'ref_AppliedPTransform_count_10\n
count:beam:transform:pardo:v1\n must follow: \n downstream_side_inputs:
ref_PCollection_PCollection_17, ref_PCollection_PCollection_18',
'ref_AppliedPTransform_format_11\n format:beam:transform:pardo:v1\n must
follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16\n
write/Write/WriteImpl/DoOnce/Read:beam:transform:read:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18, ref_PCollection_PCollection_10',
'ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17\n
write/Write/WriteImpl/InitializeWrite:beam:transform:pardo:v1\n must follow:
\n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18, ref_PCollection_PCollection_10',
'ref_AppliedPTransform_write/Write/WriteImpl/WriteBundles_18\n
write/Write/WriteImpl/WriteBundles:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/Pair_19\n
write/Write/WriteImpl/Pair:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/WindowInto(WindowIntoFn)_20\n
write/Write/WriteImpl/WindowInto(WindowIntoFn):beam:transform:window_into:v1\n
must follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18', 'write/Write/WriteImpl/GroupByKey/Write\n
write/Write/WriteImpl/GroupByKey/Write:beam:sink:runner:0.1\n must follow: \n
downstream_side_inputs: ', 'write/Write/WriteImpl/GroupByKey/Read\n
write/Write/WriteImpl/GroupByKey/Read:beam:source:runner:0.1\n must follow:
write/Write/WriteImpl/GroupByKey/Write\n downstream_side_inputs:
ref_PCollection_PCollection_17, ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/Extract_25\n
write/Write/WriteImpl/Extract:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/PreFinalize_26\n
write/Write/WriteImpl/PreFinalize:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/FinalizeWrite_27\n
write/Write/WriteImpl/FinalizeWrite:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ']
apache_beam.runners.portability.fn_api_runner_transforms: INFO:
==================== <function sink_flattens at 0x7fe8b41aa620>
====================
apache_beam.runners.portability.fn_api_runner_transforms: DEBUG: 17 [1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
apache_beam.runners.portability.fn_api_runner_transforms: DEBUG: Stages:
['ref_AppliedPTransform_read/Read_3\n read/Read:beam:transform:read:v1\n must
follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18', 'ref_AppliedPTransform_split_4\n
split:beam:transform:pardo:v1\n must follow: \n downstream_side_inputs:
ref_PCollection_PCollection_17, ref_PCollection_PCollection_18',
'ref_AppliedPTransform_pair_with_one_5\n
pair_with_one:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18', 'group/Write\n
group/Write:beam:sink:runner:0.1\n must follow: \n downstream_side_inputs: ',
'group/Read\n group/Read:beam:source:runner:0.1\n must follow: group/Write\n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18', 'ref_AppliedPTransform_count_10\n
count:beam:transform:pardo:v1\n must follow: \n downstream_side_inputs:
ref_PCollection_PCollection_17, ref_PCollection_PCollection_18',
'ref_AppliedPTransform_format_11\n format:beam:transform:pardo:v1\n must
follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16\n
write/Write/WriteImpl/DoOnce/Read:beam:transform:read:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18, ref_PCollection_PCollection_10',
'ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17\n
write/Write/WriteImpl/InitializeWrite:beam:transform:pardo:v1\n must follow:
\n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18, ref_PCollection_PCollection_10',
'ref_AppliedPTransform_write/Write/WriteImpl/WriteBundles_18\n
write/Write/WriteImpl/WriteBundles:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/Pair_19\n
write/Write/WriteImpl/Pair:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/WindowInto(WindowIntoFn)_20\n
write/Write/WriteImpl/WindowInto(WindowIntoFn):beam:transform:window_into:v1\n
must follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18', 'write/Write/WriteImpl/GroupByKey/Write\n
write/Write/WriteImpl/GroupByKey/Write:beam:sink:runner:0.1\n must follow: \n
downstream_side_inputs: ', 'write/Write/WriteImpl/GroupByKey/Read\n
write/Write/WriteImpl/GroupByKey/Read:beam:source:runner:0.1\n must follow:
write/Write/WriteImpl/GroupByKey/Write\n downstream_side_inputs:
ref_PCollection_PCollection_17, ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/Extract_25\n
write/Write/WriteImpl/Extract:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/PreFinalize_26\n
write/Write/WriteImpl/PreFinalize:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ref_PCollection_PCollection_18',
'ref_AppliedPTransform_write/Write/WriteImpl/FinalizeWrite_27\n
write/Write/WriteImpl/FinalizeWrite:beam:transform:pardo:v1\n must follow: \n
downstream_side_inputs: ']
apache_beam.runners.portability.fn_api_runner_transforms: INFO:
==================== <function greedily_fuse at 0x7fe8b41aa6a8>
====================
apache_beam.runners.portability.fn_api_runner_transforms: DEBUG: 6 [4, 3, 3, 2,
4, 7]
apache_beam.runners.portability.fn_api_runner_transforms: DEBUG: Stages:
['(((ref_AppliedPTransform_read/Read_3)+(ref_AppliedPTransform_split_4))+(ref_AppliedPTransform_pair_with_one_5))+(group/Write)\n
read/Read:beam:transform:read:v1\nsplit:beam:transform:pardo:v1\npair_with_one:beam:transform:pardo:v1\ngroup/Write:beam:sink:runner:0.1\n
must follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'((ref_PCollection_PCollection_9/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/PreFinalize_26))+(ref_PCollection_PCollection_18/Write)\n
ref_PCollection_PCollection_9/Read:beam:source:runner:0.1\nwrite/Write/WriteImpl/PreFinalize:beam:transform:pardo:v1\nref_PCollection_PCollection_18/Write:beam:sink:runner:0.1\n
must follow:
((write/Write/WriteImpl/GroupByKey/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/Extract_25))+(ref_PCollection_PCollection_17/Write),
(((ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16)+(ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17))+(ref_PCollection_PCollection_9/Write))+(ref_PCollection_PCollection_10/Write)\n
downstream_side_inputs: ref_PCollection_PCollection_18',
'((write/Write/WriteImpl/GroupByKey/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/Extract_25))+(ref_PCollection_PCollection_17/Write)\n
write/Write/WriteImpl/GroupByKey/Read:beam:source:runner:0.1\nwrite/Write/WriteImpl/Extract:beam:transform:pardo:v1\nref_PCollection_PCollection_17/Write:beam:sink:runner:0.1\n
must follow:
((((((group/Read)+(ref_AppliedPTransform_count_10))+(ref_AppliedPTransform_format_11))+(ref_AppliedPTransform_write/Write/WriteImpl/WriteBundles_18))+(ref_AppliedPTransform_write/Write/WriteImpl/Pair_19))+(ref_AppliedPTransform_write/Write/WriteImpl/WindowInto(WindowIntoFn)_20))+(write/Write/WriteImpl/GroupByKey/Write)\n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'(ref_PCollection_PCollection_9/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/FinalizeWrite_27)\n
ref_PCollection_PCollection_9/Read:beam:source:runner:0.1\nwrite/Write/WriteImpl/FinalizeWrite:beam:transform:pardo:v1\n
must follow:
((write/Write/WriteImpl/GroupByKey/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/Extract_25))+(ref_PCollection_PCollection_17/Write),
(((ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16)+(ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17))+(ref_PCollection_PCollection_9/Write))+(ref_PCollection_PCollection_10/Write),
((ref_PCollection_PCollection_9/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/PreFinalize_26))+(ref_PCollection_PCollection_18/Write)\n
downstream_side_inputs: ',
'(((ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16)+(ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17))+(ref_PCollection_PCollection_9/Write))+(ref_PCollection_PCollection_10/Write)\n
write/Write/WriteImpl/DoOnce/Read:beam:transform:read:v1\nwrite/Write/WriteImpl/InitializeWrite:beam:transform:pardo:v1\nref_PCollection_PCollection_9/Write:beam:sink:runner:0.1\nref_PCollection_PCollection_10/Write:beam:sink:runner:0.1\n
must follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18, ref_PCollection_PCollection_10',
'((((((group/Read)+(ref_AppliedPTransform_count_10))+(ref_AppliedPTransform_format_11))+(ref_AppliedPTransform_write/Write/WriteImpl/WriteBundles_18))+(ref_AppliedPTransform_write/Write/WriteImpl/Pair_19))+(ref_AppliedPTransform_write/Write/WriteImpl/WindowInto(WindowIntoFn)_20))+(write/Write/WriteImpl/GroupByKey/Write)\n
group/Read:beam:source:runner:0.1\ncount:beam:transform:pardo:v1\nformat:beam:transform:pardo:v1\nwrite/Write/WriteImpl/WriteBundles:beam:transform:pardo:v1\nwrite/Write/WriteImpl/Pair:beam:transform:pardo:v1\nwrite/Write/WriteImpl/WindowInto(WindowIntoFn):beam:transform:window_into:v1\nwrite/Write/WriteImpl/GroupByKey/Write:beam:sink:runner:0.1\n
must follow:
(((ref_AppliedPTransform_read/Read_3)+(ref_AppliedPTransform_split_4))+(ref_AppliedPTransform_pair_with_one_5))+(group/Write),
(((ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16)+(ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17))+(ref_PCollection_PCollection_9/Write))+(ref_PCollection_PCollection_10/Write)\n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18']
apache_beam.runners.portability.fn_api_runner_transforms: INFO:
==================== <function read_to_impulse at 0x7fe8b41aa730>
====================
apache_beam.runners.portability.fn_api_runner_transforms: DEBUG: 6 [5, 3, 3, 2,
5, 7]
apache_beam.runners.portability.fn_api_runner_transforms: DEBUG: Stages:
['(((ref_AppliedPTransform_read/Read_3)+(ref_AppliedPTransform_split_4))+(ref_AppliedPTransform_pair_with_one_5))+(group/Write)\n
split:beam:transform:pardo:v1\npair_with_one:beam:transform:pardo:v1\ngroup/Write:beam:sink:runner:0.1\nread/Read/Impulse:beam:transform:impulse:v1\nread/Read:beam:transform:read_from_impulse_python:v1\n
must follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'((ref_PCollection_PCollection_9/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/PreFinalize_26))+(ref_PCollection_PCollection_18/Write)\n
ref_PCollection_PCollection_9/Read:beam:source:runner:0.1\nwrite/Write/WriteImpl/PreFinalize:beam:transform:pardo:v1\nref_PCollection_PCollection_18/Write:beam:sink:runner:0.1\n
must follow:
((write/Write/WriteImpl/GroupByKey/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/Extract_25))+(ref_PCollection_PCollection_17/Write),
(((ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16)+(ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17))+(ref_PCollection_PCollection_9/Write))+(ref_PCollection_PCollection_10/Write)\n
downstream_side_inputs: ref_PCollection_PCollection_18',
'((write/Write/WriteImpl/GroupByKey/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/Extract_25))+(ref_PCollection_PCollection_17/Write)\n
write/Write/WriteImpl/GroupByKey/Read:beam:source:runner:0.1\nwrite/Write/WriteImpl/Extract:beam:transform:pardo:v1\nref_PCollection_PCollection_17/Write:beam:sink:runner:0.1\n
must follow:
((((((group/Read)+(ref_AppliedPTransform_count_10))+(ref_AppliedPTransform_format_11))+(ref_AppliedPTransform_write/Write/WriteImpl/WriteBundles_18))+(ref_AppliedPTransform_write/Write/WriteImpl/Pair_19))+(ref_AppliedPTransform_write/Write/WriteImpl/WindowInto(WindowIntoFn)_20))+(write/Write/WriteImpl/GroupByKey/Write)\n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'(ref_PCollection_PCollection_9/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/FinalizeWrite_27)\n
ref_PCollection_PCollection_9/Read:beam:source:runner:0.1\nwrite/Write/WriteImpl/FinalizeWrite:beam:transform:pardo:v1\n
must follow:
((write/Write/WriteImpl/GroupByKey/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/Extract_25))+(ref_PCollection_PCollection_17/Write),
(((ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16)+(ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17))+(ref_PCollection_PCollection_9/Write))+(ref_PCollection_PCollection_10/Write),
((ref_PCollection_PCollection_9/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/PreFinalize_26))+(ref_PCollection_PCollection_18/Write)\n
downstream_side_inputs: ',
'(((ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16)+(ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17))+(ref_PCollection_PCollection_9/Write))+(ref_PCollection_PCollection_10/Write)\n
write/Write/WriteImpl/InitializeWrite:beam:transform:pardo:v1\nref_PCollection_PCollection_9/Write:beam:sink:runner:0.1\nref_PCollection_PCollection_10/Write:beam:sink:runner:0.1\nwrite/Write/WriteImpl/DoOnce/Read/Impulse:beam:transform:impulse:v1\nwrite/Write/WriteImpl/DoOnce/Read:beam:transform:read_from_impulse_python:v1\n
must follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18, ref_PCollection_PCollection_10',
'((((((group/Read)+(ref_AppliedPTransform_count_10))+(ref_AppliedPTransform_format_11))+(ref_AppliedPTransform_write/Write/WriteImpl/WriteBundles_18))+(ref_AppliedPTransform_write/Write/WriteImpl/Pair_19))+(ref_AppliedPTransform_write/Write/WriteImpl/WindowInto(WindowIntoFn)_20))+(write/Write/WriteImpl/GroupByKey/Write)\n
group/Read:beam:source:runner:0.1\ncount:beam:transform:pardo:v1\nformat:beam:transform:pardo:v1\nwrite/Write/WriteImpl/WriteBundles:beam:transform:pardo:v1\nwrite/Write/WriteImpl/Pair:beam:transform:pardo:v1\nwrite/Write/WriteImpl/WindowInto(WindowIntoFn):beam:transform:window_into:v1\nwrite/Write/WriteImpl/GroupByKey/Write:beam:sink:runner:0.1\n
must follow:
(((ref_AppliedPTransform_read/Read_3)+(ref_AppliedPTransform_split_4))+(ref_AppliedPTransform_pair_with_one_5))+(group/Write),
(((ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16)+(ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17))+(ref_PCollection_PCollection_9/Write))+(ref_PCollection_PCollection_10/Write)\n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18']
apache_beam.runners.portability.fn_api_runner_transforms: INFO:
==================== <function impulse_to_input at 0x7fe8b41aa7b8>
====================
apache_beam.runners.portability.fn_api_runner_transforms: DEBUG: 6 [5, 3, 3, 2,
5, 7]
apache_beam.runners.portability.fn_api_runner_transforms: DEBUG: Stages:
['(((ref_AppliedPTransform_read/Read_3)+(ref_AppliedPTransform_split_4))+(ref_AppliedPTransform_pair_with_one_5))+(group/Write)\n
split:beam:transform:pardo:v1\npair_with_one:beam:transform:pardo:v1\ngroup/Write:beam:sink:runner:0.1\nread/Read:beam:transform:read_from_impulse_python:v1\nread/Read/Impulse:beam:source:runner:0.1\n
must follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'((ref_PCollection_PCollection_9/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/PreFinalize_26))+(ref_PCollection_PCollection_18/Write)\n
ref_PCollection_PCollection_9/Read:beam:source:runner:0.1\nwrite/Write/WriteImpl/PreFinalize:beam:transform:pardo:v1\nref_PCollection_PCollection_18/Write:beam:sink:runner:0.1\n
must follow:
((write/Write/WriteImpl/GroupByKey/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/Extract_25))+(ref_PCollection_PCollection_17/Write),
(((ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16)+(ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17))+(ref_PCollection_PCollection_9/Write))+(ref_PCollection_PCollection_10/Write)\n
downstream_side_inputs: ref_PCollection_PCollection_18',
'((write/Write/WriteImpl/GroupByKey/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/Extract_25))+(ref_PCollection_PCollection_17/Write)\n
write/Write/WriteImpl/GroupByKey/Read:beam:source:runner:0.1\nwrite/Write/WriteImpl/Extract:beam:transform:pardo:v1\nref_PCollection_PCollection_17/Write:beam:sink:runner:0.1\n
must follow:
((((((group/Read)+(ref_AppliedPTransform_count_10))+(ref_AppliedPTransform_format_11))+(ref_AppliedPTransform_write/Write/WriteImpl/WriteBundles_18))+(ref_AppliedPTransform_write/Write/WriteImpl/Pair_19))+(ref_AppliedPTransform_write/Write/WriteImpl/WindowInto(WindowIntoFn)_20))+(write/Write/WriteImpl/GroupByKey/Write)\n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'(ref_PCollection_PCollection_9/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/FinalizeWrite_27)\n
ref_PCollection_PCollection_9/Read:beam:source:runner:0.1\nwrite/Write/WriteImpl/FinalizeWrite:beam:transform:pardo:v1\n
must follow:
((write/Write/WriteImpl/GroupByKey/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/Extract_25))+(ref_PCollection_PCollection_17/Write),
(((ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16)+(ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17))+(ref_PCollection_PCollection_9/Write))+(ref_PCollection_PCollection_10/Write),
((ref_PCollection_PCollection_9/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/PreFinalize_26))+(ref_PCollection_PCollection_18/Write)\n
downstream_side_inputs: ',
'(((ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16)+(ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17))+(ref_PCollection_PCollection_9/Write))+(ref_PCollection_PCollection_10/Write)\n
write/Write/WriteImpl/InitializeWrite:beam:transform:pardo:v1\nref_PCollection_PCollection_9/Write:beam:sink:runner:0.1\nref_PCollection_PCollection_10/Write:beam:sink:runner:0.1\nwrite/Write/WriteImpl/DoOnce/Read:beam:transform:read_from_impulse_python:v1\nwrite/Write/WriteImpl/DoOnce/Read/Impulse:beam:source:runner:0.1\n
must follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18, ref_PCollection_PCollection_10',
'((((((group/Read)+(ref_AppliedPTransform_count_10))+(ref_AppliedPTransform_format_11))+(ref_AppliedPTransform_write/Write/WriteImpl/WriteBundles_18))+(ref_AppliedPTransform_write/Write/WriteImpl/Pair_19))+(ref_AppliedPTransform_write/Write/WriteImpl/WindowInto(WindowIntoFn)_20))+(write/Write/WriteImpl/GroupByKey/Write)\n
group/Read:beam:source:runner:0.1\ncount:beam:transform:pardo:v1\nformat:beam:transform:pardo:v1\nwrite/Write/WriteImpl/WriteBundles:beam:transform:pardo:v1\nwrite/Write/WriteImpl/Pair:beam:transform:pardo:v1\nwrite/Write/WriteImpl/WindowInto(WindowIntoFn):beam:transform:window_into:v1\nwrite/Write/WriteImpl/GroupByKey/Write:beam:sink:runner:0.1\n
must follow:
(((ref_AppliedPTransform_read/Read_3)+(ref_AppliedPTransform_split_4))+(ref_AppliedPTransform_pair_with_one_5))+(group/Write),
(((ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16)+(ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17))+(ref_PCollection_PCollection_9/Write))+(ref_PCollection_PCollection_10/Write)\n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18']
apache_beam.runners.portability.fn_api_runner_transforms: INFO:
==================== <function inject_timer_pcollections at 0x7fe8b41aa950>
====================
apache_beam.runners.portability.fn_api_runner_transforms: DEBUG: 6 [5, 3, 3, 2,
5, 7]
apache_beam.runners.portability.fn_api_runner_transforms: DEBUG: Stages:
['(((ref_AppliedPTransform_read/Read_3)+(ref_AppliedPTransform_split_4))+(ref_AppliedPTransform_pair_with_one_5))+(group/Write)\n
split:beam:transform:pardo:v1\npair_with_one:beam:transform:pardo:v1\ngroup/Write:beam:sink:runner:0.1\nread/Read:beam:transform:read_from_impulse_python:v1\nread/Read/Impulse:beam:source:runner:0.1\n
must follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'((ref_PCollection_PCollection_9/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/PreFinalize_26))+(ref_PCollection_PCollection_18/Write)\n
ref_PCollection_PCollection_9/Read:beam:source:runner:0.1\nwrite/Write/WriteImpl/PreFinalize:beam:transform:pardo:v1\nref_PCollection_PCollection_18/Write:beam:sink:runner:0.1\n
must follow:
((write/Write/WriteImpl/GroupByKey/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/Extract_25))+(ref_PCollection_PCollection_17/Write),
(((ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16)+(ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17))+(ref_PCollection_PCollection_9/Write))+(ref_PCollection_PCollection_10/Write)\n
downstream_side_inputs: ref_PCollection_PCollection_18',
'((write/Write/WriteImpl/GroupByKey/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/Extract_25))+(ref_PCollection_PCollection_17/Write)\n
write/Write/WriteImpl/GroupByKey/Read:beam:source:runner:0.1\nwrite/Write/WriteImpl/Extract:beam:transform:pardo:v1\nref_PCollection_PCollection_17/Write:beam:sink:runner:0.1\n
must follow:
((((((group/Read)+(ref_AppliedPTransform_count_10))+(ref_AppliedPTransform_format_11))+(ref_AppliedPTransform_write/Write/WriteImpl/WriteBundles_18))+(ref_AppliedPTransform_write/Write/WriteImpl/Pair_19))+(ref_AppliedPTransform_write/Write/WriteImpl/WindowInto(WindowIntoFn)_20))+(write/Write/WriteImpl/GroupByKey/Write)\n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'(ref_PCollection_PCollection_9/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/FinalizeWrite_27)\n
ref_PCollection_PCollection_9/Read:beam:source:runner:0.1\nwrite/Write/WriteImpl/FinalizeWrite:beam:transform:pardo:v1\n
must follow:
((write/Write/WriteImpl/GroupByKey/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/Extract_25))+(ref_PCollection_PCollection_17/Write),
(((ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16)+(ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17))+(ref_PCollection_PCollection_9/Write))+(ref_PCollection_PCollection_10/Write),
((ref_PCollection_PCollection_9/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/PreFinalize_26))+(ref_PCollection_PCollection_18/Write)\n
downstream_side_inputs: ',
'(((ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16)+(ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17))+(ref_PCollection_PCollection_9/Write))+(ref_PCollection_PCollection_10/Write)\n
write/Write/WriteImpl/InitializeWrite:beam:transform:pardo:v1\nref_PCollection_PCollection_9/Write:beam:sink:runner:0.1\nref_PCollection_PCollection_10/Write:beam:sink:runner:0.1\nwrite/Write/WriteImpl/DoOnce/Read:beam:transform:read_from_impulse_python:v1\nwrite/Write/WriteImpl/DoOnce/Read/Impulse:beam:source:runner:0.1\n
must follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18, ref_PCollection_PCollection_10',
'((((((group/Read)+(ref_AppliedPTransform_count_10))+(ref_AppliedPTransform_format_11))+(ref_AppliedPTransform_write/Write/WriteImpl/WriteBundles_18))+(ref_AppliedPTransform_write/Write/WriteImpl/Pair_19))+(ref_AppliedPTransform_write/Write/WriteImpl/WindowInto(WindowIntoFn)_20))+(write/Write/WriteImpl/GroupByKey/Write)\n
group/Read:beam:source:runner:0.1\ncount:beam:transform:pardo:v1\nformat:beam:transform:pardo:v1\nwrite/Write/WriteImpl/WriteBundles:beam:transform:pardo:v1\nwrite/Write/WriteImpl/Pair:beam:transform:pardo:v1\nwrite/Write/WriteImpl/WindowInto(WindowIntoFn):beam:transform:window_into:v1\nwrite/Write/WriteImpl/GroupByKey/Write:beam:sink:runner:0.1\n
must follow:
(((ref_AppliedPTransform_read/Read_3)+(ref_AppliedPTransform_split_4))+(ref_AppliedPTransform_pair_with_one_5))+(group/Write),
(((ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16)+(ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17))+(ref_PCollection_PCollection_9/Write))+(ref_PCollection_PCollection_10/Write)\n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18']
apache_beam.runners.portability.fn_api_runner_transforms: INFO:
==================== <function sort_stages at 0x7fe8b41aa9d8>
====================
apache_beam.runners.portability.fn_api_runner_transforms: DEBUG: 6 [5, 5, 7, 3,
3, 2]
apache_beam.runners.portability.fn_api_runner_transforms: DEBUG: Stages:
['(((ref_AppliedPTransform_read/Read_3)+(ref_AppliedPTransform_split_4))+(ref_AppliedPTransform_pair_with_one_5))+(group/Write)\n
split:beam:transform:pardo:v1\npair_with_one:beam:transform:pardo:v1\ngroup/Write:beam:sink:runner:0.1\nread/Read:beam:transform:read_from_impulse_python:v1\nread/Read/Impulse:beam:source:runner:0.1\n
must follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'(((ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16)+(ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17))+(ref_PCollection_PCollection_9/Write))+(ref_PCollection_PCollection_10/Write)\n
write/Write/WriteImpl/InitializeWrite:beam:transform:pardo:v1\nref_PCollection_PCollection_9/Write:beam:sink:runner:0.1\nref_PCollection_PCollection_10/Write:beam:sink:runner:0.1\nwrite/Write/WriteImpl/DoOnce/Read:beam:transform:read_from_impulse_python:v1\nwrite/Write/WriteImpl/DoOnce/Read/Impulse:beam:source:runner:0.1\n
must follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18, ref_PCollection_PCollection_10',
'((((((group/Read)+(ref_AppliedPTransform_count_10))+(ref_AppliedPTransform_format_11))+(ref_AppliedPTransform_write/Write/WriteImpl/WriteBundles_18))+(ref_AppliedPTransform_write/Write/WriteImpl/Pair_19))+(ref_AppliedPTransform_write/Write/WriteImpl/WindowInto(WindowIntoFn)_20))+(write/Write/WriteImpl/GroupByKey/Write)\n
group/Read:beam:source:runner:0.1\ncount:beam:transform:pardo:v1\nformat:beam:transform:pardo:v1\nwrite/Write/WriteImpl/WriteBundles:beam:transform:pardo:v1\nwrite/Write/WriteImpl/Pair:beam:transform:pardo:v1\nwrite/Write/WriteImpl/WindowInto(WindowIntoFn):beam:transform:window_into:v1\nwrite/Write/WriteImpl/GroupByKey/Write:beam:sink:runner:0.1\n
must follow:
(((ref_AppliedPTransform_read/Read_3)+(ref_AppliedPTransform_split_4))+(ref_AppliedPTransform_pair_with_one_5))+(group/Write),
(((ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16)+(ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17))+(ref_PCollection_PCollection_9/Write))+(ref_PCollection_PCollection_10/Write)\n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'((write/Write/WriteImpl/GroupByKey/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/Extract_25))+(ref_PCollection_PCollection_17/Write)\n
write/Write/WriteImpl/GroupByKey/Read:beam:source:runner:0.1\nwrite/Write/WriteImpl/Extract:beam:transform:pardo:v1\nref_PCollection_PCollection_17/Write:beam:sink:runner:0.1\n
must follow:
((((((group/Read)+(ref_AppliedPTransform_count_10))+(ref_AppliedPTransform_format_11))+(ref_AppliedPTransform_write/Write/WriteImpl/WriteBundles_18))+(ref_AppliedPTransform_write/Write/WriteImpl/Pair_19))+(ref_AppliedPTransform_write/Write/WriteImpl/WindowInto(WindowIntoFn)_20))+(write/Write/WriteImpl/GroupByKey/Write)\n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'((ref_PCollection_PCollection_9/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/PreFinalize_26))+(ref_PCollection_PCollection_18/Write)\n
ref_PCollection_PCollection_9/Read:beam:source:runner:0.1\nwrite/Write/WriteImpl/PreFinalize:beam:transform:pardo:v1\nref_PCollection_PCollection_18/Write:beam:sink:runner:0.1\n
must follow:
((write/Write/WriteImpl/GroupByKey/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/Extract_25))+(ref_PCollection_PCollection_17/Write),
(((ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16)+(ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17))+(ref_PCollection_PCollection_9/Write))+(ref_PCollection_PCollection_10/Write)\n
downstream_side_inputs: ref_PCollection_PCollection_18',
'(ref_PCollection_PCollection_9/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/FinalizeWrite_27)\n
ref_PCollection_PCollection_9/Read:beam:source:runner:0.1\nwrite/Write/WriteImpl/FinalizeWrite:beam:transform:pardo:v1\n
must follow:
((write/Write/WriteImpl/GroupByKey/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/Extract_25))+(ref_PCollection_PCollection_17/Write),
(((ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16)+(ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17))+(ref_PCollection_PCollection_9/Write))+(ref_PCollection_PCollection_10/Write),
((ref_PCollection_PCollection_9/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/PreFinalize_26))+(ref_PCollection_PCollection_18/Write)\n
downstream_side_inputs: ']
apache_beam.runners.portability.fn_api_runner_transforms: INFO:
==================== <function window_pcollection_coders at 0x7fe8b41aaa60>
====================
apache_beam.runners.portability.fn_api_runner_transforms: DEBUG: 6 [5, 5, 7, 3,
3, 2]
apache_beam.runners.portability.fn_api_runner_transforms: DEBUG: Stages:
['(((ref_AppliedPTransform_read/Read_3)+(ref_AppliedPTransform_split_4))+(ref_AppliedPTransform_pair_with_one_5))+(group/Write)\n
split:beam:transform:pardo:v1\npair_with_one:beam:transform:pardo:v1\ngroup/Write:beam:sink:runner:0.1\nread/Read:beam:transform:read_from_impulse_python:v1\nread/Read/Impulse:beam:source:runner:0.1\n
must follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'(((ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16)+(ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17))+(ref_PCollection_PCollection_9/Write))+(ref_PCollection_PCollection_10/Write)\n
write/Write/WriteImpl/InitializeWrite:beam:transform:pardo:v1\nref_PCollection_PCollection_9/Write:beam:sink:runner:0.1\nref_PCollection_PCollection_10/Write:beam:sink:runner:0.1\nwrite/Write/WriteImpl/DoOnce/Read:beam:transform:read_from_impulse_python:v1\nwrite/Write/WriteImpl/DoOnce/Read/Impulse:beam:source:runner:0.1\n
must follow: \n downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18, ref_PCollection_PCollection_10',
'((((((group/Read)+(ref_AppliedPTransform_count_10))+(ref_AppliedPTransform_format_11))+(ref_AppliedPTransform_write/Write/WriteImpl/WriteBundles_18))+(ref_AppliedPTransform_write/Write/WriteImpl/Pair_19))+(ref_AppliedPTransform_write/Write/WriteImpl/WindowInto(WindowIntoFn)_20))+(write/Write/WriteImpl/GroupByKey/Write)\n
group/Read:beam:source:runner:0.1\ncount:beam:transform:pardo:v1\nformat:beam:transform:pardo:v1\nwrite/Write/WriteImpl/WriteBundles:beam:transform:pardo:v1\nwrite/Write/WriteImpl/Pair:beam:transform:pardo:v1\nwrite/Write/WriteImpl/WindowInto(WindowIntoFn):beam:transform:window_into:v1\nwrite/Write/WriteImpl/GroupByKey/Write:beam:sink:runner:0.1\n
must follow:
(((ref_AppliedPTransform_read/Read_3)+(ref_AppliedPTransform_split_4))+(ref_AppliedPTransform_pair_with_one_5))+(group/Write),
(((ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16)+(ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17))+(ref_PCollection_PCollection_9/Write))+(ref_PCollection_PCollection_10/Write)\n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'((write/Write/WriteImpl/GroupByKey/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/Extract_25))+(ref_PCollection_PCollection_17/Write)\n
write/Write/WriteImpl/GroupByKey/Read:beam:source:runner:0.1\nwrite/Write/WriteImpl/Extract:beam:transform:pardo:v1\nref_PCollection_PCollection_17/Write:beam:sink:runner:0.1\n
must follow:
((((((group/Read)+(ref_AppliedPTransform_count_10))+(ref_AppliedPTransform_format_11))+(ref_AppliedPTransform_write/Write/WriteImpl/WriteBundles_18))+(ref_AppliedPTransform_write/Write/WriteImpl/Pair_19))+(ref_AppliedPTransform_write/Write/WriteImpl/WindowInto(WindowIntoFn)_20))+(write/Write/WriteImpl/GroupByKey/Write)\n
downstream_side_inputs: ref_PCollection_PCollection_17,
ref_PCollection_PCollection_18',
'((ref_PCollection_PCollection_9/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/PreFinalize_26))+(ref_PCollection_PCollection_18/Write)\n
ref_PCollection_PCollection_9/Read:beam:source:runner:0.1\nwrite/Write/WriteImpl/PreFinalize:beam:transform:pardo:v1\nref_PCollection_PCollection_18/Write:beam:sink:runner:0.1\n
must follow:
((write/Write/WriteImpl/GroupByKey/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/Extract_25))+(ref_PCollection_PCollection_17/Write),
(((ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16)+(ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17))+(ref_PCollection_PCollection_9/Write))+(ref_PCollection_PCollection_10/Write)\n
downstream_side_inputs: ref_PCollection_PCollection_18',
'(ref_PCollection_PCollection_9/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/FinalizeWrite_27)\n
ref_PCollection_PCollection_9/Read:beam:source:runner:0.1\nwrite/Write/WriteImpl/FinalizeWrite:beam:transform:pardo:v1\n
must follow:
((write/Write/WriteImpl/GroupByKey/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/Extract_25))+(ref_PCollection_PCollection_17/Write),
(((ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16)+(ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17))+(ref_PCollection_PCollection_9/Write))+(ref_PCollection_PCollection_10/Write),
((ref_PCollection_PCollection_9/Read)+(ref_AppliedPTransform_write/Write/WriteImpl/PreFinalize_26))+(ref_PCollection_PCollection_18/Write)\n
downstream_side_inputs: ']
apache_beam.runners.worker.statecache: INFO: Creating state cache with size 100
apache_beam.runners.portability.fn_api_runner: INFO: Created Worker handler
<apache_beam.runners.portability.fn_api_runner.EmbeddedWorkerHandler object at
0x7fe8b417a7f0> for environment urn: "beam:env:embedded_python:v1"
apache_beam.runners.portability.fn_api_runner: INFO: Running
(((ref_AppliedPTransform_read/Read_3)+(ref_AppliedPTransform_split_4))+(ref_AppliedPTransform_pair_with_one_5))+(group/Write)
apache_beam.runners.worker.bundle_processor: DEBUG: start <DataOutputOperation >
apache_beam.runners.worker.bundle_processor: DEBUG: start <DoOperation
pair_with_one output_tags=['out'],
receivers=[SingletonConsumerSet[pair_with_one.out0,
coder=WindowedValueCoder[TupleCoder[StrUtf8Coder, VarIntCoder]],
len(consumers)=1]]>
apache_beam.runners.worker.bundle_processor: DEBUG: start <DoOperation split
output_tags=['out'], receivers=[SingletonConsumerSet[split.out0,
coder=WindowedValueCoder[StrUtf8Coder], len(consumers)=1]]>
apache_beam.runners.worker.bundle_processor: DEBUG: start <ImpulseReadOperation
receivers=[SingletonConsumerSet[read/Read.out0,
coder=WindowedValueCoder[FastPrimitivesCoder], len(consumers)=1]]>
apache_beam.runners.worker.bundle_processor: DEBUG: start <DataInputOperation
receivers=[SingletonConsumerSet[read/Read/Impulse.out0,
coder=WindowedValueCoder[BytesCoder], len(consumers)=1]]>
apache_beam.io.filesystem: DEBUG: translate_pattern:
'gs://dataflow-samples/shakespeare/kinglear.txt' ->
'gs\\:\\/\\/dataflow\\-samples\\/shakespeare\\/kinglear\\.txt'
apache_beam.runners.worker.bundle_processor: DEBUG: finish <DataInputOperation
receivers=[SingletonConsumerSet[read/Read/Impulse.out0,
coder=WindowedValueCoder[BytesCoder], len(consumers)=1]]>
apache_beam.runners.worker.bundle_processor: DEBUG: finish
<ImpulseReadOperation receivers=[SingletonConsumerSet[read/Read.out0,
coder=WindowedValueCoder[FastPrimitivesCoder], len(consumers)=1]]>
apache_beam.runners.worker.bundle_processor: DEBUG: finish <DoOperation split
output_tags=['out'], receivers=[SingletonConsumerSet[split.out0,
coder=WindowedValueCoder[StrUtf8Coder], len(consumers)=1]]>
apache_beam.runners.worker.bundle_processor: DEBUG: finish <DoOperation
pair_with_one output_tags=['out'],
receivers=[SingletonConsumerSet[pair_with_one.out0,
coder=WindowedValueCoder[TupleCoder[StrUtf8Coder, VarIntCoder]],
len(consumers)=1]]>
apache_beam.runners.worker.bundle_processor: DEBUG: finish <DataOutputOperation
>
apache_beam.runners.portability.fn_api_runner: DEBUG: Wait for the bundle
bundle_1 to finish.
apache_beam.runners.portability.fn_api_runner: INFO: Running
(((ref_AppliedPTransform_write/Write/WriteImpl/DoOnce/Read_16)+(ref_AppliedPTransform_write/Write/WriteImpl/InitializeWrite_17))+(ref_PCollection_PCollection_9/Write))+(ref_PCollection_PCollection_10/Write)
apache_beam.runners.worker.bundle_processor: DEBUG: start <DataOutputOperation >
apache_beam.runners.worker.bundle_processor: DEBUG: start <DataOutputOperation >
apache_beam.runners.worker.bundle_processor: DEBUG: start <DoOperation
write/Write/WriteImpl/InitializeWrite output_tags=['out'],
receivers=[SingletonConsumerSet[write/Write/WriteImpl/InitializeWrite.out0,
coder=WindowedValueCoder[LengthPrefixCoder[FastPrimitivesCoder]],
len(consumers)=1]]>
apache_beam.runners.worker.bundle_processor: DEBUG: start <ImpulseReadOperation
receivers=[ConsumerSet[write/Write/WriteImpl/DoOnce/Read.out0,
coder=WindowedValueCoder[LengthPrefixCoder[FastPrimitivesCoder]],
len(consumers)=2]]>
apache_beam.runners.worker.bundle_processor: DEBUG: start <DataInputOperation
receivers=[SingletonConsumerSet[write/Write/WriteImpl/DoOnce/Read/Impulse.out0,
coder=WindowedValueCoder[BytesCoder], len(consumers)=1]]>
apache_beam.runners.worker.bundle_processor: DEBUG: finish <DataInputOperation
receivers=[SingletonConsumerSet[write/Write/WriteImpl/DoOnce/Read/Impulse.out0,
coder=WindowedValueCoder[BytesCoder], len(consumers)=1]]>
apache_beam.runners.worker.bundle_processor: DEBUG: finish
<ImpulseReadOperation
receivers=[ConsumerSet[write/Write/WriteImpl/DoOnce/Read.out0,
coder=WindowedValueCoder[LengthPrefixCoder[FastPrimitivesCoder]],
len(consumers)=2]]>
apache_beam.runners.worker.bundle_processor: DEBUG: finish <DoOperation
write/Write/WriteImpl/InitializeWrite output_tags=['out'],
receivers=[SingletonConsumerSet[write/Write/WriteImpl/InitializeWrite.out0,
coder=WindowedValueCoder[LengthPrefixCoder[FastPrimitivesCoder]],
len(consumers)=1]]>
apache_beam.runners.worker.bundle_processor: DEBUG: finish <DataOutputOperation
>
apache_beam.runners.worker.bundle_processor: DEBUG: finish <DataOutputOperation
>
apache_beam.runners.portability.fn_api_runner: DEBUG: Wait for the bundle
bundle_2 to finish.
apache_beam.runners.portability.fn_api_runner: INFO: Running
((((((group/Read)+(ref_AppliedPTransform_count_10))+(ref_AppliedPTransform_format_11))+(ref_AppliedPTransform_write/Write/WriteImpl/WriteBundles_18))+(ref_AppliedPTransform_write/Write/WriteImpl/Pair_19))+(ref_AppliedPTransform_write/Write/WriteImpl/WindowInto(WindowIntoFn)_20))+(write/Write/WriteImpl/GroupByKey/Write)
apache_beam.runners.worker.bundle_processor: DEBUG: start <DataOutputOperation >
apache_beam.runners.worker.bundle_processor: DEBUG: start <DoOperation
write/Write/WriteImpl/WindowInto(WindowIntoFn) output_tags=['out'],
receivers=[SingletonConsumerSet[write/Write/WriteImpl/WindowInto(WindowIntoFn).out0,
coder=WindowedValueCoder[TupleCoder[LengthPrefixCoder[FastPrimitivesCoder],
LengthPrefixCoder[FastPrimitivesCoder]]], len(consumers)=1]]>
apache_beam.runners.worker.bundle_processor: DEBUG: start <DoOperation
write/Write/WriteImpl/Pair output_tags=['out'],
receivers=[SingletonConsumerSet[write/Write/WriteImpl/Pair.out0,
coder=WindowedValueCoder[TupleCoder[FastPrimitivesCoder, FastPrimitivesCoder]],
len(consumers)=1]]>
apache_beam.runners.worker.bundle_processor: DEBUG: start <DoOperation
write/Write/WriteImpl/WriteBundles output_tags=['out'],
receivers=[SingletonConsumerSet[write/Write/WriteImpl/WriteBundles.out0,
coder=WindowedValueCoder[FastPrimitivesCoder], len(consumers)=1]]>
apache_beam.runners.worker.bundle_processor: DEBUG: start <DoOperation format
output_tags=['out'], receivers=[SingletonConsumerSet[format.out0,
coder=WindowedValueCoder[FastPrimitivesCoder], len(consumers)=1]]>
apache_beam.runners.worker.bundle_processor: DEBUG: start <DoOperation count
output_tags=['out'], receivers=[SingletonConsumerSet[count.out0,
coder=WindowedValueCoder[TupleCoder[StrUtf8Coder, FastPrimitivesCoder]],
len(consumers)=1]]>
apache_beam.runners.worker.bundle_processor: DEBUG: start <DataInputOperation
receivers=[SingletonConsumerSet[group/Read.out0,
coder=WindowedValueCoder[TupleCoder[StrUtf8Coder, IterableCoder[VarIntCoder]]],
len(consumers)=1]]>
apache_beam.runners.worker.bundle_processor: DEBUG: finish <DataInputOperation
receivers=[SingletonConsumerSet[group/Read.out0,
coder=WindowedValueCoder[TupleCoder[StrUtf8Coder, IterableCoder[VarIntCoder]]],
len(consumers)=1]]>
apache_beam.runners.worker.bundle_processor: DEBUG: finish <DoOperation count
output_tags=['out'], receivers=[SingletonConsumerSet[count.out0,
coder=WindowedValueCoder[TupleCoder[StrUtf8Coder, FastPrimitivesCoder]],
len(consumers)=1]]>
apache_beam.runners.worker.bundle_processor: DEBUG: finish <DoOperation format
output_tags=['out'], receivers=[SingletonConsumerSet[format.out0,
coder=WindowedValueCoder[FastPrimitivesCoder], len(consumers)=1]]>
apache_beam.runners.worker.bundle_processor: DEBUG: finish <DoOperation
write/Write/WriteImpl/WriteBundles output_tags=['out'],
receivers=[SingletonConsumerSet[write/Write/WriteImpl/WriteBundles.out0,
coder=WindowedValueCoder[FastPrimitivesCoder], len(consumers)=1]]>
apache_beam.io.gcp.gcsio: ERROR: Error in _start_upload while inserting file
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1574813463545/beam-temp-results-678bb78610aa11ea8fd542010a8000a5/71448830-ba96-4de2-b6e5-2b72f388cceb.results:
Traceback (most recent call last):
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/sdks/python/apache_beam/io/gcp/gcsio.py";,>
line 568, in _start_upload
self._client.objects.Insert(self._insert_request, upload=self._upload)
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py";,>
line 1156, in Insert
upload=upload, upload_config=upload_config)
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/build/gradleenv/1398941890/lib/python3.6/site-packages/apitools/base/py/base_api.py";,>
line 715, in _RunMethod
http_request, client=self.client)
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/build/gradleenv/1398941890/lib/python3.6/site-packages/apitools/base/py/transfer.py";,>
line 908, in InitializeUpload
return self.StreamInChunks()
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/build/gradleenv/1398941890/lib/python3.6/site-packages/apitools/base/py/transfer.py";,>
line 1020, in StreamInChunks
additional_headers=additional_headers)
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/build/gradleenv/1398941890/lib/python3.6/site-packages/apitools/base/py/transfer.py";,>
line 971, in __StreamMedia
self.RefreshResumableUploadState()
File
"<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/build/gradleenv/1398941890/lib/python3.6/site-packages/apitools/base/py/transfer.py";,>
line 875, in RefreshResumableUploadState
raise exceptions.HttpError.FromResponse(refresh_response)
apitools.base.py.exceptions.HttpError: HttpError accessing
<https://www.googleapis.com/resumable/upload/storage/v1/b/temp-storage-for-end-to-end-tests/o?alt=json&name=py-it-cloud%2Foutput%2F1574813463545%2Fbeam-temp-results-678bb78610aa11ea8fd542010a8000a5%2F71448830-ba96-4de2-b6e5-2b72f388cceb.results&uploadType=resumable&upload_id=AEnB2UqduGnn-MrJq8b8sZJU8blP4yaFHPmrZ-jm07AuVU_i1LyCtHBekeTUonMNpYl-tCsWldmMC-FSYD84VVkUByb7JF7jbg>:
response: <{'x-guploader-uploadid':
'AEnB2UqduGnn-MrJq8b8sZJU8blP4yaFHPmrZ-jm07AuVU_i1LyCtHBekeTUonMNpYl-tCsWldmMC-FSYD84VVkUByb7JF7jbg',
'vary': 'Origin, X-Origin', 'content-type': 'application/json; charset=UTF-8',
'content-length': '177', 'date': 'Wed, 27 Nov 2019 00:11:16 GMT', 'server':
'UploadServer', 'status': '410'}>, content <{
"error": {
"errors": [
{
"domain": "global",
"reason": "backendError",
"message": "Backend Error"
}
],
"code": 503,
"message": "Backend Error"
}
}
>
apache_beam.io.filesystem: DEBUG: Listing files in
'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1574813463545/results'
apache_beam.io.filesystem: DEBUG: translate_pattern:
'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1574813463545/results*'
->
'gs\\:\\/\\/temp\\-storage\\-for\\-end\\-to\\-end\\-tests\\/py\\-it\\-cloud\\/output\\/1574813463545\\/results[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 0 files in 0.06312036514282227
seconds.
--------------------- >> end captured logging << ---------------------
<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/sdks/python/apache_beam/io/gcp/bigquery.py>:1217:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
self.table_reference.projectId = pcoll.pipeline.options.view_as(
<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/sdks/python/apache_beam/io/gcp/bigquery.py>:1217:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
self.table_reference.projectId = pcoll.pipeline.options.view_as(
<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/sdks/python/apache_beam/io/gcp/bigquery_file_loads.py>:797:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
temp_location = p.options.view_as(GoogleCloudOptions).temp_location
----------------------------------------------------------------------
XML: nosetests-postCommitIT-direct-py36.xml
----------------------------------------------------------------------
XML:
<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 15 tests in 22.868s
FAILED (SKIP=1, errors=1)
> Task :sdks:python:test-suites:direct:py36:postCommitIT FAILED
FAILURE: Build completed with 3 failures.
1: Task failed with an exception.
-----------
* What went wrong:
Execution failed for task
':sdks:python:test-suites:portable:py36:installGcpTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug
option to get more log output. Run with --scan to get full insights.
==============================================================================
2: Task failed with an exception.
-----------
* Where:
Build file
'<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/sdks/python/test-suites/dataflow/py36/build.gradle'>
line: 56
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py36:postCommitIT'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug
option to get more log output. Run with --scan to get full insights.
==============================================================================
3: Task failed with an exception.
-----------
* Where:
Build file
'<https://builds.apache.org/job/beam_PostCommit_Python36/ws/src/sdks/python/test-suites/direct/py36/build.gradle'>
line: 51
* What went wrong:
Execution failed for task ':sdks:python:test-suites:direct:py36:postCommitIT'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug
option to get more log output. Run with --scan to get full insights.
==============================================================================
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with
Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See
https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 1m 23s
80 actionable tasks: 59 executed, 21 from cache
Publishing build scan...
https://gradle.com/s/ezb65ivuiyvxa
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
