See
<https://builds.apache.org/job/beam_PostCommit_Python_Verify/4602/display/redirect>
------------------------------------------
[...truncated 5.31 MB...]
component_coder_ids: "ref_Coder_TupleCoder_10"
component_coder_ids: "ref_Coder_GlobalWindowCoder_6"
}
}
coders {
key: "ref_Coder_WindowedValueCoder_9_bytes"
value {
spec {
spec {
urn: "beam:coder:windowed_value:v1"
}
}
component_coder_ids: "ref_Coder_TupleCoder_10_bytes"
component_coder_ids: "ref_Coder_GlobalWindowCoder_6"
}
}
coders {
key: "ref_Coder_WindowedValueCoder_9_len_prefix"
value {
spec {
spec {
urn: "beam:coder:windowed_value:v1"
}
}
component_coder_ids: "ref_Coder_TupleCoder_10_len_prefix"
component_coder_ids: "ref_Coder_GlobalWindowCoder_6"
}
}
}
}
root: DEBUG: CONTROL RESPONSE instruction_id: "control_2"
register {
}
root: DEBUG: CONTROL REQUEST instruction_id: "bundle_538"
process_bundle {
process_bundle_descriptor_reference: "1"
}
root: INFO: start <DataOutputOperation
ReadAllFromTFRecord/ReadAllFiles/Reshard/ReshufflePerKey/GroupByKey/Write >
root: INFO: start <DoOperation
ReadAllFromTFRecord/ReadAllFiles/Reshard/ReshufflePerKey/Map(reify_timestamps)
output_tags=['out']>
root: INFO: start <DoOperation
ReadAllFromTFRecord/ReadAllFiles/Reshard/AddRandomKeys output_tags=['out']>
root: INFO: start <DoOperation
ReadAllFromTFRecord/ReadAllFiles/ExpandIntoRanges output_tags=['out']>
root: INFO: start <ReadOperation Create/Read source=SourceBundle(weight=1.0,
source=<apache_beam.transforms.core._CreateSource object at 0x7f8c6fb40ed0>,
start_position=None, stop_position=None)>
--------------------- >> end captured logging << ---------------------
======================================================================
ERROR: test_file_pattern_verify_details
(apache_beam.io.vcfio_test.VcfSourceTest)
----------------------------------------------------------------------
Traceback (most recent call last):
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/vcfio_test.py";,>
line 317, in test_file_pattern_verify_details
read_data = self._read_records(os.path.join(tempdir.get_path(), '*.vcf'))
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/vcfio_test.py";,>
line 102, in _read_records
VcfSource(file_or_pattern, **kwargs))
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/vcfio.py";,>
line 244, in __init__
validate=validate)
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/filebasedsource.py";,>
line 119, in __init__
self._validate()
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/options/value_provider.py";,>
line 124, in _f
return fnc(self, *args, **kwargs)
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/filebasedsource.py";,>
line 176, in _validate
match_result = FileSystems.match([pattern], limits=[1])[0]
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/filesystems.py";,>
line 164, in match
return filesystem.match(patterns, limits)
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/filesystem.py";,>
line 590, in match
raise BeamIOError("Match operation failed", exceptions)
BeamIOError: Match operation failed with exceptions {'/tmp/tmp5_aFqg/*.vcf':
ValueError('Could not parse URL: %s', '/tmp/tmp5_aFqg/')}
======================================================================
ERROR: test_invalid_file (apache_beam.io.vcfio_test.VcfSourceTest)
----------------------------------------------------------------------
Traceback (most recent call last):
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/vcfio_test.py";,>
line 345, in test_invalid_file
self._read_records(os.path.join(tempdir.get_path(), '*.vcf'))
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/vcfio_test.py";,>
line 102, in _read_records
VcfSource(file_or_pattern, **kwargs))
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/vcfio.py";,>
line 244, in __init__
validate=validate)
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/filebasedsource.py";,>
line 119, in __init__
self._validate()
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/options/value_provider.py";,>
line 124, in _f
return fnc(self, *args, **kwargs)
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/filebasedsource.py";,>
line 176, in _validate
match_result = FileSystems.match([pattern], limits=[1])[0]
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/filesystems.py";,>
line 164, in match
return filesystem.match(patterns, limits)
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/filesystem.py";,>
line 590, in match
raise BeamIOError("Match operation failed", exceptions)
BeamIOError: Match operation failed with exceptions {'/tmp/tmpWeJZ0J/*.vcf':
ValueError('Could not parse URL: %s', '/tmp/tmpWeJZ0J/')}
======================================================================
ERROR: test_pipeline_read_file_pattern_large
(apache_beam.io.vcfio_test.VcfSourceTest)
----------------------------------------------------------------------
Traceback (most recent call last):
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/vcfio_test.py";,>
line 507, in test_pipeline_read_file_pattern_large
os.path.join(get_full_dir(), 'valid-*.vcf'))
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/vcfio.py";,>
line 479, in __init__
allow_malformed_records=allow_malformed_records)
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/vcfio.py";,>
line 244, in __init__
validate=validate)
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/filebasedsource.py";,>
line 119, in __init__
self._validate()
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/options/value_provider.py";,>
line 124, in _f
return fnc(self, *args, **kwargs)
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/filebasedsource.py";,>
line 176, in _validate
match_result = FileSystems.match([pattern], limits=[1])[0]
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/filesystems.py";,>
line 164, in match
return filesystem.match(patterns, limits)
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/filesystem.py";,>
line 590, in match
raise BeamIOError("Match operation failed", exceptions)
BeamIOError: Match operation failed with exceptions
{'<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/../testing/data/vcf/valid-*.vcf'>:
ValueError('Could not parse URL: %s',
'<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/../testing/data/vcf/valid-')}>
-------------------- >> begin captured logging << --------------------
root: INFO: Missing pipeline option (runner). Executing pipeline using the
default runner: DirectRunner.
--------------------- >> end captured logging << ---------------------
======================================================================
ERROR: test_read_file_pattern_large (apache_beam.io.vcfio_test.VcfSourceTest)
----------------------------------------------------------------------
Traceback (most recent call last):
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/vcfio_test.py";,>
line 281, in test_read_file_pattern_large
os.path.join(get_full_dir(), 'valid-*.vcf'))
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/vcfio_test.py";,>
line 102, in _read_records
VcfSource(file_or_pattern, **kwargs))
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/vcfio.py";,>
line 244, in __init__
validate=validate)
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/filebasedsource.py";,>
line 119, in __init__
self._validate()
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/options/value_provider.py";,>
line 124, in _f
return fnc(self, *args, **kwargs)
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/filebasedsource.py";,>
line 176, in _validate
match_result = FileSystems.match([pattern], limits=[1])[0]
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/filesystems.py";,>
line 164, in match
return filesystem.match(patterns, limits)
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/filesystem.py";,>
line 590, in match
raise BeamIOError("Match operation failed", exceptions)
BeamIOError: Match operation failed with exceptions
{'<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/../testing/data/vcf/valid-*.vcf'>:
ValueError('Could not parse URL: %s',
'<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/../testing/data/vcf/valid-')}>
======================================================================
ERROR: test_file_checksum_matcher_sleep_before_verify
(apache_beam.testing.pipeline_verifiers_test.PipelineVerifiersTest)
----------------------------------------------------------------------
Traceback (most recent call last):
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/target/.tox/py27-gcp/local/lib/python2.7/site-packages/mock/mock.py";,>
line 1305, in patched
return func(*args, **keywargs)
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/testing/pipeline_verifiers_test.py";,>
line 143, in test_file_checksum_matcher_sleep_before_verify
hc_assert_that(self._mock_result, matcher)
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/target/.tox/py27-gcp/local/lib/python2.7/site-packages/hamcrest/core/assert_that.py";,>
line 43, in assert_that
_assert_match(actual=arg1, matcher=arg2, reason=arg3)
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/target/.tox/py27-gcp/local/lib/python2.7/site-packages/hamcrest/core/assert_that.py";,>
line 49, in _assert_match
if not matcher.matches(actual):
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/target/.tox/py27-gcp/local/lib/python2.7/site-packages/hamcrest/core/base_matcher.py";,>
line 28, in matches
match_result = self._matches(item)
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/testing/pipeline_verifiers.py";,>
line 137, in _matches
read_lines = self._read_with_retry()
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/utils/retry.py";,>
line 193, in wrapper
six.raise_from(exn, exn_traceback)
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/target/.tox/py27-gcp/local/lib/python2.7/site-packages/six.py";,>
line 737, in raise_from
raise value
BeamIOError: Match operation failed with exceptions {'/tmp/tmprOg5P9/*':
ValueError('Could not parse URL: %s', '/tmp/tmprOg5P9/')}
-------------------- >> begin captured logging << --------------------
root: INFO: Wait 10 seconds...
--------------------- >> end captured logging << ---------------------
======================================================================
ERROR: test_file_checksum_matcher_success
(apache_beam.testing.pipeline_verifiers_test.PipelineVerifiersTest)
----------------------------------------------------------------------
Traceback (most recent call last):
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/testing/pipeline_verifiers_test.py";,>
line 102, in test_file_checksum_matcher_success
hc_assert_that(self._mock_result, matcher)
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/target/.tox/py27-gcp/local/lib/python2.7/site-packages/hamcrest/core/assert_that.py";,>
line 43, in assert_that
_assert_match(actual=arg1, matcher=arg2, reason=arg3)
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/target/.tox/py27-gcp/local/lib/python2.7/site-packages/hamcrest/core/assert_that.py";,>
line 49, in _assert_match
if not matcher.matches(actual):
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/target/.tox/py27-gcp/local/lib/python2.7/site-packages/hamcrest/core/base_matcher.py";,>
line 28, in matches
match_result = self._matches(item)
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/testing/pipeline_verifiers.py";,>
line 137, in _matches
read_lines = self._read_with_retry()
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/utils/retry.py";,>
line 193, in wrapper
six.raise_from(exn, exn_traceback)
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/target/.tox/py27-gcp/local/lib/python2.7/site-packages/six.py";,>
line 737, in raise_from
raise value
BeamIOError: Match operation failed with exceptions {'/tmp/tmp7pUqjQ/*':
ValueError('Could not parse URL: %s', '/tmp/tmp7pUqjQ/')}
======================================================================
FAIL: test_validation_failing
(apache_beam.io.filebasedsource_test.TestFileBasedSource)
----------------------------------------------------------------------
Traceback (most recent call last):
File
"<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/filebasedsource_test.py";,>
line 260, in test_validation_failing
LineSource(os.path.join(temp_dir, '*'))
AssertionError: "No files found based on the file pattern*" does not match
"Match operation failed with exceptions {'/tmp/tmplv_0T1/*': ValueError('Could
not parse URL: %s', '/tmp/tmplv_0T1/')}"
----------------------------------------------------------------------
Ran 1556 tests in 160.782s
FAILED (failures=1, errors=46, skipped=44)
Test failed: <unittest.runner.TextTestResult run=1556 errors=46 failures=1>
error: Test failed: <unittest.runner.TextTestResult run=1556 errors=46
failures=1>
ERROR: InvocationError for command
'<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/target/.tox/py27-gcp/bin/python>
setup.py test' (exited with code 1)
py27-lint create:
<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/target/.tox/py27-lint>
py27-lint installdeps: pycodestyle==2.3.1, pylint==1.7.2, future==0.16.0,
isort==4.2.15, flake8==3.5.0
py27-lint inst:
<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/target/.tox/dist/apache-beam-2.5.0.dev0.zip>
py27-lint installed:
apache-beam==2.5.0.dev0,astroid==1.6.3,avro==1.8.2,backports.functools-lru-cache==1.5,certifi==2018.1.18,chardet==3.0.4,configparser==3.5.0,crcmod==1.7,dill==0.2.6,docopt==0.6.2,enum34==1.1.6,flake8==3.5.0,funcsigs==1.0.2,future==0.16.0,futures==3.2.0,grpcio==1.10.0,hdfs==2.1.0,httplib2==0.9.2,idna==2.6,isort==4.2.15,lazy-object-proxy==1.3.1,mccabe==0.6.1,mock==2.0.0,nose==1.3.7,oauth2client==4.1.2,pbr==4.0.1,protobuf==3.5.2.post1,pyasn1==0.4.2,pyasn1-modules==0.2.1,pycodestyle==2.3.1,pyflakes==1.6.0,PyHamcrest==1.9.0,pylint==1.7.2,pytz==2018.3,PyVCF==0.6.8,PyYAML==3.12,requests==2.18.4,rsa==3.4.2,singledispatch==3.4.0.3,six==1.11.0,typing==3.6.4,urllib3==1.22,wrapt==1.10.11
py27-lint runtests: PYTHONHASHSEED='1763880107'
py27-lint runtests: commands[0] | python --version
Python 2.7.6
py27-lint runtests: commands[1] | pip --version
pip 9.0.3 from
<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/target/.tox/py27-lint/local/lib/python2.7/site-packages>
(python 2.7)
py27-lint runtests: commands[2] | time
<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/run_pylint.sh>
Skipping lint for generated files: bigquery_v2_client.py,
bigquery_v2_messages.py, dataflow_v1b3_client.py, dataflow_v1b3_messages.py,
storage_v1_client.py, storage_v1_messages.py,
proto2_coder_test_messages_pb2.py, beam_artifact_api_pb2_grpc.py,
beam_artifact_api_pb2.py, beam_fn_api_pb2_grpc.py, beam_fn_api_pb2.py,
beam_job_api_pb2_grpc.py, beam_job_api_pb2.py, beam_provision_api_pb2_grpc.py,
beam_provision_api_pb2.py, beam_runner_api_pb2_grpc.py, beam_runner_api_pb2.py,
endpoints_pb2_grpc.py, endpoints_pb2.py, standard_window_fns_pb2_grpc.py,
standard_window_fns_pb2.py
Running pylint for module apache_beam:
--------------------------------------------------------------------
Your code has been rated at 10.00/10 (previous run: 10.00/10, +0.00)
Running pycodestyle for module apache_beam:
Running flake8 for module apache_beam:
0
Running isort for module apache_beam:
<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam>
<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python>
Skipped 72 files
<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python>
Checking for files requiring stage 1 refactoring from futurize
No future changes needed
Checking unittest.main for module apache_beam:
384.32user 7.39system 1:51.34elapsed 351%CPU (0avgtext+0avgdata
255984maxresident)k
592inputs+168outputs (0major+752675minor)pagefaults 0swaps
py3-lint create:
<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/target/.tox/py3-lint>
py3-lint installdeps: pycodestyle==2.3.1, pylint==1.7.2, future==0.16.0,
isort==4.2.15, flake8==3.5.0
py3-lint inst:
<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/target/.tox/dist/apache-beam-2.5.0.dev0.zip>
py3-lint installed:
apache-beam==2.5.0.dev0,astroid==1.6.3,avro==1.8.2,certifi==2018.1.18,chardet==3.0.4,crcmod==1.7,dill==0.2.6,docopt==0.6.2,flake8==3.5.0,future==0.16.0,futures==3.1.1,grpcio==1.10.0,hdfs==2.1.0,httplib2==0.9.2,idna==2.6,isort==4.2.15,lazy-object-proxy==1.3.1,mccabe==0.6.1,mock==2.0.0,nose==1.3.7,oauth2client==4.1.2,pbr==4.0.1,protobuf==3.5.2.post1,pyasn1==0.4.2,pyasn1-modules==0.2.1,pycodestyle==2.3.1,pyflakes==1.6.0,PyHamcrest==1.9.0,pylint==1.7.2,pytz==2018.3,PyVCF==0.6.8,PyYAML==3.12,requests==2.18.4,rsa==3.4.2,six==1.11.0,typing==3.6.4,urllib3==1.22,wrapt==1.10.11
py3-lint runtests: PYTHONHASHSEED='1763880107'
py3-lint runtests: commands[0] | python --version
Python 3.4.3
py3-lint runtests: commands[1] | pip --version
pip 9.0.3 from
<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/target/.tox/py3-lint/lib/python3.4/site-packages>
(python 3.4)
py3-lint runtests: commands[2] | time
<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/run_mini_py3lint.sh>
Running flake8 for module apache_beam:
0
35.47user 0.18system 0:13.41elapsed 265%CPU (0avgtext+0avgdata
47256maxresident)k
8inputs+0outputs (1major+67732minor)pagefaults 0swaps
___________________________________ summary ____________________________________
ERROR: cover: commands failed
ERROR: docs: commands failed
ERROR: py27: commands failed
ERROR: py27-cython: commands failed
ERROR: py27-gcp: commands failed
py27-lint: commands succeeded
py3-lint: commands succeeded
Build step 'Execute shell' marked build as failure
