See
<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/3655/display/redirect?page=changes>
Changes:
[github] [BEAM-7013] Add HLL doc link to Beam website
------------------------------------------
[...truncated 199.57 KB...]
copying apache_beam/testing/benchmarks/nexmark/models/__init__.py ->
apache-beam-2.15.0.dev0/apache_beam/testing/benchmarks/nexmark/models
copying apache_beam/testing/benchmarks/nexmark/models/nexmark_model.py ->
apache-beam-2.15.0.dev0/apache_beam/testing/benchmarks/nexmark/models
copying apache_beam/testing/benchmarks/nexmark/queries/__init__.py ->
apache-beam-2.15.0.dev0/apache_beam/testing/benchmarks/nexmark/queries
copying apache_beam/testing/benchmarks/nexmark/queries/query0.py ->
apache-beam-2.15.0.dev0/apache_beam/testing/benchmarks/nexmark/queries
copying apache_beam/testing/benchmarks/nexmark/queries/query1.py ->
apache-beam-2.15.0.dev0/apache_beam/testing/benchmarks/nexmark/queries
copying apache_beam/testing/benchmarks/nexmark/queries/query2.py ->
apache-beam-2.15.0.dev0/apache_beam/testing/benchmarks/nexmark/queries
copying apache_beam/testing/data/trigger_transcripts.yaml ->
apache-beam-2.15.0.dev0/apache_beam/testing/data
copying apache_beam/testing/load_tests/__init__.py ->
apache-beam-2.15.0.dev0/apache_beam/testing/load_tests
copying apache_beam/testing/load_tests/co_group_by_key_test.py ->
apache-beam-2.15.0.dev0/apache_beam/testing/load_tests
copying apache_beam/testing/load_tests/combine_test.py ->
apache-beam-2.15.0.dev0/apache_beam/testing/load_tests
copying apache_beam/testing/load_tests/group_by_key_test.py ->
apache-beam-2.15.0.dev0/apache_beam/testing/load_tests
copying apache_beam/testing/load_tests/load_test.py ->
apache-beam-2.15.0.dev0/apache_beam/testing/load_tests
copying apache_beam/testing/load_tests/load_test_metrics_utils.py ->
apache-beam-2.15.0.dev0/apache_beam/testing/load_tests
copying apache_beam/testing/load_tests/pardo_test.py ->
apache-beam-2.15.0.dev0/apache_beam/testing/load_tests
copying apache_beam/testing/load_tests/sideinput_test.py ->
apache-beam-2.15.0.dev0/apache_beam/testing/load_tests
copying apache_beam/tools/__init__.py ->
apache-beam-2.15.0.dev0/apache_beam/tools
copying apache_beam/tools/coders_microbenchmark.py ->
apache-beam-2.15.0.dev0/apache_beam/tools
copying apache_beam/tools/distribution_counter_microbenchmark.py ->
apache-beam-2.15.0.dev0/apache_beam/tools
copying apache_beam/tools/map_fn_microbenchmark.py ->
apache-beam-2.15.0.dev0/apache_beam/tools
copying apache_beam/tools/microbenchmarks_test.py ->
apache-beam-2.15.0.dev0/apache_beam/tools
copying apache_beam/tools/sideinput_microbenchmark.py ->
apache-beam-2.15.0.dev0/apache_beam/tools
copying apache_beam/tools/utils.py -> apache-beam-2.15.0.dev0/apache_beam/tools
copying apache_beam/transforms/__init__.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/combiners.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/combiners_test.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/core.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/create_source.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/create_test.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/cy_combiners.pxd ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/cy_combiners.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/cy_dataflow_distribution_counter.pxd ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/cy_dataflow_distribution_counter.pyx ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/dataflow_distribution_counter_test.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/display.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/display_test.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/dofn_lifecycle_test.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/external.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/external_test.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/external_test_it.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/ptransform.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/ptransform_test.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/py_dataflow_distribution_counter.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/sideinputs.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/sideinputs_test.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/stats.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/stats_test.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/timeutil.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/trigger.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/trigger_test.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/userstate.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/userstate_test.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/util.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/util_test.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/window.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/window_test.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/transforms/write_ptransform_test.py ->
apache-beam-2.15.0.dev0/apache_beam/transforms
copying apache_beam/typehints/__init__.py ->
apache-beam-2.15.0.dev0/apache_beam/typehints
copying apache_beam/typehints/decorators.py ->
apache-beam-2.15.0.dev0/apache_beam/typehints
copying apache_beam/typehints/native_type_compatibility.py ->
apache-beam-2.15.0.dev0/apache_beam/typehints
copying apache_beam/typehints/native_type_compatibility_test.py ->
apache-beam-2.15.0.dev0/apache_beam/typehints
copying apache_beam/typehints/opcodes.py ->
apache-beam-2.15.0.dev0/apache_beam/typehints
copying apache_beam/typehints/trivial_inference.py ->
apache-beam-2.15.0.dev0/apache_beam/typehints
copying apache_beam/typehints/trivial_inference_test.py ->
apache-beam-2.15.0.dev0/apache_beam/typehints
copying apache_beam/typehints/typecheck.py ->
apache-beam-2.15.0.dev0/apache_beam/typehints
copying apache_beam/typehints/typed_pipeline_test.py ->
apache-beam-2.15.0.dev0/apache_beam/typehints
copying apache_beam/typehints/typehints.py ->
apache-beam-2.15.0.dev0/apache_beam/typehints
copying apache_beam/typehints/typehints_test.py ->
apache-beam-2.15.0.dev0/apache_beam/typehints
copying apache_beam/utils/__init__.py ->
apache-beam-2.15.0.dev0/apache_beam/utils
copying apache_beam/utils/annotations.py ->
apache-beam-2.15.0.dev0/apache_beam/utils
copying apache_beam/utils/annotations_test.py ->
apache-beam-2.15.0.dev0/apache_beam/utils
copying apache_beam/utils/counters.pxd ->
apache-beam-2.15.0.dev0/apache_beam/utils
copying apache_beam/utils/counters.py ->
apache-beam-2.15.0.dev0/apache_beam/utils
copying apache_beam/utils/counters_test.py ->
apache-beam-2.15.0.dev0/apache_beam/utils
copying apache_beam/utils/plugin.py -> apache-beam-2.15.0.dev0/apache_beam/utils
copying apache_beam/utils/processes.py ->
apache-beam-2.15.0.dev0/apache_beam/utils
copying apache_beam/utils/processes_test.py ->
apache-beam-2.15.0.dev0/apache_beam/utils
copying apache_beam/utils/profiler.py ->
apache-beam-2.15.0.dev0/apache_beam/utils
copying apache_beam/utils/proto_utils.py ->
apache-beam-2.15.0.dev0/apache_beam/utils
copying apache_beam/utils/retry.py -> apache-beam-2.15.0.dev0/apache_beam/utils
copying apache_beam/utils/retry_test.py ->
apache-beam-2.15.0.dev0/apache_beam/utils
copying apache_beam/utils/timestamp.py ->
apache-beam-2.15.0.dev0/apache_beam/utils
copying apache_beam/utils/timestamp_test.py ->
apache-beam-2.15.0.dev0/apache_beam/utils
copying apache_beam/utils/urns.py -> apache-beam-2.15.0.dev0/apache_beam/utils
copying apache_beam/utils/windowed_value.pxd ->
apache-beam-2.15.0.dev0/apache_beam/utils
copying apache_beam/utils/windowed_value.py ->
apache-beam-2.15.0.dev0/apache_beam/utils
copying apache_beam/utils/windowed_value_test.py ->
apache-beam-2.15.0.dev0/apache_beam/utils
Writing apache-beam-2.15.0.dev0/setup.cfg
creating dist
Creating tar archive
removing 'apache-beam-2.15.0.dev0' (and everything under it)
SDK_LOCATION=$(find dist/apache-beam-*.tar.gz)
find dist/apache-beam-*.tar.gz
# Run ValidatesRunner tests on Google Cloud Dataflow service
echo ">>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST"
>>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST
python setup.py nosetests \
--attr ValidatesContainer \
--nologcapture \
--processes=1 \
--process-timeout=900 \
--test-pipeline-options=" \
--runner=TestDataflowRunner \
--project=$PROJECT \
--worker_harness_container_image=$CONTAINER:$TAG \
--staging_location=$GCS_LOCATION/staging-validatesrunner-test \
--temp_location=$GCS_LOCATION/temp-validatesrunner-test \
--output=$GCS_LOCATION/output \
--sdk_location=$SDK_LOCATION \
--num_workers=1"
<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/container/venv/python/local/lib/python2.7/site-packages/setuptools/dist.py>:472:
UserWarning: Normalizing '2.15.0.dev' to '2.15.0.dev0'
normalized_version,
running nosetests
running egg_info
<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/gen_protos.py>:57:
UserWarning: Installing grpcio-tools is recommended for development.
warnings.warn('Installing grpcio-tools is recommended for development.')
writing requirements to apache_beam.egg-info/requires.txt
writing apache_beam.egg-info/PKG-INFO
writing top-level names to apache_beam.egg-info/top_level.txt
writing dependency_links to apache_beam.egg-info/dependency_links.txt
writing entry points to apache_beam.egg-info/entry_points.txt
reading manifest file 'apache_beam.egg-info/SOURCES.txt'
reading manifest template 'MANIFEST.in'
warning: no files found matching 'README.md'
warning: no files found matching 'NOTICE'
warning: no files found matching 'LICENSE'
writing manifest file 'apache_beam.egg-info/SOURCES.txt'
WARNING:root:Using deprecated Datastore client.
This client will be removed in Beam 3.0 (next Beam major release).
Please migrate to apache_beam.io.gcp.datastore.v1new.datastoreio.
WARNING:root:python-snappy is not installed; some tests will be skipped.
WARNING:root:Tensorflow is not installed, so skipping some tests.
test_wordcount_fnapi_it (apache_beam.examples.wordcount_it_test.WordCountIT)
... ERROR
WARNING:root:Discarding unparseable args:
['--output=gs://temp-storage-for-end-to-end-tests/output']
WARNING:root:Discarding unparseable args:
['--output=gs://temp-storage-for-end-to-end-tests/output']
test_metrics_fnapi_it
(apache_beam.runners.dataflow.dataflow_exercise_metrics_pipeline_test.ExerciseMetricsPipelineTest)
... ok
======================================================================
ERROR: test_wordcount_fnapi_it
(apache_beam.examples.wordcount_it_test.WordCountIT)
----------------------------------------------------------------------
Traceback (most recent call last):
File
"<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/examples/wordcount_it_test.py";,>
line 52, in test_wordcount_fnapi_it
self._run_wordcount_it(wordcount.run, experiment='beam_fn_api')
File
"<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/examples/wordcount_it_test.py";,>
line 84, in _run_wordcount_it
run_wordcount(test_pipeline.get_full_options_as_args(**extra_opts))
File
"<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/examples/wordcount.py";,>
line 114, in run
result = p.run()
File
"<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/pipeline.py";,>
line 406, in run
self._options).run(False)
File
"<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/pipeline.py";,>
line 419, in run
return self.runner.run_pipeline(self, self._options)
File
"<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py";,>
line 64, in run_pipeline
self.result.wait_until_finish(duration=wait_duration)
File
"<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py";,>
line 1338, in wait_until_finish
(self.state, getattr(self._runner, 'last_error_msg', None)), self)
DataflowRuntimeException: Dataflow pipeline failed. State: FAILED, Error:
java.util.concurrent.ExecutionException: java.lang.RuntimeException: Error
received from SDK harness for instruction -366: Traceback (most recent call
last):
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 157, in _execute
response = task()
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 190, in <lambda>
self._execute(lambda: worker.do_instruction(work), work)
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 342, in do_instruction
request.instruction_id)
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 368, in process_bundle
bundle_processor.process_bundle(instruction_id))
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 593, in process_bundle
data.ptransform_id].process_encoded(data.data)
KeyError: u'\n\x04-344\x12\x04-342'
at
java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:357)
at
java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1895)
at org.apache.beam.sdk.util.MoreFutures.get(MoreFutures.java:57)
at
org.apache.beam.runners.dataflow.worker.fn.control.RegisterAndProcessBundleOperation.finish(RegisterAndProcessBundleOperation.java:285)
at
org.apache.beam.runners.dataflow.worker.util.common.worker.MapTaskExecutor.execute(MapTaskExecutor.java:85)
at
org.apache.beam.runners.dataflow.worker.fn.control.BeamFnMapTaskExecutor.execute(BeamFnMapTaskExecutor.java:125)
at
org.apache.beam.runners.dataflow.worker.BatchDataflowWorker.executeWork(BatchDataflowWorker.java:412)
at
org.apache.beam.runners.dataflow.worker.BatchDataflowWorker.doWork(BatchDataflowWorker.java:381)
at
org.apache.beam.runners.dataflow.worker.BatchDataflowWorker.getAndPerformWork(BatchDataflowWorker.java:306)
at
org.apache.beam.runners.dataflow.worker.DataflowRunnerHarness.start(DataflowRunnerHarness.java:195)
at
org.apache.beam.runners.dataflow.worker.DataflowRunnerHarness.main(DataflowRunnerHarness.java:123)
Suppressed: java.lang.IllegalStateException: Already closed.
at
org.apache.beam.sdk.fn.data.BeamFnDataBufferingOutboundObserver.close(BeamFnDataBufferingOutboundObserver.java:95)
at
org.apache.beam.runners.dataflow.worker.fn.data.RemoteGrpcPortWriteOperation.abort(RemoteGrpcPortWriteOperation.java:215)
at
org.apache.beam.runners.dataflow.worker.util.common.worker.MapTaskExecutor.execute(MapTaskExecutor.java:91)
... 6 more
Caused by: java.lang.RuntimeException: Error received from SDK harness for
instruction -366: Traceback (most recent call last):
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 157, in _execute
response = task()
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 190, in <lambda>
self._execute(lambda: worker.do_instruction(work), work)
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 342, in do_instruction
request.instruction_id)
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 368, in process_bundle
bundle_processor.process_bundle(instruction_id))
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 593, in process_bundle
data.ptransform_id].process_encoded(data.data)
KeyError: u'\n\x04-344\x12\x04-342'
at
org.apache.beam.runners.fnexecution.control.FnApiControlClient$ResponseStreamObserver.onNext(FnApiControlClient.java:157)
at
org.apache.beam.runners.fnexecution.control.FnApiControlClient$ResponseStreamObserver.onNext(FnApiControlClient.java:140)
at
org.apache.beam.vendor.grpc.v1p13p1.io.grpc.stub.ServerCalls$StreamingServerCallHandler$StreamingServerCallListener.onMessage(ServerCalls.java:248)
at
org.apache.beam.vendor.grpc.v1p13p1.io.grpc.ForwardingServerCallListener.onMessage(ForwardingServerCallListener.java:33)
at
org.apache.beam.vendor.grpc.v1p13p1.io.grpc.Contexts$ContextualizedServerCallListener.onMessage(Contexts.java:76)
at
org.apache.beam.vendor.grpc.v1p13p1.io.grpc.internal.ServerCallImpl$ServerStreamListenerImpl.messagesAvailable(ServerCallImpl.java:263)
at
org.apache.beam.vendor.grpc.v1p13p1.io.grpc.internal.ServerImpl$JumpToApplicationThreadServerStreamListener$1MessagesAvailable.runInContext(ServerImpl.java:683)
at
org.apache.beam.vendor.grpc.v1p13p1.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)
at
org.apache.beam.vendor.grpc.v1p13p1.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
-------------------- >> begin captured stdout << ---------------------
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-06-25_16_48_19-15664827633725863484?project=apache-beam-testing.
--------------------- >> end captured stdout << ----------------------
----------------------------------------------------------------------
XML:
<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 2 tests in 919.403s
FAILED (errors=1)
cleanup_container
Untagged: us.gcr.io/apache-beam-testing/jenkins/python:20190625-234111
Untagged:
us.gcr.io/apache-beam-testing/jenkins/python@sha256:eca632fd489e510f5a118a2fc7d6283989520a58e182c9c5847f8604444af13e
Deleted: sha256:6e6bbb7959863979de964a09f868e8aa9cb52f1f8c4cf3f04b4061c7f5f004e7
Deleted: sha256:3edc65a3a1869591abaa568c21d0c290616d200f48d0cab0b385272fbc7a509b
Deleted: sha256:b4b3a4a2ae772dd3ad6eec9dc6a579743992711ebabf3cc5617f5d07bd82649e
Deleted: sha256:7b8e505c7461a276409986a6665957f5f42dde5a71862ac466f28799cca4d5b1
Deleted: sha256:ea4562241d925cf565fc2f2eb8ed5beb969a47293ac34d909130dbf2fcf9452a
Deleted: sha256:ee4cde875442128f929ec6449ae8aaecc136a543326afd6024da7e310801b5ed
Deleted: sha256:f60508ed2016e8f2a1283e9207a844bfdaa97196b8826f5fe5a81ce40fd9b5cd
Deleted: sha256:b7ed0e832c2e4444ed67bf34073a0c19b35696235519062b5405c880eeb47197
Deleted: sha256:6b2442e056ec5ddb8e35c36a6dcd01fd84c881bff03d65fc7d9e5573cfb7861f
Deleted: sha256:239ef455ceac7649124e0d629f0c40bbadff5eabaa8a96c2709141e7dc9eeed8
Deleted: sha256:58e9c384cd1cf65cb01080152f16380343cbc844e26a59878af8916920db998e
Deleted: sha256:8448184ef498f913e80dff1265ed66319aeb4c645fd565b984eee565d04417a1
Deleted: sha256:375e38dc8147a31408e8631a8b8a0727438d28fc8ca03043068463d65d513735
Deleted: sha256:ea72c63ed9ed92cd448fac95fe1d5ba32d97ec12ec38993ca140155f5342e886
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/python@sha256:eca632fd489e510f5a118a2fc7d6283989520a58e182c9c5847f8604444af13e
Associated tags:
- 20190625-234111
Tags:
- us.gcr.io/apache-beam-testing/jenkins/python:20190625-234111
Deleted [us.gcr.io/apache-beam-testing/jenkins/python:20190625-234111].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/python@sha256:eca632fd489e510f5a118a2fc7d6283989520a58e182c9c5847f8604444af13e].
Removed the container
Build step 'Execute shell' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
