See
<https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Flink/5106/display/redirect?page=changes>
Changes:
[zhoufek] [BEAM-9487] Disable allowing unsafe triggers by default
[noreply] [BEAM-10917] Add support for BigQuery Read API in Python BEAM (#15185)
[ryanthompson591] Change filter to also retry on 408 errors.
------------------------------------------
[...truncated 77.32 KB...]
exit 1
;;
esac
done
# Go to the root of the repository
cd $(git rev-parse --show-toplevel)
git rev-parse --show-toplevel
# Verify in the root of the repository
test -d sdks/go/test
# Hacky python script to find a free port. Note there is a small chance the
chosen port could
# get taken before being claimed by the job server.
SOCKET_SCRIPT="
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('localhost', 0))
print(s.getsockname()[1])
s.close()
"
# Set up environment based on runner.
if [[ "$RUNNER" == "dataflow" ]]; then
if [[ -z "$DATAFLOW_WORKER_JAR" ]]; then
DATAFLOW_WORKER_JAR=$(find
$(pwd)/runners/google-cloud-dataflow-java/worker/build/libs/beam-runners-google-cloud-dataflow-java-fn-api-worker-*.jar)
fi
echo "Using Dataflow worker jar: $DATAFLOW_WORKER_JAR"
if [[ -z "$TEST_EXPANSION_ADDR" && -n "$TEST_EXPANSION_JAR" ]]; then
EXPANSION_PORT=$(python3 -c "$SOCKET_SCRIPT")
TEST_EXPANSION_ADDR="localhost:$EXPANSION_PORT"
echo "No test expansion address specified; starting a new test expansion
server on $TEST_EXPANSION_ADDR"
java -jar $TEST_EXPANSION_JAR $EXPANSION_PORT &
TEST_EXPANSION_PID=$!
fi
if [[ -z "$IO_EXPANSION_ADDR" && -n "$IO_EXPANSION_JAR" ]]; then
EXPANSION_PORT=$(python3 -c "$SOCKET_SCRIPT")
IO_EXPANSION_ADDR="localhost:$EXPANSION_PORT"
echo "No IO expansion address specified; starting a new IO expansion server
on $IO_EXPANSION_ADDR"
java -jar $IO_EXPANSION_JAR $EXPANSION_PORT &
IO_EXPANSION_PID=$!
fi
elif [[ "$RUNNER" == "flink" || "$RUNNER" == "spark" || "$RUNNER" == "samza" ||
"$RUNNER" == "portable" ]]; then
if [[ -z "$ENDPOINT" ]]; then
JOB_PORT=$(python3 -c "$SOCKET_SCRIPT")
ENDPOINT="localhost:$JOB_PORT"
echo "No endpoint specified; starting a new $RUNNER job server on $ENDPOINT"
if [[ "$RUNNER" == "flink" ]]; then
java \
-jar $FLINK_JOB_SERVER_JAR \
--flink-master [local] \
--job-port $JOB_PORT \
--expansion-port 0 \
--artifact-port 0 &
SIMULTANEOUS=1
elif [[ "$RUNNER" == "samza" ]]; then
java \
-jar $SAMZA_JOB_SERVER_JAR \
--job-port $JOB_PORT \
--expansion-port 0 \
--artifact-port 0 &
SIMULTANEOUS=1
elif [[ "$RUNNER" == "spark" ]]; then
java \
-jar $SPARK_JOB_SERVER_JAR \
--spark-master-url local \
--job-port $JOB_PORT \
--expansion-port 0 \
--artifact-port 0 &
SIMULTANEOUS=1 # Spark runner fails if jobs are run concurrently.
elif [[ "$RUNNER" == "portable" ]]; then
python3 \
-m apache_beam.runners.portability.local_job_service_main \
--port $JOB_PORT &
else
echo "Unknown runner: $RUNNER"
exit 1;
fi
JOBSERVER_PID=$!
fi
if [[ -z "$TEST_EXPANSION_ADDR" && -n "$TEST_EXPANSION_JAR" ]]; then
EXPANSION_PORT=$(python3 -c "$SOCKET_SCRIPT")
TEST_EXPANSION_ADDR="localhost:$EXPANSION_PORT"
echo "No test expansion address specified; starting a new test expansion
server on $TEST_EXPANSION_ADDR"
java -jar $TEST_EXPANSION_JAR $EXPANSION_PORT &
TEST_EXPANSION_PID=$!
fi
if [[ -z "$IO_EXPANSION_ADDR" && -n "$IO_EXPANSION_JAR" ]]; then
EXPANSION_PORT=$(python3 -c "$SOCKET_SCRIPT")
IO_EXPANSION_ADDR="localhost:$EXPANSION_PORT"
echo "No IO expansion address specified; starting a new IO expansion server
on $IO_EXPANSION_ADDR"
java -jar $IO_EXPANSION_JAR $EXPANSION_PORT &
IO_EXPANSION_PID=$!
fi
fi
python3 -c "$SOCKET_SCRIPT"
No endpoint specified; starting a new flink job server on localhost:36255
python3 -c "$SOCKET_SCRIPT"
No test expansion address specified; starting a new test expansion server on
localhost:41649
if [[ "$RUNNER" == "dataflow" ]]; then
# Verify docker and gcloud commands exist
command -v docker
docker -v
command -v gcloud
gcloud --version
# ensure gcloud is version 186 or above
TMPDIR=$(mktemp -d)
gcloud_ver=$(gcloud -v | head -1 | awk '{print $4}')
if [[ "$gcloud_ver" < "186" ]]
then
pushd $TMPDIR
curl
https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-186.0.0-linux-x86_64.tar.gz
--output gcloud.tar.gz
tar xf gcloud.tar.gz
./google-cloud-sdk/install.sh --quiet
. ./google-cloud-sdk/path.bash.inc
popd
gcloud components update --quiet || echo 'gcloud components update failed'
gcloud -v
fi
# Build the container
TAG=$(date +%Y%m%d-%H%M%S)
CONTAINER=us.gcr.io/$PROJECT/$USER/beam_go_sdk
echo "Using container $CONTAINER"
./gradlew :sdks:go:container:docker
-Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG
# Verify it exists
docker images | grep $TAG
# Push the container
gcloud docker -- push $CONTAINER:$TAG
if [[ -n "$TEST_EXPANSION_ADDR" || -n "$IO_EXPANSION_ADDR" ]]; then
ARGS="$ARGS --experiments=use_portable_job_submission"
if [[ -z "$SDK_OVERRIDES" ]]; then
# Build the java container for cross-language
JAVA_TAG=$(date +%Y%m%d-%H%M%S)
JAVA_CONTAINER=us.gcr.io/$PROJECT/$USER/beam_java11_sdk
echo "Using container $JAVA_CONTAINER for cross-language java transforms"
./gradlew :sdks:java:container:java11:docker
-Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$JAVA_TAG
# Verify it exists
docker images | grep $JAVA_TAG
# Push the container
gcloud docker -- push $JAVA_CONTAINER:$JAVA_TAG
SDK_OVERRIDES=".*java.*,$JAVA_CONTAINER:$JAVA_TAG"
fi
fi
else
TAG=dev
./gradlew :sdks:go:container:docker -Pdocker-tag=$TAG
CONTAINER=apache/beam_go_sdk
fi
Starting expansion service at localhost:41649
Aug 21, 2021 12:04:33 AM org.apache.beam.sdk.expansion.service.ExpansionService
loadRegisteredTransforms
INFO: Registering external transforms: [beam:transforms:xlang:test:cgbk,
beam:transforms:xlang:test:flatten, beam:transforms:xlang:test:prefix,
beam:transforms:xlang:test:multi, beam:transforms:xlang:test:gbk,
beam:transforms:xlang:test:comgl, beam:transforms:xlang:test:compk,
beam:transforms:xlang:count, beam:transforms:xlang:filter_less_than_eq,
beam:transforms:xlang:test:partition, beam:transforms:xlang:test:parquet_write,
beam:transforms:xlang:parquet_read, beam:transforms:xlang:textio_read,
beam:external:java:generate_sequence:v1]
beam:transforms:xlang:test:cgbk:
org.apache.beam.sdk.testing.expansion.TestExpansionService$TestServiceRegistrar$TestCoGroupByKeyTransformProvider@747ddf94
beam:transforms:xlang:test:flatten:
org.apache.beam.sdk.testing.expansion.TestExpansionService$TestServiceRegistrar$TestFlattenTransformProvider@35e2d654
beam:transforms:xlang:test:prefix:
org.apache.beam.sdk.expansion.service.ExpansionService$ExternalTransformRegistrarLoader$$Lambda$47/2005169944@1bd4fdd
beam:transforms:xlang:test:multi:
org.apache.beam.sdk.expansion.service.ExpansionService$ExternalTransformRegistrarLoader$$Lambda$47/2005169944@55183b20
beam:transforms:xlang:test:gbk:
org.apache.beam.sdk.expansion.service.ExpansionService$ExternalTransformRegistrarLoader$$Lambda$47/2005169944@4f83df68
beam:transforms:xlang:test:comgl:
org.apache.beam.sdk.expansion.service.ExpansionService$ExternalTransformRegistrarLoader$$Lambda$47/2005169944@6cf0e0ba
beam:transforms:xlang:test:compk:
org.apache.beam.sdk.expansion.service.ExpansionService$ExternalTransformRegistrarLoader$$Lambda$47/2005169944@7d8995e
beam:transforms:xlang:count:
org.apache.beam.sdk.expansion.service.ExpansionService$ExternalTransformRegistrarLoader$$Lambda$47/2005169944@130d63be
beam:transforms:xlang:filter_less_than_eq:
org.apache.beam.sdk.expansion.service.ExpansionService$ExternalTransformRegistrarLoader$$Lambda$47/2005169944@42a48628
beam:transforms:xlang:test:partition:
org.apache.beam.sdk.expansion.service.ExpansionService$ExternalTransformRegistrarLoader$$Lambda$47/2005169944@293a5bf6
beam:transforms:xlang:test:parquet_write:
org.apache.beam.sdk.expansion.service.ExpansionService$ExternalTransformRegistrarLoader$$Lambda$47/2005169944@6aeb35e6
beam:transforms:xlang:parquet_read:
org.apache.beam.sdk.expansion.service.ExpansionService$ExternalTransformRegistrarLoader$$Lambda$47/2005169944@1283bb96
beam:transforms:xlang:textio_read:
org.apache.beam.sdk.expansion.service.ExpansionService$ExternalTransformRegistrarLoader$$Lambda$47/2005169944@74f0ea28
beam:external:java:generate_sequence:v1:
org.apache.beam.sdk.expansion.service.ExpansionService$ExternalTransformRegistrarLoader$$Lambda$47/2005169944@f6efaab
> Task :buildSrc:compileJava NO-SOURCE
Aug 21, 2021 12:04:34 AM
org.apache.beam.sdk.io.aws2.s3.DefaultS3ClientBuilderFactory createBuilder
INFO: The AWS S3 Beam extension was included in this build, but the awsRegion
flag was not specified. If you don't plan to use S3, then ignore this message.
The message received from the daemon indicates that the daemon has disappeared.
Build request sent: Build{id=dd247bfe-d0db-4989-8426-36abaf4a43f6,
currentDir=<https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Flink/ws/src}>
Attempting to read last messages from the daemon log...
Daemon pid: 1037
log file: /home/jenkins/.gradle/daemon/6.8.3/daemon-1037.out.log
----- Last 20 lines from daemon log file - daemon-1037.out.log -----
2021-08-21T00:04:33.030+0000 [DEBUG]
[org.gradle.launcher.daemon.server.DefaultDaemonConnection] thread 243:
Received non-IO message from client:
Build{id=dd247bfe-d0db-4989-8426-36abaf4a43f6,
currentDir=<https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Flink/ws/src}>
2021-08-21T00:04:33.030+0000 [INFO]
[org.gradle.launcher.daemon.server.DefaultIncomingConnectionHandler] Received
command: Build{id=dd247bfe-d0db-4989-8426-36abaf4a43f6,
currentDir=<https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Flink/ws/src}.>
2021-08-21T00:04:33.030+0000 [DEBUG]
[org.gradle.launcher.daemon.server.DefaultIncomingConnectionHandler] Starting
executing command: Build{id=dd247bfe-d0db-4989-8426-36abaf4a43f6,
currentDir=<https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Flink/ws/src}>
with connection: socket connection from /127.0.0.1:43055 to /127.0.0.1:34734.
2021-08-21T00:04:33.031+0000 [ERROR]
[org.gradle.launcher.daemon.server.DaemonStateCoordinator] Command execution:
started DaemonCommandExecution[command =
Build{id=dd247bfe-d0db-4989-8426-36abaf4a43f6,
currentDir=<https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Flink/ws/src},>
connection = DefaultDaemonConnection: socket connection from /127.0.0.1:43055
to /127.0.0.1:34734] after 1.0 minutes of idle
2021-08-21T00:04:33.031+0000 [INFO]
[org.gradle.launcher.daemon.server.DaemonRegistryUpdater] Marking the daemon as
busy, address: [83f1c32c-b3a8-461e-ba2e-c8dce8eb0d3d port:43055,
addresses:[localhost/127.0.0.1]]
2021-08-21T00:04:33.031+0000 [DEBUG]
[org.gradle.launcher.daemon.registry.PersistentDaemonRegistry] Marking busy by
address: [83f1c32c-b3a8-461e-ba2e-c8dce8eb0d3d port:43055,
addresses:[localhost/127.0.0.1]]
2021-08-21T00:04:33.031+0000 [DEBUG]
[org.gradle.cache.internal.DefaultFileLockManager] Waiting to acquire exclusive
lock on daemon addresses registry.
2021-08-21T00:04:33.032+0000 [DEBUG]
[org.gradle.cache.internal.DefaultFileLockManager] Lock acquired on daemon
addresses registry.
2021-08-21T00:04:33.033+0000 [DEBUG]
[org.gradle.cache.internal.DefaultFileLockManager] Releasing lock on daemon
addresses registry.
2021-08-21T00:04:33.033+0000 [DEBUG]
[org.gradle.launcher.daemon.server.DaemonStateCoordinator] resetting idle timer
2021-08-21T00:04:33.033+0000 [DEBUG]
[org.gradle.launcher.daemon.server.DaemonStateCoordinator] daemon is running.
Sleeping until state changes.
2021-08-21T00:04:33.034+0000 [INFO]
[org.gradle.launcher.daemon.server.exec.StartBuildOrRespondWithBusy] Daemon is
about to start building Build{id=dd247bfe-d0db-4989-8426-36abaf4a43f6,
currentDir=<https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Flink/ws/src}.>
Dispatching build started information...
2021-08-21T00:04:33.034+0000 [DEBUG]
[org.gradle.launcher.daemon.server.SynchronizedDispatchConnection] thread 245:
dispatching org.gradle.launcher.daemon.protocol.BuildStarted@1b6889ff
2021-08-21T00:04:33.035+0000 [DEBUG]
[org.gradle.launcher.daemon.server.exec.EstablishBuildEnvironment] Configuring
env variables: [PATH, RUN_DISPLAY_URL, HUDSON_HOME, RUN_CHANGES_DISPLAY_URL,
JOB_URL, HUDSON_COOKIE, MAIL, JENKINS_SERVER_COOKIE, LOGNAME, PWD,
RUN_TESTS_DISPLAY_URL, JENKINS_URL, SHELL, BUILD_TAG, ROOT_BUILD_CAUSE,
BUILD_CAUSE_TIMERTRIGGER, OLDPWD, GIT_CHECKOUT_DIR, JENKINS_HOME, sha1,
CODECOV_TOKEN, NODE_NAME, BUILD_DISPLAY_NAME, JOB_DISPLAY_URL, GIT_BRANCH,
SETUPTOOLS_USE_DISTUTILS, SHLVL, WORKSPACE_TMP, GIT_PREVIOUS_COMMIT, JAVA_HOME,
BUILD_ID, LANG, XDG_SESSION_ID, JOB_NAME, SPARK_LOCAL_IP, BUILD_CAUSE,
GIT_PREVIOUS_SUCCESSFUL_COMMIT, NODE_LABELS, HUDSON_URL, WORKSPACE,
ROOT_BUILD_CAUSE_TIMERTRIGGER, _, GIT_COMMIT, COVERALLS_REPO_TOKEN,
EXECUTOR_NUMBER, HUDSON_SERVER_COOKIE, SSH_CLIENT, JOB_BASE_NAME, USER,
SSH_CONNECTION, BUILD_NUMBER, BUILD_URL, RUN_ARTIFACTS_DISPLAY_URL, GIT_URL,
XDG_RUNTIME_DIR, HOME]
2021-08-21T00:04:33.042+0000 [DEBUG]
[org.gradle.launcher.daemon.server.exec.LogToClient] About to start relaying
all logs to the client via the connection.
2021-08-21T00:04:33.042+0000 [INFO]
[org.gradle.launcher.daemon.server.exec.LogToClient] The client will now
receive all logging from the daemon (pid: 1037). The daemon log file:
/home/jenkins/.gradle/daemon/6.8.3/daemon-1037.out.log
2021-08-21T00:04:33.048+0000 [INFO]
[org.gradle.launcher.daemon.server.exec.LogAndCheckHealth] Starting 2nd build
in daemon [uptime: 4 mins 4.869 secs, performance: 98%, GC rate: 0.03/s, heap
usage: 0% of 17 GiB]
2021-08-21T00:04:33.051+0000 [DEBUG]
[org.gradle.launcher.daemon.server.exec.ExecuteBuild] The daemon has started
executing the build.
2021-08-21T00:04:33.051+0000 [DEBUG]
[org.gradle.launcher.daemon.server.exec.ExecuteBuild] Executing build with
daemon context:
DefaultDaemonContext[uid=268b40f6-9163-4944-8e34-40b197e1ba36,javaHome=/usr/lib/jvm/java-8-openjdk-amd64,daemonRegistryDir=/home/jenkins/.gradle/daemon,pid=1037,idleTimeout=10800000,priority=NORMAL,daemonOpts=-Xss10240k,-Dfile.encoding=UTF-8,-Duser.country=US,-Duser.language=en,-Duser.variant]
Daemon vm is shutting down... The daemon has exited normally or was terminated
in response to a user interrupt.
----- End of the daemon log -----
FAILURE: Build failed with an exception.
* What went wrong:
Gradle build daemon disappeared unexpectedly (it may have been killed or may
have crashed)
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug
option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
> Task :buildSrc:compileGroovy
exit_background_processes
> Task :sdks:go:test:flinkValidatesRunner FAILED
FAILURE: Build failed with an exception.
* Where:
Build file
'<https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Flink/ws/src/sdks/go/test/build.gradle'>
line: 88
* What went wrong:
Execution failed for task ':sdks:go:test:flinkValidatesRunner'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug
option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with
Gradle 7.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See
https://docs.gradle.org/6.8.3/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 4m 9s
163 actionable tasks: 115 executed, 43 from cache, 5 up-to-date
Publishing build scan...
https://gradle.com/s/pau4ctgmwrgwa
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
