See
<https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Flink/4553/display/redirect>
Changes:
------------------------------------------
[...truncated 74.51 KB...]
;;
--expansion_addr)
EXPANSION_ADDR="$2"
shift # past argument
shift # past value
;;
--jenkins)
JENKINS=true
shift # past argument
;;
*) # unknown option
echo "Unknown option: $1"
exit 1
;;
esac
done
# Go to the root of the repository
cd $(git rev-parse --show-toplevel)
git rev-parse --show-toplevel
# Verify in the root of the repository
test -d sdks/go/test
# Hacky python script to find a free port. Note there is a small chance the
chosen port could
# get taken before being claimed by the job server.
SOCKET_SCRIPT="
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('localhost', 0))
print(s.getsockname()[1])
s.close()
"
# Set up environment based on runner.
if [[ "$RUNNER" == "dataflow" ]]; then
if [[ -z "$DATAFLOW_WORKER_JAR" ]]; then
DATAFLOW_WORKER_JAR=$(find
$(pwd)/runners/google-cloud-dataflow-java/worker/build/libs/beam-runners-google-cloud-dataflow-java-fn-api-worker-*.jar)
fi
echo "Using Dataflow worker jar: $DATAFLOW_WORKER_JAR"
if [[ -z "$EXPANSION_ADDR" && -n "$EXPANSION_SERVICE_JAR" ]]; then
EXPANSION_PORT=$(python -c "$SOCKET_SCRIPT")
EXPANSION_ADDR="localhost:$EXPANSION_PORT"
echo "No expansion address specified; starting a new expansion server on
$EXPANSION_ADDR"
java -jar $EXPANSION_SERVICE_JAR $EXPANSION_PORT &
EXPANSION_PID=$!
fi
elif [[ "$RUNNER" == "flink" || "$RUNNER" == "spark" || "$RUNNER" == "portable"
]]; then
if [[ -z "$ENDPOINT" ]]; then
JOB_PORT=$(python -c "$SOCKET_SCRIPT")
ENDPOINT="localhost:$JOB_PORT"
echo "No endpoint specified; starting a new $RUNNER job server on $ENDPOINT"
if [[ "$RUNNER" == "flink" ]]; then
java \
-jar $FLINK_JOB_SERVER_JAR \
--flink-master [local] \
--job-port $JOB_PORT \
--expansion-port 0 \
--artifact-port 0 &
elif [[ "$RUNNER" == "spark" ]]; then
java \
-jar $SPARK_JOB_SERVER_JAR \
--spark-master-url local \
--job-port $JOB_PORT \
--expansion-port 0 \
--artifact-port 0 &
ARGS="-p 1" # Spark runner fails if jobs are run concurrently.
elif [[ "$RUNNER" == "portable" ]]; then
python \
-m apache_beam.runners.portability.local_job_service_main \
--port $JOB_PORT &
else
echo "Unknown runner: $RUNNER"
exit 1;
fi
JOBSERVER_PID=$!
fi
if [[ -z "$EXPANSION_ADDR" && -n "$EXPANSION_SERVICE_JAR" ]]; then
EXPANSION_PORT=$(python -c "$SOCKET_SCRIPT")
EXPANSION_ADDR="localhost:$EXPANSION_PORT"
echo "No expansion address specified; starting a new expansion server on
$EXPANSION_ADDR"
java -jar $EXPANSION_SERVICE_JAR $EXPANSION_PORT &
EXPANSION_PID=$!
fi
fi
python -c "$SOCKET_SCRIPT"
python -c "$SOCKET_SCRIPT"
if [[ "$RUNNER" == "dataflow" ]]; then
# Verify docker and gcloud commands exist
command -v docker
docker -v
command -v gcloud
gcloud --version
# ensure gcloud is version 186 or above
TMPDIR=$(mktemp -d)
gcloud_ver=$(gcloud -v | head -1 | awk '{print $4}')
if [[ "$gcloud_ver" < "186" ]]
then
pushd $TMPDIR
curl
https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-186.0.0-linux-x86_64.tar.gz
--output gcloud.tar.gz
tar xf gcloud.tar.gz
./google-cloud-sdk/install.sh --quiet
. ./google-cloud-sdk/path.bash.inc
popd
gcloud components update --quiet || echo 'gcloud components update failed'
gcloud -v
fi
# Build the container
TAG=$(date +%Y%m%d-%H%M%S)
CONTAINER=us.gcr.io/$PROJECT/$USER/beam_go_sdk
echo "Using container $CONTAINER"
./gradlew :sdks:go:container:docker
-Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG
# Verify it exists
docker images | grep $TAG
# Push the container
gcloud docker -- push $CONTAINER
else
TAG=dev
./gradlew :sdks:go:container:docker -Pdocker-tag=$TAG
CONTAINER=apache/beam_go_sdk
fi
Mar 28, 2021 12:10:44 PM org.apache.beam.sdk.expansion.service.ExpansionService
loadRegisteredTransforms
INFO: Registering external transforms: [beam:transforms:xlang:test:cgbk,
beam:transforms:xlang:test:flatten, beam:transforms:xlang:test:prefix,
beam:transforms:xlang:test:multi, beam:transforms:xlang:test:gbk,
beam:transforms:xlang:test:comgl, beam:transforms:xlang:test:compk,
beam:transforms:xlang:count, beam:transforms:xlang:filter_less_than_eq,
beam:transforms:xlang:test:partition, beam:transforms:xlang:test:parquet_write,
beam:transforms:xlang:parquet_read, beam:transforms:xlang:textio_read,
beam:external:java:generate_sequence:v1]
Mar 28, 2021 12:10:49 PM org.apache.beam.runners.jobsubmission.JobServerDriver
createArtifactStagingService
INFO: ArtifactStagingService started on localhost:33613
Mar 28, 2021 12:10:49 PM org.apache.beam.runners.jobsubmission.JobServerDriver
createExpansionService
INFO: Java ExpansionService started on localhost:40463
Mar 28, 2021 12:10:49 PM org.apache.beam.runners.jobsubmission.JobServerDriver
createJobServer
INFO: JobService started on localhost:47647
Mar 28, 2021 12:10:49 PM org.apache.beam.runners.jobsubmission.JobServerDriver
run
INFO: Job server now running, terminate with Ctrl+C
ARGS="$ARGS --timeout=$TIMEOUT"
ARGS="$ARGS --runner=$RUNNER"
ARGS="$ARGS --project=$DATAFLOW_PROJECT"
ARGS="$ARGS --region=$REGION"
ARGS="$ARGS --environment_type=DOCKER"
ARGS="$ARGS --environment_config=$CONTAINER:$TAG"
ARGS="$ARGS --staging_location=$GCS_LOCATION/staging-validatesrunner-test"
ARGS="$ARGS --temp_location=$GCS_LOCATION/temp-validatesrunner-test"
ARGS="$ARGS --dataflow_worker_jar=$DATAFLOW_WORKER_JAR"
ARGS="$ARGS --endpoint=$ENDPOINT"
if [[ -n "$EXPANSION_ADDR" ]]; then
ARGS="$ARGS --expansion_addr=$EXPANSION_ADDR"
fi
# Running "go test" requires some additional setup on Jenkins.
if [[ "$JENKINS" == true ]]; then
# Copy the go repo as it is on Jenkins, to ensure we compile with the code
# being tested.
cd ..
mkdir -p temp_gopath/src/github.com/apache/beam/sdks
cp -a ./src/sdks/go ./temp_gopath/src/github.com/apache/beam/sdks
TEMP_GOPATH=$(pwd)/temp_gopath
cd ./src
echo ">>> RUNNING $RUNNER VALIDATESRUNNER TESTS"
GOPATH=$TEMP_GOPATH go test -v
github.com/apache/beam/sdks/go/test/integration/... $ARGS \
|| TEST_EXIT_CODE=$? # don't fail fast here; clean up environment before
exiting
else
echo ">>> RUNNING $RUNNER VALIDATESRUNNER TESTS"
go test -v ./sdks/go/test/integration/... $ARGS \
|| TEST_EXIT_CODE=$? # don't fail fast here; clean up environment before
exiting
fi
sdks/go/test/integration/integration.go:41:2: cannot find package
"github.com/apache/beam/sdks/go/pkg/beam/testing/ptest" in any of:
/usr/lib/go-1.12/src/github.com/apache/beam/sdks/go/pkg/beam/testing/ptest
(from $GOROOT)
/home/jenkins/go/src/github.com/apache/beam/sdks/go/pkg/beam/testing/ptest
(from $GOPATH)
sdks/go/test/integration/driver/driver.go:26:2: cannot find package
"github.com/apache/beam/sdks/go/pkg/beam" in any of:
/usr/lib/go-1.12/src/github.com/apache/beam/sdks/go/pkg/beam (from
$GOROOT)
/home/jenkins/go/src/github.com/apache/beam/sdks/go/pkg/beam (from
$GOPATH)
sdks/go/test/integration/driver/driver.go:27:2: cannot find package
"github.com/apache/beam/sdks/go/pkg/beam/io/filesystem/memfs" in any of:
/usr/lib/go-1.12/src/github.com/apache/beam/sdks/go/pkg/beam/io/filesystem/memfs
(from $GOROOT)
/home/jenkins/go/src/github.com/apache/beam/sdks/go/pkg/beam/io/filesystem/memfs
(from $GOPATH)
sdks/go/test/integration/driver/driver.go:28:2: cannot find package
"github.com/apache/beam/sdks/go/pkg/beam/log" in any of:
/usr/lib/go-1.12/src/github.com/apache/beam/sdks/go/pkg/beam/log (from
$GOROOT)
/home/jenkins/go/src/github.com/apache/beam/sdks/go/pkg/beam/log (from
$GOPATH)
sdks/go/test/integration/driver/driver.go:29:2: cannot find package
"github.com/apache/beam/sdks/go/pkg/beam/x/beamx" in any of:
/usr/lib/go-1.12/src/github.com/apache/beam/sdks/go/pkg/beam/x/beamx
(from $GOROOT)
/home/jenkins/go/src/github.com/apache/beam/sdks/go/pkg/beam/x/beamx
(from $GOPATH)
sdks/go/test/integration/driver/driver.go:30:2: cannot find package
"github.com/apache/beam/sdks/go/test/integration/primitives" in any of:
/usr/lib/go-1.12/src/github.com/apache/beam/sdks/go/test/integration/primitives
(from $GOROOT)
/home/jenkins/go/src/github.com/apache/beam/sdks/go/test/integration/primitives
(from $GOPATH)
sdks/go/test/integration/driver/driver.go:31:2: cannot find package
"github.com/apache/beam/sdks/go/test/integration/synthetic" in any of:
/usr/lib/go-1.12/src/github.com/apache/beam/sdks/go/test/integration/synthetic
(from $GOROOT)
/home/jenkins/go/src/github.com/apache/beam/sdks/go/test/integration/synthetic
(from $GOPATH)
sdks/go/test/integration/driver/driver.go:32:2: cannot find package
"github.com/apache/beam/sdks/go/test/integration/wordcount" in any of:
/usr/lib/go-1.12/src/github.com/apache/beam/sdks/go/test/integration/wordcount
(from $GOROOT)
/home/jenkins/go/src/github.com/apache/beam/sdks/go/test/integration/wordcount
(from $GOPATH)
sdks/go/test/integration/primitives/cogbk.go:22:2: cannot find package
"github.com/apache/beam/sdks/go/pkg/beam/testing/passert" in any of:
/usr/lib/go-1.12/src/github.com/apache/beam/sdks/go/pkg/beam/testing/passert
(from $GOROOT)
/home/jenkins/go/src/github.com/apache/beam/sdks/go/pkg/beam/testing/passert
(from $GOPATH)
sdks/go/test/integration/synthetic/synthetic.go:21:2: cannot find package
"github.com/apache/beam/sdks/go/pkg/beam/io/synthetic" in any of:
/usr/lib/go-1.12/src/github.com/apache/beam/sdks/go/pkg/beam/io/synthetic (from
$GOROOT)
/home/jenkins/go/src/github.com/apache/beam/sdks/go/pkg/beam/io/synthetic (from
$GOPATH)
sdks/go/test/integration/wordcount/wordcount.go:27:2: cannot find package
"github.com/apache/beam/sdks/go/pkg/beam/io/textio" in any of:
/usr/lib/go-1.12/src/github.com/apache/beam/sdks/go/pkg/beam/io/textio
(from $GOROOT)
/home/jenkins/go/src/github.com/apache/beam/sdks/go/pkg/beam/io/textio
(from $GOPATH)
sdks/go/test/integration/wordcount/wordcount.go:29:2: cannot find package
"github.com/apache/beam/sdks/go/pkg/beam/transforms/stats" in any of:
/usr/lib/go-1.12/src/github.com/apache/beam/sdks/go/pkg/beam/transforms/stats
(from $GOROOT)
/home/jenkins/go/src/github.com/apache/beam/sdks/go/pkg/beam/transforms/stats
(from $GOPATH)
if [[ "$RUNNER" == "dataflow" ]]; then
# Delete the container locally and remotely
docker rmi $CONTAINER:$TAG || echo "Failed to remove container"
gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to
delete container"
# Clean up tempdir
rm -rf $TMPDIR
fi
exit $TEST_EXIT_CODE
exit_background_processes
> Task :sdks:go:test:flinkValidatesRunner FAILED
FAILURE: Build failed with an exception.
* Where:
Build file
'<https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Flink/ws/src/sdks/go/test/build.gradle'>
line: 86
* What went wrong:
Execution failed for task ':sdks:go:test:flinkValidatesRunner'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug
option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with
Gradle 7.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See
https://docs.gradle.org/6.8/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 11m 45s
139 actionable tasks: 96 executed, 41 from cache, 2 up-to-date
Gradle was unable to watch the file system for changes. The inotify watches
limit is too low.
Publishing build scan...
https://gradle.com/s/j5fp3rt4fqizw
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
