This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new c367536 [BEAM-9714] [Go SDK] Require --region flag in Dataflow runner.
new 76d7732 Merge pull request #11348 from ibzib/beam-9199-go
c367536 is described below
commit c367536269a59515e4881771712a118c9602b32a
Author: Kyle Weaver <[email protected]>
AuthorDate: Fri Apr 3 13:44:29 2020 -0400
[BEAM-9714] [Go SDK] Require --region flag in Dataflow runner.
---
sdks/go/README.md | 2 +-
sdks/go/pkg/beam/runners/dataflow/dataflow.go | 7 ++-----
sdks/go/test/run_integration_tests.sh | 7 +++++++
website/src/get-started/quickstart-go.md | 1 +
website/src/get-started/wordcount-example.md | 2 ++
5 files changed, 13 insertions(+), 6 deletions(-)
diff --git a/sdks/go/README.md b/sdks/go/README.md
index f2d56d4..2d6faac 100644
--- a/sdks/go/README.md
+++ b/sdks/go/README.md
@@ -87,7 +87,7 @@ purse: 6
To run wordcount on dataflow runner do:
```
-$ go run wordcount.go --runner=dataflow --project=<YOUR_GCP_PROJECT>
--staging_location=<YOUR_GCS_LOCATION>/staging
--worker_harness_container_image=<YOUR_SDK_HARNESS_IMAGE_LOCATION>
--output=<YOUR_GCS_LOCATION>/output
+$ go run wordcount.go --runner=dataflow --project=<YOUR_GCP_PROJECT>
--region=<YOUR_GCP_REGION> --staging_location=<YOUR_GCS_LOCATION>/staging
--worker_harness_container_image=<YOUR_SDK_HARNESS_IMAGE_LOCATION>
--output=<YOUR_GCS_LOCATION>/output
```
The output is a GCS file in this case:
diff --git a/sdks/go/pkg/beam/runners/dataflow/dataflow.go
b/sdks/go/pkg/beam/runners/dataflow/dataflow.go
index 4543a91..8681ceb 100644
--- a/sdks/go/pkg/beam/runners/dataflow/dataflow.go
+++ b/sdks/go/pkg/beam/runners/dataflow/dataflow.go
@@ -53,7 +53,7 @@ var (
maxNumWorkers = flag.Int64("max_num_workers", 0, "Maximum number
of workers during scaling (optional).")
autoscalingAlgorithm = flag.String("autoscaling_algorithm", "",
"Autoscaling mode to use (optional).")
zone = flag.String("zone", "", "GCP zone (optional)")
- region = flag.String("region", "", "GCP Region (optional
but encouraged)")
+ region = flag.String("region", "", "GCP Region
(required)")
network = flag.String("network", "", "GCP network
(optional)")
subnetwork = flag.String("subnetwork", "", "GCP subnetwork
(optional)")
noUsePublicIPs = flag.Bool("no_use_public_ips", false, "Workers
must not use public IP addresses (optional)")
@@ -93,10 +93,7 @@ func Execute(ctx context.Context, p *beam.Pipeline) error {
return errors.New("no GCS staging location specified. Use
--staging_location=gs://<bucket>/<path>")
}
if *region == "" {
- *region = "us-central1"
- log.Warn(ctx, "--region not set; will default to us-central1.
Future releases of Beam will "+
- "require the user to set the region explicitly. "+
-
"https://cloud.google.com/compute/docs/regions-zones/regions-zones";)
+ return errors.New("No Google Cloud region specified. Use
--region=<region>. See
https://cloud.google.com/dataflow/docs/concepts/regional-endpoints";)
}
if *image == "" {
*image = getContainerImage(ctx)
diff --git a/sdks/go/test/run_integration_tests.sh
b/sdks/go/test/run_integration_tests.sh
index 78aef1a..d07e33d 100755
--- a/sdks/go/test/run_integration_tests.sh
+++ b/sdks/go/test/run_integration_tests.sh
@@ -36,6 +36,7 @@ GCS_LOCATION=gs://temp-storage-for-end-to-end-tests
# Project for the container and integration test
PROJECT=apache-beam-testing
DATAFLOW_PROJECT=apache-beam-testing
+REGION=us-central1
# Number of tests to run in parallel
PARALLEL=10
@@ -54,6 +55,11 @@ case $key in
shift # past argument
shift # past value
;;
+ --region)
+ REGION="$2"
+ shift # past argument
+ shift # past value
+ ;;
--dataflow_project)
DATAFLOW_PROJECT="$2"
shift # past argument
@@ -174,6 +180,7 @@ echo ">>> RUNNING $RUNNER INTEGRATION TESTS"
./sdks/go/build/bin/integration \
--runner=$RUNNER \
--project=$DATAFLOW_PROJECT \
+ --region=$REGION \
--environment_type=DOCKER \
--environment_config=$CONTAINER:$TAG \
--staging_location=$GCS_LOCATION/staging-validatesrunner-test \
diff --git a/website/src/get-started/quickstart-go.md
b/website/src/get-started/quickstart-go.md
index ac3fcb4..c25b72c 100644
--- a/website/src/get-started/quickstart-go.md
+++ b/website/src/get-started/quickstart-go.md
@@ -69,6 +69,7 @@ $ wordcount --input
gs://dataflow-samples/shakespeare/kinglear.txt \
--output gs://<your-gcs-bucket>/counts \
--runner dataflow \
--project your-gcp-project \
+ --region your-gcp-region \
--temp_location gs://<your-gcs-bucket>/tmp/ \
--staging_location gs://<your-gcs-bucket>/binaries/ \
--worker_harness_container_image=apache/beam_go_sdk:latest
diff --git a/website/src/get-started/wordcount-example.md
b/website/src/get-started/wordcount-example.md
index 547111d..ffa605e 100644
--- a/website/src/get-started/wordcount-example.md
+++ b/website/src/get-started/wordcount-example.md
@@ -497,6 +497,7 @@ $ wordcount --input
gs://dataflow-samples/shakespeare/kinglear.txt \
--output gs://<your-gcs-bucket>/counts \
--runner dataflow \
--project your-gcp-project \
+ --region your-gcp-region \
--temp_location gs://<your-gcs-bucket>/tmp/ \
--staging_location gs://<your-gcs-bucket>/binaries/ \
--worker_harness_container_image=apache/beam_go_sdk:latest
@@ -860,6 +861,7 @@ $ debugging_wordcount --input
gs://dataflow-samples/shakespeare/kinglear.txt \
--output gs://<your-gcs-bucket>/counts \
--runner dataflow \
--project your-gcp-project \
+ --region your-gcp-region \
--temp_location gs://<your-gcs-bucket>/tmp/ \
--staging_location gs://<your-gcs-bucket>/binaries/ \
--worker_harness_container_image=apache-docker-beam-snapshots-docker.bintray.io/beam/go:20180515
