[
https://issues.apache.org/jira/browse/BEAM-14509?focusedWorklogId=774694&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-774694
]
ASF GitHub Bot logged work on BEAM-14509:
-----------------------------------------
Author: ASF GitHub Bot
Created on: 25/May/22 17:00
Start Date: 25/May/22 17:00
Worklog Time Spent: 10m
Work Description: damccorm commented on code in PR #17752:
URL: https://github.com/apache/beam/pull/17752#discussion_r881905113
##########
sdks/go/pkg/beam/runners/dataflow/dataflow.go:
##########
@@ -51,26 +51,31 @@ import (
// TODO(herohde) 5/16/2017: the Dataflow flags should match the other SDKs.
var (
- endpoint = flag.String("dataflow_endpoint", "", "Dataflow
endpoint (optional).")
- stagingLocation = flag.String("staging_location", "", "GCS staging
location (required).")
- image = flag.String("worker_harness_container_image",
"", "Worker harness container image (required).")
- labels = flag.String("labels", "", "JSON-formatted
map[string]string of job labels (optional).")
- serviceAccountEmail = flag.String("service_account_email", "",
"Service account email (optional).")
- numWorkers = flag.Int64("num_workers", 0, "Number of workers
(optional).")
- maxNumWorkers = flag.Int64("max_num_workers", 0, "Maximum number
of workers during scaling (optional).")
- diskSizeGb = flag.Int64("disk_size_gb", 0, "Size of root disk
for VMs, in GB (optional).")
- diskType = flag.String("disk_type", "", "Type of root disk
for VMs (optional).")
- autoscalingAlgorithm = flag.String("autoscaling_algorithm", "",
"Autoscaling mode to use (optional).")
- zone = flag.String("zone", "", "GCP zone (optional)")
- network = flag.String("network", "", "GCP network
(optional)")
- subnetwork = flag.String("subnetwork", "", "GCP subnetwork
(optional)")
- noUsePublicIPs = flag.Bool("no_use_public_ips", false, "Workers
must not use public IP addresses (optional)")
- tempLocation = flag.String("temp_location", "", "Temp location
(optional)")
- machineType = flag.String("worker_machine_type", "", "GCE
machine type (optional)")
- minCPUPlatform = flag.String("min_cpu_platform", "", "GCE minimum
cpu platform (optional)")
- workerJar = flag.String("dataflow_worker_jar", "", "Dataflow
worker jar (optional)")
- workerRegion = flag.String("worker_region", "", "Dataflow
worker region (optional)")
- workerZone = flag.String("worker_zone", "", "Dataflow worker
zone (optional)")
+ endpoint = flag.String("dataflow_endpoint", "", "Dataflow
endpoint (optional).")
+ stagingLocation = flag.String("staging_location", "", "GCS
staging location (required).")
+ image = flag.String("worker_harness_container_image",
"", "Worker harness container image (required).")
+ labels = flag.String("labels", "", "JSON-formatted
map[string]string of job labels (optional).")
+ serviceAccountEmail = flag.String("service_account_email", "",
"Service account email (optional).")
+ numWorkers = flag.Int64("num_workers", 0, "Number of
workers (optional).")
+ workerHarnessThreads = flag.Int64("number_of_worker_harness_threads",
0, "The number of threads per each worker harness process (optional).")
+ maxNumWorkers = flag.Int64("max_num_workers", 0, "Maximum
number of workers during scaling (optional).")
+ diskSizeGb = flag.Int64("disk_size_gb", 0, "Size of root
disk for VMs, in GB (optional).")
+ diskType = flag.String("disk_type", "", "Type of root
disk for VMs (optional).")
+ autoscalingAlgorithm = flag.String("autoscaling_algorithm", "",
"Autoscaling mode to use (optional).")
+ zone = flag.String("zone", "", "GCP zone (optional)")
+ kmsKey = flag.String("dataflow_kms_key", "", "The Cloud
KMS key identifier used to encrypt data at rest (optional).")
+ network = flag.String("network", "", "GCP network
(optional)")
+ subnetwork = flag.String("subnetwork", "", "GCP subnetwork
(optional)")
+ noUsePublicIPs = flag.Bool("no_use_public_ips", false, "Workers
must not use public IP addresses (optional)")
+ tempLocation = flag.String("temp_location", "", "Temp
location (optional)")
+ machineType = flag.String("worker_machine_type", "", "GCE
machine type (optional)")
+ minCPUPlatform = flag.String("min_cpu_platform", "", "GCE
minimum cpu platform (optional)")
+ workerJar = flag.String("dataflow_worker_jar", "",
"Dataflow worker jar (optional)")
+ workerRegion = flag.String("worker_region", "", "Dataflow
worker region (optional)")
+ workerZone = flag.String("worker_zone", "", "Dataflow
worker zone (optional)")
+ dataflowServiceOptions = flag.String("dataflow_service_options", "",
"Comma separated list of additional job modes and configurations (optional)")
+ flexRSGoal = flag.String("flexrs_goal", "", "Which Flexible
Resource Scheduling mode to run in (optional)")
+ enableHotKeyLogging = flag.Bool("enable_hot_key_logging", false,
"Specifies that when a hot key is detected in the pipeline, the literal,
human-readable key is printed in the user's Cloud Logging project (optional).")
Review Comment:
Done!
##########
sdks/go/pkg/beam/runners/dataflow/dataflow.go:
##########
@@ -90,29 +95,34 @@ func init() {
// New flags that are already put into pipeline options
// should be added to this map.
var flagFilter = map[string]bool{
- "dataflow_endpoint": true,
- "staging_location": true,
- "worker_harness_container_image": true,
- "labels": true,
- "service_account_email": true,
- "num_workers": true,
- "max_num_workers": true,
- "disk_size_gb": true,
- "disk_type": true,
- "autoscaling_algorithm": true,
- "zone": true,
- "network": true,
- "subnetwork": true,
- "no_use_public_ips": true,
- "temp_location": true,
- "worker_machine_type": true,
- "min_cpu_platform": true,
- "dataflow_worker_jar": true,
- "worker_region": true,
- "worker_zone": true,
- "teardown_policy": true,
- "cpu_profiling": true,
- "session_recording": true,
+ "dataflow_endpoint": true,
+ "staging_location": true,
+ "worker_harness_container_image": true,
+ "labels": true,
+ "service_account_email": true,
+ "num_workers": true,
+ "max_num_workers": true,
+ "number_of_worker_harness_threads": true,
Review Comment:
Yeah, that's a good call. I removed the new items
Issue Time Tracking
-------------------
Worklog Id: (was: 774694)
Time Spent: 1h 50m (was: 1h 40m)
> Add Dataflow flags to Go
> ------------------------
>
> Key: BEAM-14509
> URL: https://issues.apache.org/jira/browse/BEAM-14509
> Project: Beam
> Issue Type: Improvement
> Components: runner-dataflow, sdk-go
> Reporter: Danny McCormick
> Assignee: Danny McCormick
> Priority: P2
> Time Spent: 1h 50m
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian Jira
(v8.20.7#820007)
