github-actions[bot] commented on code in PR #40859:
URL: https://github.com/apache/beam/pull/40859#discussion_r4593545452
##########
sdks/go/test/integration/expansions.go:
##########
@@ -91,47 +98,90 @@ func (es *ExpansionServices) GetAddr(label string) (string,
error) {
return "", fmt.Errorf("expansion service labeled \"%s\" not
found: %w", label, err)
}
- // Start jar on open port.
- port, err := ports.GetOpenTCP()
- if err != nil {
- return "", fmt.Errorf("cannot get open port for expansion
service labeled \"%s\": %w", label, err)
+ // A different service may claim the port between selection and jar
startup.
+ const startupAttempts = 3
+ for attempt := 0; ; attempt++ {
+ port, err := ports.GetOpenTCP()
+ if err != nil {
+ return "", fmt.Errorf("cannot get open port for
expansion service labeled \"%s\": %w", label, err)
+ }
+ portStr := strconv.Itoa(port)
+ proc, err := es.run(*ExpansionTimeout, jar, portStr)
+ if err != nil {
+ return "", fmt.Errorf("cannot run jar for expansion
service labeled \"%s\": %w", label, err)
+ }
+ addr := "localhost:" + portStr
+ if es.testMode {
+ time.Sleep(es.waitTime)
+ } else {
+ exited := make(chan error, 1)
+ go func() { exited <- proc.Wait() }()
+ if err := waitForExpansionService(addr, exited); err !=
nil {
+ proc.Kill()
+ // Retry only startup collisions, never
expansion or pipeline RPCs. A jar exits
+ // when it cannot bind the port, and another
service answers with Unimplemented.
+ collision := status.Code(err) ==
codes.Unimplemented || errors.Is(err, errJarExited)
+ if collision && attempt+1 < startupAttempts {
+ continue
+ }
+ return "", fmt.Errorf("expansion service
labeled \"%s\" failed to start at %s: %w", label, addr, err)
+ }
+ }
+ es.procs = append(es.procs, proc)
+ es.addrs[label] = addr
+ return addr, nil
}
- portStr := strconv.Itoa(port)
+}
- // Run jar and cache its info.
- proc, err := es.run(*ExpansionTimeout, jar, portStr)
+// errJarExited reports that the jar process stopped before the expansion
service was ready.
+var errJarExited = errors.New("jar exited before becoming ready")
+
+func waitForExpansionService(addr string, exited <-chan error) error {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ conn, err := grpc.NewClient("passthrough:///"+addr,
+ grpc.WithTransportCredentials(insecure.NewCredentials()),
+ // The default reconnect backoff grows longer than the polling
interval.
+ // Use a fixed delay so that the probe sees a late bind before
the deadline.
+ grpc.WithConnectParams(grpc.ConnectParams{
+ Backoff: backoff.Config{BaseDelay: 250 *
time.Millisecond, Multiplier: 1, MaxDelay: 250 * time.Millisecond},
+ MinConnectTimeout: 5 * time.Second,
+ }))
if err != nil {
- return "", fmt.Errorf("cannot run jar for expansion service
labeled \"%s\": %w", label, err)
+ return err
}
-
- addr := "localhost:" + portStr
-
- // Use different wait strategies for test mode vs production
- if es.testMode {
- // In test mode, use simple wait time for compatibility with
mock processes
- time.Sleep(es.waitTime)
- } else {
- // In production, wait for the jar to start with improved retry
logic
- maxRetries := 30
- retryDelay := time.Second
-
- for i := 0; i < maxRetries; i++ {
- time.Sleep(retryDelay)
- // Try to connect to the expansion service to verify
it's ready
- conn, err := net.DialTimeout("tcp", addr, 2*time.Second)
- if err == nil {
- conn.Close()
- break
- }
- if i == maxRetries-1 {
- return "", fmt.Errorf("expansion service
labeled \"%s\" failed to start after %d retries: %w", label, maxRetries, err)
+ defer conn.Close()
+ client := jobpb.NewExpansionServiceClient(conn)
+ for {
+ probeCtx, probeCancel := context.WithTimeout(ctx, time.Second)
+ _, err := client.DiscoverSchemaTransform(probeCtx,
&jobpb.DiscoverSchemaTransformRequest{}, grpc.WaitForReady(true))
+ probeCancel()
+ if status.Code(err) == codes.Unimplemented {
+ // Older services only implement Expand. An empty
request cannot expand
+ // a transform, but an ExpansionResponse (including its
error field)
+ // proves that the service implements the expected RPC.
Allow the
+ // remaining startup deadline for first-request
initialization.
+ _, err = client.Expand(ctx, &jobpb.ExpansionRequest{},
grpc.WaitForReady(true))
+ }
+ if err == nil {
+ select {
+ case err := <-exited:
+ return fmt.Errorf("%w: %v", errJarExited, err)
+ default:
+ return nil
}
}
+ if status.Code(err) != codes.Unavailable && status.Code(err) !=
codes.DeadlineExceeded {
+ return err
Review Comment:
**🟡 Medium**: If a non-gRPC service (such as an HTTP/1.x endpoint or another
TCP service) claims the port before the jar binds, the jar will exit with a
bind error while `DiscoverSchemaTransform` fails with a non-`Unimplemented`
gRPC status (typically `codes.Internal` due to a malformed HTTP/2
frame/preface). Returning `err` here without checking `exited` causes `GetAddr`
to see neither `codes.Unimplemented` nor `errJarExited`, skipping the retry
loop on port collision.
Consider checking `select { case exitErr := <-exited: ... default: }` before
returning `err`, or canceling `ctx` as soon as `exited` receives so child
process exit always takes precedence.
##########
sdks/go/test/integration/expansions.go:
##########
@@ -91,47 +98,90 @@ func (es *ExpansionServices) GetAddr(label string) (string,
error) {
return "", fmt.Errorf("expansion service labeled \"%s\" not
found: %w", label, err)
}
- // Start jar on open port.
- port, err := ports.GetOpenTCP()
- if err != nil {
- return "", fmt.Errorf("cannot get open port for expansion
service labeled \"%s\": %w", label, err)
+ // A different service may claim the port between selection and jar
startup.
+ const startupAttempts = 3
+ for attempt := 0; ; attempt++ {
+ port, err := ports.GetOpenTCP()
+ if err != nil {
+ return "", fmt.Errorf("cannot get open port for
expansion service labeled \"%s\": %w", label, err)
+ }
+ portStr := strconv.Itoa(port)
+ proc, err := es.run(*ExpansionTimeout, jar, portStr)
+ if err != nil {
+ return "", fmt.Errorf("cannot run jar for expansion
service labeled \"%s\": %w", label, err)
+ }
+ addr := "localhost:" + portStr
+ if es.testMode {
+ time.Sleep(es.waitTime)
+ } else {
+ exited := make(chan error, 1)
+ go func() { exited <- proc.Wait() }()
+ if err := waitForExpansionService(addr, exited); err !=
nil {
+ proc.Kill()
+ // Retry only startup collisions, never
expansion or pipeline RPCs. A jar exits
+ // when it cannot bind the port, and another
service answers with Unimplemented.
+ collision := status.Code(err) ==
codes.Unimplemented || errors.Is(err, errJarExited)
+ if collision && attempt+1 < startupAttempts {
+ continue
+ }
+ return "", fmt.Errorf("expansion service
labeled \"%s\" failed to start at %s: %w", label, addr, err)
+ }
+ }
+ es.procs = append(es.procs, proc)
+ es.addrs[label] = addr
+ return addr, nil
}
- portStr := strconv.Itoa(port)
+}
- // Run jar and cache its info.
- proc, err := es.run(*ExpansionTimeout, jar, portStr)
+// errJarExited reports that the jar process stopped before the expansion
service was ready.
+var errJarExited = errors.New("jar exited before becoming ready")
+
+func waitForExpansionService(addr string, exited <-chan error) error {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ conn, err := grpc.NewClient("passthrough:///"+addr,
+ grpc.WithTransportCredentials(insecure.NewCredentials()),
+ // The default reconnect backoff grows longer than the polling
interval.
+ // Use a fixed delay so that the probe sees a late bind before
the deadline.
+ grpc.WithConnectParams(grpc.ConnectParams{
+ Backoff: backoff.Config{BaseDelay: 250 *
time.Millisecond, Multiplier: 1, MaxDelay: 250 * time.Millisecond},
+ MinConnectTimeout: 5 * time.Second,
+ }))
if err != nil {
- return "", fmt.Errorf("cannot run jar for expansion service
labeled \"%s\": %w", label, err)
+ return err
}
-
- addr := "localhost:" + portStr
-
- // Use different wait strategies for test mode vs production
- if es.testMode {
- // In test mode, use simple wait time for compatibility with
mock processes
- time.Sleep(es.waitTime)
- } else {
- // In production, wait for the jar to start with improved retry
logic
- maxRetries := 30
- retryDelay := time.Second
-
- for i := 0; i < maxRetries; i++ {
- time.Sleep(retryDelay)
- // Try to connect to the expansion service to verify
it's ready
- conn, err := net.DialTimeout("tcp", addr, 2*time.Second)
- if err == nil {
- conn.Close()
- break
- }
- if i == maxRetries-1 {
- return "", fmt.Errorf("expansion service
labeled \"%s\" failed to start after %d retries: %w", label, maxRetries, err)
+ defer conn.Close()
+ client := jobpb.NewExpansionServiceClient(conn)
+ for {
+ probeCtx, probeCancel := context.WithTimeout(ctx, time.Second)
+ _, err := client.DiscoverSchemaTransform(probeCtx,
&jobpb.DiscoverSchemaTransformRequest{}, grpc.WaitForReady(true))
+ probeCancel()
+ if status.Code(err) == codes.Unimplemented {
+ // Older services only implement Expand. An empty
request cannot expand
+ // a transform, but an ExpansionResponse (including its
error field)
+ // proves that the service implements the expected RPC.
Allow the
+ // remaining startup deadline for first-request
initialization.
+ _, err = client.Expand(ctx, &jobpb.ExpansionRequest{},
grpc.WaitForReady(true))
Review Comment:
**🟢 Low**: Because neither `ctx` nor `probeCtx` is canceled when `exited`
receives, an early jar exit will block for the full 1s `probeCtx` timeout on
`DiscoverSchemaTransform` (plus another 1s in `time.After(time.Second)` between
probes), or up to the full 30s `ctx` timeout if `Expand` is waiting with
`grpc.WaitForReady(true)`. Watching `exited` in a goroutine to cancel `ctx`
immediately on process exit would abort in-flight probes without delay.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]