bvolpato commented on code in PR #40859:
URL: https://github.com/apache/beam/pull/40859#discussion_r4593744775
##########
sdks/go/test/integration/expansions.go:
##########
@@ -91,47 +98,90 @@ func (es *ExpansionServices) GetAddr(label string) (string,
error) {
return "", fmt.Errorf("expansion service labeled \"%s\" not
found: %w", label, err)
}
- // Start jar on open port.
- port, err := ports.GetOpenTCP()
- if err != nil {
- return "", fmt.Errorf("cannot get open port for expansion
service labeled \"%s\": %w", label, err)
+ // A different service may claim the port between selection and jar
startup.
+ const startupAttempts = 3
+ for attempt := 0; ; attempt++ {
+ port, err := ports.GetOpenTCP()
+ if err != nil {
+ return "", fmt.Errorf("cannot get open port for
expansion service labeled \"%s\": %w", label, err)
+ }
+ portStr := strconv.Itoa(port)
+ proc, err := es.run(*ExpansionTimeout, jar, portStr)
+ if err != nil {
+ return "", fmt.Errorf("cannot run jar for expansion
service labeled \"%s\": %w", label, err)
+ }
+ addr := "localhost:" + portStr
+ if es.testMode {
+ time.Sleep(es.waitTime)
+ } else {
+ exited := make(chan error, 1)
+ go func() { exited <- proc.Wait() }()
+ if err := waitForExpansionService(addr, exited); err !=
nil {
+ proc.Kill()
+ // Retry only startup collisions, never
expansion or pipeline RPCs. A jar exits
+ // when it cannot bind the port, and another
service answers with Unimplemented.
+ collision := status.Code(err) ==
codes.Unimplemented || errors.Is(err, errJarExited)
+ if collision && attempt+1 < startupAttempts {
+ continue
+ }
+ return "", fmt.Errorf("expansion service
labeled \"%s\" failed to start at %s: %w", label, addr, err)
+ }
+ }
+ es.procs = append(es.procs, proc)
+ es.addrs[label] = addr
+ return addr, nil
}
- portStr := strconv.Itoa(port)
+}
- // Run jar and cache its info.
- proc, err := es.run(*ExpansionTimeout, jar, portStr)
+// errJarExited reports that the jar process stopped before the expansion
service was ready.
+var errJarExited = errors.New("jar exited before becoming ready")
+
+func waitForExpansionService(addr string, exited <-chan error) error {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ conn, err := grpc.NewClient("passthrough:///"+addr,
+ grpc.WithTransportCredentials(insecure.NewCredentials()),
+ // The default reconnect backoff grows longer than the polling
interval.
+ // Use a fixed delay so that the probe sees a late bind before
the deadline.
+ grpc.WithConnectParams(grpc.ConnectParams{
+ Backoff: backoff.Config{BaseDelay: 250 *
time.Millisecond, Multiplier: 1, MaxDelay: 250 * time.Millisecond},
+ MinConnectTimeout: 5 * time.Second,
+ }))
if err != nil {
- return "", fmt.Errorf("cannot run jar for expansion service
labeled \"%s\": %w", label, err)
+ return err
}
-
- addr := "localhost:" + portStr
-
- // Use different wait strategies for test mode vs production
- if es.testMode {
- // In test mode, use simple wait time for compatibility with
mock processes
- time.Sleep(es.waitTime)
- } else {
- // In production, wait for the jar to start with improved retry
logic
- maxRetries := 30
- retryDelay := time.Second
-
- for i := 0; i < maxRetries; i++ {
- time.Sleep(retryDelay)
- // Try to connect to the expansion service to verify
it's ready
- conn, err := net.DialTimeout("tcp", addr, 2*time.Second)
- if err == nil {
- conn.Close()
- break
- }
- if i == maxRetries-1 {
- return "", fmt.Errorf("expansion service
labeled \"%s\" failed to start after %d retries: %w", label, maxRetries, err)
+ defer conn.Close()
+ client := jobpb.NewExpansionServiceClient(conn)
+ for {
+ probeCtx, probeCancel := context.WithTimeout(ctx, time.Second)
+ _, err := client.DiscoverSchemaTransform(probeCtx,
&jobpb.DiscoverSchemaTransformRequest{}, grpc.WaitForReady(true))
+ probeCancel()
+ if status.Code(err) == codes.Unimplemented {
+ // Older services only implement Expand. An empty
request cannot expand
+ // a transform, but an ExpansionResponse (including its
error field)
+ // proves that the service implements the expected RPC.
Allow the
+ // remaining startup deadline for first-request
initialization.
+ _, err = client.Expand(ctx, &jobpb.ExpansionRequest{},
grpc.WaitForReady(true))
+ }
+ if err == nil {
+ select {
+ case err := <-exited:
+ return fmt.Errorf("%w: %v", errJarExited, err)
+ default:
+ return nil
}
}
+ if status.Code(err) != codes.Unavailable && status.Code(err) !=
codes.DeadlineExceeded {
+ return err
Review Comment:
Fixed in dadd9ba54a. Once the jar exits, startup returns the wrapped process
exit error even if the probe also fails. That keeps the fresh-port retry
available for a non-gRPC listener. The retry test now uses an HTTP/2 server
that returns a non-gRPC response.
##########
sdks/go/test/integration/expansions.go:
##########
@@ -91,47 +98,90 @@ func (es *ExpansionServices) GetAddr(label string) (string,
error) {
return "", fmt.Errorf("expansion service labeled \"%s\" not
found: %w", label, err)
}
- // Start jar on open port.
- port, err := ports.GetOpenTCP()
- if err != nil {
- return "", fmt.Errorf("cannot get open port for expansion
service labeled \"%s\": %w", label, err)
+ // A different service may claim the port between selection and jar
startup.
+ const startupAttempts = 3
+ for attempt := 0; ; attempt++ {
+ port, err := ports.GetOpenTCP()
+ if err != nil {
+ return "", fmt.Errorf("cannot get open port for
expansion service labeled \"%s\": %w", label, err)
+ }
+ portStr := strconv.Itoa(port)
+ proc, err := es.run(*ExpansionTimeout, jar, portStr)
+ if err != nil {
+ return "", fmt.Errorf("cannot run jar for expansion
service labeled \"%s\": %w", label, err)
+ }
+ addr := "localhost:" + portStr
+ if es.testMode {
+ time.Sleep(es.waitTime)
+ } else {
+ exited := make(chan error, 1)
+ go func() { exited <- proc.Wait() }()
+ if err := waitForExpansionService(addr, exited); err !=
nil {
+ proc.Kill()
+ // Retry only startup collisions, never
expansion or pipeline RPCs. A jar exits
+ // when it cannot bind the port, and another
service answers with Unimplemented.
+ collision := status.Code(err) ==
codes.Unimplemented || errors.Is(err, errJarExited)
+ if collision && attempt+1 < startupAttempts {
+ continue
+ }
+ return "", fmt.Errorf("expansion service
labeled \"%s\" failed to start at %s: %w", label, addr, err)
+ }
+ }
+ es.procs = append(es.procs, proc)
+ es.addrs[label] = addr
+ return addr, nil
}
- portStr := strconv.Itoa(port)
+}
- // Run jar and cache its info.
- proc, err := es.run(*ExpansionTimeout, jar, portStr)
+// errJarExited reports that the jar process stopped before the expansion
service was ready.
+var errJarExited = errors.New("jar exited before becoming ready")
+
+func waitForExpansionService(addr string, exited <-chan error) error {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ conn, err := grpc.NewClient("passthrough:///"+addr,
+ grpc.WithTransportCredentials(insecure.NewCredentials()),
+ // The default reconnect backoff grows longer than the polling
interval.
+ // Use a fixed delay so that the probe sees a late bind before
the deadline.
+ grpc.WithConnectParams(grpc.ConnectParams{
+ Backoff: backoff.Config{BaseDelay: 250 *
time.Millisecond, Multiplier: 1, MaxDelay: 250 * time.Millisecond},
+ MinConnectTimeout: 5 * time.Second,
+ }))
if err != nil {
- return "", fmt.Errorf("cannot run jar for expansion service
labeled \"%s\": %w", label, err)
+ return err
}
-
- addr := "localhost:" + portStr
-
- // Use different wait strategies for test mode vs production
- if es.testMode {
- // In test mode, use simple wait time for compatibility with
mock processes
- time.Sleep(es.waitTime)
- } else {
- // In production, wait for the jar to start with improved retry
logic
- maxRetries := 30
- retryDelay := time.Second
-
- for i := 0; i < maxRetries; i++ {
- time.Sleep(retryDelay)
- // Try to connect to the expansion service to verify
it's ready
- conn, err := net.DialTimeout("tcp", addr, 2*time.Second)
- if err == nil {
- conn.Close()
- break
- }
- if i == maxRetries-1 {
- return "", fmt.Errorf("expansion service
labeled \"%s\" failed to start after %d retries: %w", label, maxRetries, err)
+ defer conn.Close()
+ client := jobpb.NewExpansionServiceClient(conn)
+ for {
+ probeCtx, probeCancel := context.WithTimeout(ctx, time.Second)
+ _, err := client.DiscoverSchemaTransform(probeCtx,
&jobpb.DiscoverSchemaTransformRequest{}, grpc.WaitForReady(true))
+ probeCancel()
+ if status.Code(err) == codes.Unimplemented {
+ // Older services only implement Expand. An empty
request cannot expand
+ // a transform, but an ExpansionResponse (including its
error field)
+ // proves that the service implements the expected RPC.
Allow the
+ // remaining startup deadline for first-request
initialization.
+ _, err = client.Expand(ctx, &jobpb.ExpansionRequest{},
grpc.WaitForReady(true))
Review Comment:
Fixed in dadd9ba54a. The process waiter cancels the startup context as soon
as the jar exits, interrupting discovery, Expand, and the wait between probes.
A regression test exits the process while Expand is in flight and checks that
startup promptly returns the exit error.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]