bvolpato commented on code in PR #40859:
URL: https://github.com/apache/beam/pull/40859#discussion_r4593744775


##########
sdks/go/test/integration/expansions.go:
##########
@@ -91,47 +98,90 @@ func (es *ExpansionServices) GetAddr(label string) (string, 
error) {
                return "", fmt.Errorf("expansion service labeled \"%s\" not 
found: %w", label, err)
        }
 
-       // Start jar on open port.
-       port, err := ports.GetOpenTCP()
-       if err != nil {
-               return "", fmt.Errorf("cannot get open port for expansion 
service labeled \"%s\": %w", label, err)
+       // A different service may claim the port between selection and jar 
startup.
+       const startupAttempts = 3
+       for attempt := 0; ; attempt++ {
+               port, err := ports.GetOpenTCP()
+               if err != nil {
+                       return "", fmt.Errorf("cannot get open port for 
expansion service labeled \"%s\": %w", label, err)
+               }
+               portStr := strconv.Itoa(port)
+               proc, err := es.run(*ExpansionTimeout, jar, portStr)
+               if err != nil {
+                       return "", fmt.Errorf("cannot run jar for expansion 
service labeled \"%s\": %w", label, err)
+               }
+               addr := "localhost:" + portStr
+               if es.testMode {
+                       time.Sleep(es.waitTime)
+               } else {
+                       exited := make(chan error, 1)
+                       go func() { exited <- proc.Wait() }()
+                       if err := waitForExpansionService(addr, exited); err != 
nil {
+                               proc.Kill()
+                               // Retry only startup collisions, never 
expansion or pipeline RPCs. A jar exits
+                               // when it cannot bind the port, and another 
service answers with Unimplemented.
+                               collision := status.Code(err) == 
codes.Unimplemented || errors.Is(err, errJarExited)
+                               if collision && attempt+1 < startupAttempts {
+                                       continue
+                               }
+                               return "", fmt.Errorf("expansion service 
labeled \"%s\" failed to start at %s: %w", label, addr, err)
+                       }
+               }
+               es.procs = append(es.procs, proc)
+               es.addrs[label] = addr
+               return addr, nil
        }
-       portStr := strconv.Itoa(port)
+}
 
-       // Run jar and cache its info.
-       proc, err := es.run(*ExpansionTimeout, jar, portStr)
+// errJarExited reports that the jar process stopped before the expansion 
service was ready.
+var errJarExited = errors.New("jar exited before becoming ready")
+
+func waitForExpansionService(addr string, exited <-chan error) error {
+       ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+       defer cancel()
+       conn, err := grpc.NewClient("passthrough:///"+addr,
+               grpc.WithTransportCredentials(insecure.NewCredentials()),
+               // The default reconnect backoff grows longer than the polling 
interval.
+               // Use a fixed delay so that the probe sees a late bind before 
the deadline.
+               grpc.WithConnectParams(grpc.ConnectParams{
+                       Backoff:           backoff.Config{BaseDelay: 250 * 
time.Millisecond, Multiplier: 1, MaxDelay: 250 * time.Millisecond},
+                       MinConnectTimeout: 5 * time.Second,
+               }))
        if err != nil {
-               return "", fmt.Errorf("cannot run jar for expansion service 
labeled \"%s\": %w", label, err)
+               return err
        }
-       
-       addr := "localhost:" + portStr
-
-       // Use different wait strategies for test mode vs production
-       if es.testMode {
-               // In test mode, use simple wait time for compatibility with 
mock processes
-               time.Sleep(es.waitTime)
-       } else {
-               // In production, wait for the jar to start with improved retry 
logic
-               maxRetries := 30
-               retryDelay := time.Second
-               
-               for i := 0; i < maxRetries; i++ {
-                       time.Sleep(retryDelay)
-                       // Try to connect to the expansion service to verify 
it's ready
-                       conn, err := net.DialTimeout("tcp", addr, 2*time.Second)
-                       if err == nil {
-                               conn.Close()
-                               break
-                       }
-                       if i == maxRetries-1 {
-                               return "", fmt.Errorf("expansion service 
labeled \"%s\" failed to start after %d retries: %w", label, maxRetries, err)
+       defer conn.Close()
+       client := jobpb.NewExpansionServiceClient(conn)
+       for {
+               probeCtx, probeCancel := context.WithTimeout(ctx, time.Second)
+               _, err := client.DiscoverSchemaTransform(probeCtx, 
&jobpb.DiscoverSchemaTransformRequest{}, grpc.WaitForReady(true))
+               probeCancel()
+               if status.Code(err) == codes.Unimplemented {
+                       // Older services only implement Expand. An empty 
request cannot expand
+                       // a transform, but an ExpansionResponse (including its 
error field)
+                       // proves that the service implements the expected RPC. 
Allow the
+                       // remaining startup deadline for first-request 
initialization.
+                       _, err = client.Expand(ctx, &jobpb.ExpansionRequest{}, 
grpc.WaitForReady(true))
+               }
+               if err == nil {
+                       select {
+                       case err := <-exited:
+                               return fmt.Errorf("%w: %v", errJarExited, err)
+                       default:
+                               return nil
                        }
                }
+               if status.Code(err) != codes.Unavailable && status.Code(err) != 
codes.DeadlineExceeded {
+                       return err

Review Comment:
   Fixed in dadd9ba54a. Once the jar exits, startup returns the wrapped process 
exit error even if the probe also fails. That keeps the fresh-port retry 
available for a non-gRPC listener. The retry test now uses an HTTP/2 server 
that returns a non-gRPC response.



##########
sdks/go/test/integration/expansions.go:
##########
@@ -91,47 +98,90 @@ func (es *ExpansionServices) GetAddr(label string) (string, 
error) {
                return "", fmt.Errorf("expansion service labeled \"%s\" not 
found: %w", label, err)
        }
 
-       // Start jar on open port.
-       port, err := ports.GetOpenTCP()
-       if err != nil {
-               return "", fmt.Errorf("cannot get open port for expansion 
service labeled \"%s\": %w", label, err)
+       // A different service may claim the port between selection and jar 
startup.
+       const startupAttempts = 3
+       for attempt := 0; ; attempt++ {
+               port, err := ports.GetOpenTCP()
+               if err != nil {
+                       return "", fmt.Errorf("cannot get open port for 
expansion service labeled \"%s\": %w", label, err)
+               }
+               portStr := strconv.Itoa(port)
+               proc, err := es.run(*ExpansionTimeout, jar, portStr)
+               if err != nil {
+                       return "", fmt.Errorf("cannot run jar for expansion 
service labeled \"%s\": %w", label, err)
+               }
+               addr := "localhost:" + portStr
+               if es.testMode {
+                       time.Sleep(es.waitTime)
+               } else {
+                       exited := make(chan error, 1)
+                       go func() { exited <- proc.Wait() }()
+                       if err := waitForExpansionService(addr, exited); err != 
nil {
+                               proc.Kill()
+                               // Retry only startup collisions, never 
expansion or pipeline RPCs. A jar exits
+                               // when it cannot bind the port, and another 
service answers with Unimplemented.
+                               collision := status.Code(err) == 
codes.Unimplemented || errors.Is(err, errJarExited)
+                               if collision && attempt+1 < startupAttempts {
+                                       continue
+                               }
+                               return "", fmt.Errorf("expansion service 
labeled \"%s\" failed to start at %s: %w", label, addr, err)
+                       }
+               }
+               es.procs = append(es.procs, proc)
+               es.addrs[label] = addr
+               return addr, nil
        }
-       portStr := strconv.Itoa(port)
+}
 
-       // Run jar and cache its info.
-       proc, err := es.run(*ExpansionTimeout, jar, portStr)
+// errJarExited reports that the jar process stopped before the expansion 
service was ready.
+var errJarExited = errors.New("jar exited before becoming ready")
+
+func waitForExpansionService(addr string, exited <-chan error) error {
+       ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+       defer cancel()
+       conn, err := grpc.NewClient("passthrough:///"+addr,
+               grpc.WithTransportCredentials(insecure.NewCredentials()),
+               // The default reconnect backoff grows longer than the polling 
interval.
+               // Use a fixed delay so that the probe sees a late bind before 
the deadline.
+               grpc.WithConnectParams(grpc.ConnectParams{
+                       Backoff:           backoff.Config{BaseDelay: 250 * 
time.Millisecond, Multiplier: 1, MaxDelay: 250 * time.Millisecond},
+                       MinConnectTimeout: 5 * time.Second,
+               }))
        if err != nil {
-               return "", fmt.Errorf("cannot run jar for expansion service 
labeled \"%s\": %w", label, err)
+               return err
        }
-       
-       addr := "localhost:" + portStr
-
-       // Use different wait strategies for test mode vs production
-       if es.testMode {
-               // In test mode, use simple wait time for compatibility with 
mock processes
-               time.Sleep(es.waitTime)
-       } else {
-               // In production, wait for the jar to start with improved retry 
logic
-               maxRetries := 30
-               retryDelay := time.Second
-               
-               for i := 0; i < maxRetries; i++ {
-                       time.Sleep(retryDelay)
-                       // Try to connect to the expansion service to verify 
it's ready
-                       conn, err := net.DialTimeout("tcp", addr, 2*time.Second)
-                       if err == nil {
-                               conn.Close()
-                               break
-                       }
-                       if i == maxRetries-1 {
-                               return "", fmt.Errorf("expansion service 
labeled \"%s\" failed to start after %d retries: %w", label, maxRetries, err)
+       defer conn.Close()
+       client := jobpb.NewExpansionServiceClient(conn)
+       for {
+               probeCtx, probeCancel := context.WithTimeout(ctx, time.Second)
+               _, err := client.DiscoverSchemaTransform(probeCtx, 
&jobpb.DiscoverSchemaTransformRequest{}, grpc.WaitForReady(true))
+               probeCancel()
+               if status.Code(err) == codes.Unimplemented {
+                       // Older services only implement Expand. An empty 
request cannot expand
+                       // a transform, but an ExpansionResponse (including its 
error field)
+                       // proves that the service implements the expected RPC. 
Allow the
+                       // remaining startup deadline for first-request 
initialization.
+                       _, err = client.Expand(ctx, &jobpb.ExpansionRequest{}, 
grpc.WaitForReady(true))

Review Comment:
   Fixed in dadd9ba54a. The process waiter cancels the startup context as soon 
as the jar exits, interrupting discovery, Expand, and the wait between probes. 
A regression test exits the process while Expand is in flight and checks that 
startup promptly returns the exit error.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to