Nataneljpwd commented on code in PR #58827:
URL: https://github.com/apache/airflow/pull/58827#discussion_r2639248425
##########
go-sdk/pkg/bundles/shared/discovery.go:
##########
@@ -147,60 +150,66 @@ func (d *Discovery) DiscoverBundles(ctx context.Context)
error {
if err != nil {
self = ""
}
+ maxProcesses := runtime.GOMAXPROCS(0)
+ g, ctx := errgroup.WithContext(ctx)
+ g.SetLimit(maxProcesses)
+ var mu sync.Mutex
for _, file := range files {
- if ctx.Err() != nil {
- // Check if we are done.
- return ctx.Err()
- }
-
- // Check if file is executable
- if !isExecutable(file) {
- continue
- }
+ g.Go(func() error {
+ if ctx.Err() != nil {
+ // Check if we are done.
+ return ctx.Err()
+ }
- abs, err := filepath.Abs(file)
- if err != nil {
- d.logger.Warn("Unable to load resolve file path",
"file", file, "err", err)
- continue
- }
- if self != "" && self == abs {
- d.logger.Warn("Not trying to load ourselves as a
plugin", "file", file)
- continue
- }
+ // Check if file is executable
+ if !isExecutable(file) {
+ return nil
+ }
- d.logger.Debug("Found potential bundle", slog.String("path",
file))
+ abs, err := filepath.Abs(file)
+ if err != nil {
+ d.logger.Warn("Unable to load resolve file
path", "file", file, "err", err)
+ return nil
+ }
+ if self != "" && self == abs {
+ d.logger.Warn("Not trying to load ourselves as
a plugin", "file", file)
+ return nil
+ }
- // TODO: Use a sync.WaitGroup to parallelize running multiple
procs without blowing concurrency up and fork-bombing
- // the host
- bundle, err := d.getBundleVersionInfo(file)
- if err != nil {
- d.logger.Warn("Unable to load BundleMetadata", "file",
file, "err", err)
- continue
- }
+ d.logger.Debug("Found potential bundle",
slog.String("path", file))
- versions, exists := d.bundles[bundle.Name]
- if !exists {
- versions = make(map[string]string)
- d.bundles[bundle.Name] = versions
- }
+ bundle, err := d.getBundleVersionInfo(file)
+ if err != nil {
+ d.logger.Warn("Unable to load BundleMetadata",
"file", file, "err", err)
+ return nil
+ }
+ mu.Lock()
+ versions, exists := d.bundles[bundle.Name]
+ if !exists {
+ versions = make(map[string]string)
+ d.bundles[bundle.Name] = versions
+ }
- var key string
- logAs := bundle.Name
- if bundle.Version != nil {
- key = *bundle.Version
- logAs = fmt.Sprintf("%s@%s", bundle.Name, key)
- }
- d.logger.Info(
- "Discovered bundle",
- "key",
- logAs,
- "file",
- file,
- )
- versions[key] = file
+ var key string
+ logAs := bundle.Name
+ if bundle.Version != nil {
+ key = *bundle.Version
+ logAs = fmt.Sprintf("%s@%s", bundle.Name, key)
+ }
+ d.logger.Info(
+ "Discovered bundle",
+ "key",
+ logAs,
+ "file",
+ file,
+ )
+ versions[key] = file
+ mu.Unlock()
+ return nil
+ })
}
- return nil
+ return g.Wait()
Review Comment:
Understood, thank you
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
