This is an automated email from the ASF dual-hosted git repository.
shunping pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 7052f01e45a Add an experiment to disable pip build isolation (#37331)
7052f01e45a is described below
commit 7052f01e45a55507f5e5a9271296d2fd1da89c15
Author: Shunping Huang <[email protected]>
AuthorDate: Mon Feb 2 16:44:29 2026 -0500
Add an experiment to disable pip build isolation (#37331)
* Disable build isolation for workflow tarball by default. Add an env var
to enable it if needed.
* Apply build isolation setting to all pip install commands. Use experiment
to control.
* Only search experiment in experiment options.
---
sdks/python/container/boot.go | 39 +++++++++++++++++++++++++++++++++++++++
sdks/python/container/piputil.go | 14 ++++++++++++++
2 files changed, 53 insertions(+)
diff --git a/sdks/python/container/boot.go b/sdks/python/container/boot.go
index 847325d4f83..572dbf01113 100644
--- a/sdks/python/container/boot.go
+++ b/sdks/python/container/boot.go
@@ -30,6 +30,7 @@ import (
"os/signal"
"path/filepath"
"regexp"
+ "slices"
"strings"
"sync"
"syscall"
@@ -116,6 +117,37 @@ func main() {
}
}
+// The json string of pipeline options is in the following format.
+// We only focus on experiments here.
+//
+// {
+// "display_data": [
+// {...},
+// ],
+// "options": {
+// ...
+// "experiments": [
+// ...
+// ],
+// }
+// }
+type PipelineOptionsData struct {
+ Options OptionsData `json:"options"`
+}
+
+type OptionsData struct {
+ Experiments []string `json:"experiments"`
+}
+
+func getExperiments(options string) []string {
+ var opts PipelineOptionsData
+ err := json.Unmarshal([]byte(options), &opts)
+ if err != nil {
+ return nil
+ }
+ return opts.Options.Experiments
+}
+
func launchSDKProcess() error {
ctx := grpcx.WriteWorkerID(context.Background(), *id)
@@ -155,6 +187,13 @@ func launchSDKProcess() error {
logger.Fatalf(ctx, "Failed to convert pipeline options: %v",
err)
}
+ experiments := getExperiments(options)
+ pipNoBuildIsolation = false
+ if slices.Contains(experiments, "pip_no_build_isolation") {
+ pipNoBuildIsolation = true
+ logger.Printf(ctx, "Disabled build isolation when installing
packages with pip")
+ }
+
// (2) Retrieve and install the staged packages.
//
// No log.Fatalf() from here on, otherwise deferred cleanups will not
be called!
diff --git a/sdks/python/container/piputil.go b/sdks/python/container/piputil.go
index d6250ad2fdc..1faf8421a02 100644
--- a/sdks/python/container/piputil.go
+++ b/sdks/python/container/piputil.go
@@ -32,6 +32,11 @@ import (
"github.com/apache/beam/sdks/v2/go/pkg/beam/util/execx"
)
+var (
+ // Whether to append "--no-build-isolation" flag to pip install command
+ pipNoBuildIsolation bool
+)
+
const pipLogFlushInterval time.Duration = 15 * time.Second
const unrecoverableURL string =
"https://beam.apache.org/documentation/sdks/python-unrecoverable-errors/index.html#pip-dependency-resolution-failures";
@@ -112,6 +117,9 @@ func pipInstallPackage(ctx context.Context, logger
*tools.Logger, files []string
// installed if necessary. This achieves our
goal outlined above.
args := []string{"-m", "pip", "install",
"--no-cache-dir", "--disable-pip-version-check", "--upgrade",
"--force-reinstall", "--no-deps",
filepath.Join(dir, packageSpec)}
+ if pipNoBuildIsolation {
+ args = append(args,
"--no-build-isolation")
+ }
err := execx.ExecuteEnvWithIO(nil, os.Stdin,
bufLogger, bufLogger, pythonVersion, args...)
if err != nil {
bufLogger.FlushAtError(ctx)
@@ -120,6 +128,9 @@ func pipInstallPackage(ctx context.Context, logger
*tools.Logger, files []string
bufLogger.FlushAtDebug(ctx)
}
args = []string{"-m", "pip", "install",
"--no-cache-dir", "--disable-pip-version-check", filepath.Join(dir,
packageSpec)}
+ if pipNoBuildIsolation {
+ args = append(args,
"--no-build-isolation")
+ }
err = execx.ExecuteEnvWithIO(nil, os.Stdin,
bufLogger, bufLogger, pythonVersion, args...)
if err != nil {
bufLogger.FlushAtError(ctx)
@@ -131,6 +142,9 @@ func pipInstallPackage(ctx context.Context, logger
*tools.Logger, files []string
// Case when we do not perform a forced reinstall.
args := []string{"-m", "pip", "install",
"--no-cache-dir", "--disable-pip-version-check", filepath.Join(dir,
packageSpec)}
+ if pipNoBuildIsolation {
+ args = append(args, "--no-build-isolation")
+ }
err := execx.ExecuteEnvWithIO(nil, os.Stdin, bufLogger,
bufLogger, pythonVersion, args...)
if err != nil {
bufLogger.FlushAtError(ctx)
