This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 65ef48888fa Reduce Python INFO logging verbosity at start-up (#27577)
65ef48888fa is described below
commit 65ef48888fa8ee5e4c61cf3eeaf5900f1e8be65b
Author: Jack McCluskey <[email protected]>
AuthorDate: Thu Jul 20 13:51:43 2023 -0400
Reduce Python INFO logging verbosity at start-up (#27577)
* Reduce Python INFO logging verbosity at start-up
* Formatting
---
sdks/python/apache_beam/runners/worker/sdk_worker_main.py | 4 ++--
sdks/python/container/boot.go | 10 +++++-----
sdks/python/container/piputil.go | 11 ++++++-----
3 files changed, 13 insertions(+), 12 deletions(-)
diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker_main.py
b/sdks/python/apache_beam/runners/worker/sdk_worker_main.py
index abbcfb72382..87cf06e862a 100644
--- a/sdks/python/apache_beam/runners/worker/sdk_worker_main.py
+++ b/sdks/python/apache_beam/runners/worker/sdk_worker_main.py
@@ -52,7 +52,7 @@ def _import_beam_plugins(plugins):
for plugin in plugins:
try:
importlib.import_module(plugin)
- _LOGGER.info('Imported beam-plugin %s', plugin)
+ _LOGGER.debug('Imported beam-plugin %s', plugin)
except ImportError:
try:
_LOGGER.debug((
@@ -61,7 +61,7 @@ def _import_beam_plugins(plugins):
plugin)
module, _ = plugin.rsplit('.', 1)
importlib.import_module(module)
- _LOGGER.info('Imported %s for beam-plugin %s', module, plugin)
+ _LOGGER.debug('Imported %s for beam-plugin %s', module, plugin)
except ImportError as exc:
_LOGGER.warning('Failed to import beam-plugin %s', plugin,
exc_info=exc)
diff --git a/sdks/python/container/boot.go b/sdks/python/container/boot.go
index 1e70e0db151..da2f3cc28f5 100644
--- a/sdks/python/container/boot.go
+++ b/sdks/python/container/boot.go
@@ -164,11 +164,11 @@ func launchSDKProcess() error {
if err != nil {
return errors.New(
"failed to create a virtual environment. If
running on Ubuntu systems, " +
- "you might need to install `python3-venv`
package. " +
- "To run the SDK process in default environment
instead, " +
- "set the environment variable
`RUN_PYTHON_SDK_IN_DEFAULT_ENVIRONMENT=1`. " +
- "In custom Docker images, you can do that with
an `ENV` statement. " +
- fmt.Sprintf("Encountered error: %v", err))
+ "you might need to install
`python3-venv` package. " +
+ "To run the SDK process in default
environment instead, " +
+ "set the environment variable
`RUN_PYTHON_SDK_IN_DEFAULT_ENVIRONMENT=1`. " +
+ "In custom Docker images, you can do
that with an `ENV` statement. " +
+ fmt.Sprintf("Encountered error: %v",
err))
}
cleanupFunc := func() {
os.RemoveAll(venvDir)
diff --git a/sdks/python/container/piputil.go b/sdks/python/container/piputil.go
index a00e017445e..c9e396b0e6a 100644
--- a/sdks/python/container/piputil.go
+++ b/sdks/python/container/piputil.go
@@ -37,14 +37,15 @@ func pipInstallRequirements(files []string, dir, name
string) error {
// as possible PyPI downloads. In the first round the
--find-links
// option will make sure that only things staged in the
worker will be
// used without following their dependencies.
- args := []string{"-m", "pip", "install", "-r",
filepath.Join(dir, name), "--no-cache-dir", "--disable-pip-version-check",
"--no-index", "--no-deps", "--find-links", dir}
+ args := []string{"-m", "pip", "install", "-r",
filepath.Join(dir, name), "--no-cache-dir", "--disable-pip-version-check",
"--no-index", "--no-deps", "--find-links",
+ "-q", dir}
if err := execx.Execute("python", args...); err != nil {
fmt.Println("Some packages could not be
installed solely from the requirements cache. Installing packages from PyPI.")
}
// The second install round opens up the search for
packages on PyPI and
// also installs dependencies. The key is that if all
the packages have
// been installed in the first round then this command
will be a no-op.
- args = []string{"-m", "pip", "install", "-r",
filepath.Join(dir, name), "--no-cache-dir", "--disable-pip-version-check",
"--find-links", dir}
+ args = []string{"-m", "pip", "install", "-r",
filepath.Join(dir, name), "--no-cache-dir", "--disable-pip-version-check",
"--find-links", "-q", dir}
return execx.Execute("python", args...)
}
}
@@ -76,18 +77,18 @@ func pipInstallPackage(files []string, dir, name string,
force, optional bool, e
// installed version will match the package
specified, the package itself
// will not be reinstalled, but its
dependencies will now be resolved and
// installed if necessary. This achieves our
goal outlined above.
- args := []string{"-m", "pip", "install",
"--no-cache-dir", "--disable-pip-version-check", "--upgrade",
"--force-reinstall", "--no-deps",
+ args := []string{"-m", "pip", "install",
"--no-cache-dir", "--disable-pip-version-check", "--upgrade",
"--force-reinstall", "--no-deps", "-q",
filepath.Join(dir, packageSpec)}
err := execx.Execute("python", args...)
if err != nil {
return err
}
- args = []string{"-m", "pip", "install",
"--no-cache-dir", "--disable-pip-version-check", filepath.Join(dir,
packageSpec)}
+ args = []string{"-m", "pip", "install",
"--no-cache-dir", "--disable-pip-version-check", "-q", filepath.Join(dir,
packageSpec)}
return execx.Execute("python", args...)
}
// Case when we do not perform a forced reinstall.
- args := []string{"-m", "pip", "install",
"--no-cache-dir", "--disable-pip-version-check", filepath.Join(dir,
packageSpec)}
+ args := []string{"-m", "pip", "install",
"--no-cache-dir", "--disable-pip-version-check", "-q", filepath.Join(dir,
packageSpec)}
return execx.Execute("python", args...)
}
}
