From: pshem <[email protected]>
Committer: GitHub <[email protected]>
Branch: master
Handle dependencies automatically
- optimize execution speed by only launching python3 once,
- prevent `rsync` from synching the same files more than twice(although
lines 23-27 could potentially be removed for a more narrow python
distribution, removing the double `rsync` problem)
- avoid dropping submodules into the root directory(in the previous
iteration, `mysql.connector` would work(put in the right place when mysql
was synched), but an extra `connector.py` file would be present in the root
directory of python modules),
- include dependencies based on imports tracked by `sys.modules`
---
diff --git a/python3x/GET b/python3x/GET
--- a/python3x/GET
+++ b/python3x/GET
@@ -8,7 +8,7 @@ PYTHON_PREFIX_DIR=`python3 -c 'import sys;
print(sys.prefix)'`
PYTHON_MAJOR_VERSION=`python3 -c 'import sys;
print(sys.version_info.major)'`
PYTHON_MINOR_VERSION=`python3 -c 'import sys;
print(sys.version_info.minor)'`
PYTHON_VERSION="${PYTHON_MAJOR_VERSION}.${PYTHON_MINOR_VERSION}"
-PYTHON_MODULES="" #put additional modules here. Include their
dependencies. Example: "django mysql mysql.connector"
+PYTHON_MODULES="" #put additional modules here. Example: "django mysql
mysql.connector"
install_shlibs() {
@@ -27,6 +27,45 @@ ldd $SHLIBS | grep -Po '(?<=> )/[^ ]+' | sort | uniq |
grep -Pv 'lib(c|gcc|dl|m|
echo "$SHLIBS_COUNT"
}
+get_dependencies() {
+python3 - <<'EOF' $1
+import sys
+
+#get a dict of modules imported at the start
+default_modules = sys.modules.copy()
+
+#import all additional modules
+for mod in sys.argv[1].split(" "):
+ exec('import ' + mod)
+
+unique_paths = set()
+
+def path_exploration(l_modules):
+ for mo in l_modules:
+ exec('import ' + mo)
+#for submodules(x.y), copy the top level module(x)
+ l_mod = mo.split('.')[0]
+
+#real modules have file. Some modules have __path__, and their file is
__init__.py
+ try:
+ if str(eval(l_mod + '.__file__').split('/')[-1]) != '__init__.py':
+ if eval(l_mod + '.__file__')[-3:] not in ['.so', 'in>']:
#skip .so and <stdin>
+ unique_paths.add(eval(l_mod + '.__file__') + ' ')
+ else:
+ unique_paths.add(str(eval(l_mod + '.__path__[0]')) + '/ ')
+ except NameError: #module not imported. Try to import
+ path_exploration(set(l_mod))
+ except AttributeError:
+ pass #built-in modules don't have __file__
+
+modules = set(sys.modules.keys()).difference(set(default_modules.keys()))
+path_exploration(modules)
+
+print(" ".join(unique_paths))
+
+EOF
+}
+
main() {
mkdir -p build/
gcc -o build/python.so python.c -fPIC -shared -lpython${PYTHON_VERSION}m
@@ -43,10 +82,9 @@ do
--exclude '*.pyc' --exclude '*.pyo' --exclude '*.egg-info'
--exclude 'site-packages' --exclude 'dist-packages'
done
-for i in ${PYTHON_MODULES}
+for i in $(get_dependencies "${PYTHON_MODULES}")
do
- rsync -a "$(python3 -c "import sys; module = sys.argv[1];
exec('import ' + module); print(eval(module + '.__file__')) if
str(eval(module + '.__file__').split('/')[-1]) != '__init__.py' else
print(eval(module + '.__path__[0]'))" $i)" \
- $ROOTFS/lib/python${PYTHON_VERSION}/ --safe-links --exclude test
--exclude unittest \
+ rsync -a "$i" $ROOTFS/lib/python${PYTHON_VERSION}/ --safe-links \
--exclude '*.pyc' --exclude '*.pyo' --exclude '*.egg-info'
done
--
You received this message because you are subscribed to the Google Groups "OSv
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/d/optout.
