Changes:
bundle.py
Added zip-agnostic _is_file() method to bundle class; used in activitybundle
for checking MANIFEST without unpacking
activitybundle.py
Moved basic logic for processing MANIFEST, including list_files, to
activitybundle.py ; this is necessary for checking when installing a bundle.
moved dependencies on activity registry and env to inline imports in
relevant methods; some bundle tasks should be doable with just these three
files. (This is doing only half of the job, the other half would be falling
back to importing the other modules not as submodules).
some simple globbing in list_files for ignored files, to allow ignoring
*.pyc.
reads manifest into a list of lines; then replaces invalid lines in place
with blank lines. This allows fix_manifest in bundlebuilder to leave all
existing files on same MANIFEST line number, which will help with diffs
later.
separate unpack from install (= unpack + register). Unpack checks for valid
MANIFEST but is set to only complain, not throw exceptions, by default, to
give folks time to clean up.
bundlebuilder.py:
fix_manifest function rewrites MANIFEST file.
SourcePackager.get_files calls out to git to get file names, or falls back
to MANIFEST otherwise.
On Fri, Jun 6, 2008 at 1:24 PM, Marco Pesenti Gritti <[EMAIL PROTECTED]>
wrote:
> On Fri, Jun 6, 2008 at 6:13 PM, Jameson Chema Quinn
> <[EMAIL PROTECTED]> wrote:
> > Note: this patch does not expose the fix_manifest function when running
> > bundlebuilder as a script. That would be a 2 or 3 line patch, seperately.
>
> Some initial notes:
>
> * Please run pylint on the changed files. You can use
> sugar-jhbuild/scripts/data/pylintrc. That will catch several problems
> and nitpicks that I'd ask you to fix anyway.
> * I dont understand all the
> warnings.warn(MalformedBundleException(...)). What's the purpose of
> those. Shouldn't they be real exceptions? What's the advantage of
> using the warnings module over logging.warning?
> * I'm not sure what you are using dest_dir for, but it looks like it
> should be part of Config.
> * Why are you moving the activity and env imports inline.
>
> As we discussed in irc, it would be good to split up patches. But
> anyway in the case of big patches like this, containing unrelated
> changes is good to provide a list of the changes you made and the
> reason you made them (can you provide such list when you resubmit the
> patch with pylint fixed? Thanks!).
>
> Marco
> _______________________________________________
> Sugar mailing list
> Sugar@lists.laptop.org
> http://lists.laptop.org/listinfo/sugar
>
diff --git a/src/sugar/activity/bundlebuilder.py b/src/sugar/activity/bundlebuilder.py
index 1063f72..3c8f7dc 100644
--- a/src/sugar/activity/bundlebuilder.py
+++ b/src/sugar/activity/bundlebuilder.py
@@ -19,46 +19,36 @@ import os
import zipfile
import tarfile
import shutil
-import subprocess
+from subprocess import Popen, PIPE
import re
import gettext
from optparse import OptionParser
+import warnings
+import subprocess
from sugar import env
-from sugar.bundle.activitybundle import ActivityBundle
-
-def list_files(base_dir, ignore_dirs=None, ignore_files=None):
- result = []
+from sugar.bundle.activitybundle import ActivityBundle, MANIFEST, list_files
- for root, dirs, files in os.walk(base_dir):
- for f in files:
- if ignore_files and f not in ignore_files:
- rel_path = root[len(base_dir) + 1:]
- result.append(os.path.join(rel_path, f))
- if ignore_dirs and root == base_dir:
- for ignore in ignore_dirs:
- if ignore in dirs:
- dirs.remove(ignore)
-
- return result
class Config(object):
- def __init__(self, bundle_name, source_dir=None):
- if source_dir:
- self.source_dir = source_dir
- else:
- self.source_dir = os.getcwd()
+ def __init__(self, bundle_name, source_dir=None, dist_dir = None,
+ dist_name = None):
+ self.source_dir = source_dir or os.getcwd()
+ self.dist_dir = dist_dir or os.path.join(self.source_dir, 'dist')
bundle = ActivityBundle(self.source_dir)
version = bundle.get_activity_version()
+ self.bundle = bundle
self.bundle_name = bundle_name
- self.xo_name = '%s-%d.xo' % (self.bundle_name, version)
- self.tarball_name = '%s-%d.tar.bz2' % (self.bundle_name, version)
+ self.xo_name = dist_name or '%s-%d.xo' % (self.bundle_name, version)
+ self.tarball_name = (dist_name or
+ '%s-%d.tar.bz2' % (self.bundle_name, version))
self.bundle_id = bundle.get_bundle_id()
self.bundle_root_dir = self.bundle_name + '.activity'
self.tarball_root_dir = '%s-%d' % (self.bundle_name, version)
+
info_path = os.path.join(self.source_dir, 'activity', 'activity.info')
f = open(info_path,'r')
@@ -105,48 +95,65 @@ class Builder(object):
class Packager(object):
def __init__(self, config):
self.config = config
- self.dist_dir = os.path.join(self.config.source_dir, 'dist')
self.package_path = None
- if not os.path.exists(self.dist_dir):
- os.mkdir(self.dist_dir)
+ if not os.path.exists(self.config.dist_dir):
+ os.mkdir(self.config.dist_dir)
class BuildPackager(Packager):
- def __init__(self, config):
- Packager.__init__(self, config)
- self.build_dir = self.config.source_dir
-
def get_files(self):
- return list_files(self.build_dir,
- ignore_dirs=['po', 'dist', '.git'],
- ignore_files=['.gitignore'])
+ return self.config.bundle.get_files()
+
+ def fix_manifest(self):
+ allfiles = list_files(self.config.source_dir,
+ ignore_dirs=['dist', '.git'],
+ ignore_files=['.gitignore', 'MANIFEST',
+ '*.pyc', '*~', '*.bak'])
+ for afile in allfiles:
+ if afile not in self.config.bundle.manifest:
+ self.config.bundle.manifest.append(afile)
+ manifestfile = open(os.path.join(self.config.source_dir,
+ MANIFEST),
+ "wb")
+ for line in self.config.bundle.manifest:
+ manifestfile.write(line + "\n")
class XOPackager(BuildPackager):
def __init__(self, config):
BuildPackager.__init__(self, config)
- self.package_path = os.path.join(self.dist_dir, self.config.xo_name)
+ self.package_path = os.path.join(self.config.dist_dir,
+ self.config.xo_name)
def package(self):
bundle_zip = zipfile.ZipFile(self.package_path, 'w',
zipfile.ZIP_DEFLATED)
for f in self.get_files():
- bundle_zip.write(os.path.join(self.build_dir, f),
+ bundle_zip.write(os.path.join(self.config.build_dir, f),
os.path.join(self.config.bundle_root_dir, f))
bundle_zip.close()
-class SourcePackager(Packager):
+class SourcePackager(BuildPackager):
def __init__(self, config):
- Packager.__init__(self, config)
- self.package_path = os.path.join(self.dist_dir,
+ BuildPackager.__init__(self, config)
+ self.package_path = os.path.join(self.config.dist_dir,
self.config.tarball_name)
def get_files(self):
- return list_files(self.config.source_dir,
- ignore_dirs=['locale', 'dist', '.git'],
- ignore_files=['.gitignore'])
+ git_ls = Popen('git-ls-files', stdout=PIPE, cwd=self.config.source_dir)
+ if git_ls.wait():
+ #non-0 return code - failed
+ return BuildPackager.get_files(self)
+ f = git_ls.stdout
+ files = []
+ for line in f.readlines():
+ filename = line.strip()
+ if not filename.startswith('.'):
+ files.append(filename)
+ f.close()
+ return files
def package(self):
tar = tarfile.open(self.package_path, "w")
diff --git a/src/sugar/bundle/activitybundle.py b/src/sugar/bundle/activitybundle.py
index db30555..4011de9 100644
--- a/src/sugar/bundle/activitybundle.py
+++ b/src/sugar/bundle/activitybundle.py
@@ -21,16 +21,32 @@ from ConfigParser import ConfigParser
import locale
import os
import tempfile
+from fnmatch import fnmatch
from sugar.bundle.bundle import Bundle, MalformedBundleException, \
AlreadyInstalledException, RegistrationException, \
NotInstalledException
-from sugar import activity
-from sugar import env
-
import logging
+MANIFEST = "MANIFEST"
+
+def list_files(base_dir, ignore_dirs=None, ignore_files=None):
+ result = []
+
+ for root, dirs, files in os.walk(base_dir):
+ for f in files:
+ if not (ignore_files and
+ [True for pat in ignore_files if fnmatch(f,pat)]):
+ #result matches a pattern in ignore_files, ignore it
+ rel_path = root[len(base_dir) + 1:]
+ result.append(os.path.join(rel_path, f))
+ if ignore_dirs and root == base_dir:
+ for ignore in ignore_dirs:
+ if ignore in dirs:
+ dirs.remove(ignore)
+ return result
+
class ActivityBundle(Bundle):
"""A Sugar activity bundle
@@ -64,7 +80,70 @@ class ActivityBundle(Bundle):
linfo_file = self._get_linfo_file()
if linfo_file:
self._parse_linfo(linfo_file)
-
+
+ self.read_manifest()
+
+ def _raw_manifest(self):
+ try:
+ f = self._get_file(MANIFEST)
+ except IOError:
+ f = None
+ if not f:
+ logging.warning("Activity directory lacks a MANIFEST file.")
+ return []
+
+ #strip trailing \n and other whitespace
+ ret = [line.strip() for line in f.readlines()]
+ f.close()
+ return ret
+
+ def read_manifest(self):
+ """read_manifest: sets self.manifest to list of lines in MANIFEST,
+ with invalid lines replaced by empty lines.
+
+ Since absolute order carries information on file history, it should
+ be preserved.
+ For instance, when renaming a file, you should leave the new name
+ on the same line as the old one.
+ """
+ manifestlines = self._raw_manifest()
+ for num, line in enumerate(manifestlines):
+ if line:
+ if line in manifestlines[0:num]:
+ manifestlines[num] = ""
+ logging.warning("Bundle %s: duplicate entry in MANIFEST: %s"
+ %(self._name,line))
+ continue
+ if line == MANIFEST:
+ manifestlines[num] = ""
+ logging.warning("Bundle %s: MANIFEST includes itself: %s"
+ %(self._name,line))
+ if line.endswith("/"):
+ if not self._is_dir(line):
+ manifestlines[num] = ""
+ logging.warning("Bundle %s: invalid dir "
+ "entry in MANIFEST: %s"
+ %(self._name,line))
+ else:
+ if not self._is_file(line):
+ manifestlines[num] = ""
+ logging.warning("Bundle %s: "
+ "invalid entry in MANIFEST: %s"
+ %(self._name,line))
+ #remove trailing newlines - unlike internal newlines,
+ # they do not help keep absolute position
+ while manifestlines and manifestlines[-1] == "":
+ manifestlines = manifestlines[:-1]
+ self.manifest = manifestlines
+
+ def get_files(self, manifest = None):
+ return [MANIFEST] + [line for line in (manifest or self.manifest)
+ if line and not line.endswith("/")]
+
+ def get_dirs(self):
+ return [line for line in self.manifest
+ if line.endswith("/")]
+
def _parse_info(self, info_file):
cp = ConfigParser()
cp.readfp(info_file)
@@ -116,6 +195,7 @@ class ActivityBundle(Bundle):
raise MalformedBundleException(
'Activity bundle %s has invalid version number %s' %
(self._path, version))
+
def _get_linfo_file(self):
lang = locale.getdefaultlocale()[0]
@@ -209,28 +289,53 @@ class ActivityBundle(Bundle):
return self._show_launcher
def is_installed(self):
+ from sugar import activity
+
if activity.get_registry().get_activity(self._bundle_id):
return True
else:
return False
def need_upgrade(self):
+ from sugar import activity
+
act = activity.get_registry().get_activity(self._bundle_id)
if act is None or act.version != self._activity_version:
return True
else:
return False
-
- def install(self):
- activities_path = env.get_user_activities_path()
- act = activity.get_registry().get_activity(self._bundle_id)
- if act is not None and act.path.startswith(activities_path):
- raise AlreadyInstalledException
-
- install_dir = env.get_user_activities_path()
+
+ def unpack(self, install_dir, strict_manifest=False):
self._unzip(install_dir)
install_path = os.path.join(install_dir, self._zip_root_dir)
+
+ #check installed files against the MANIFEST
+ manifestfiles = self.get_files(self._raw_manifest())
+ for afile in list_files(install_path):
+ if afile in manifestfiles:
+ manifestfiles.remove(file)
+ elif afile != MANIFEST:
+ logging.warning("Bundle %s: %s not in MANIFEST"%
+ (self._name,file))
+ if strict_manifest:
+ os.remove(os.path.join(install_path, file))
+ if manifestfiles:
+ err = ("Bundle %s: files in MANIFEST not included: %s"%
+ (self._name,str(manifestfiles)))
+ if strict_manifest:
+ raise MalformedBundleException(err)
+ else:
+ logging.warning(err)
+
+ #create empty directories
+ for adir in self.get_dirs():
+ dirpath = os.path.join(install_path, adir)
+ if os.path.isdir(dirpath):
+ logging.warning("Bunldle %s: non-empty dir %s in MANIFEST"%
+ (self._name,dirpath))
+ else:
+ os.makedirs(dirpath)
xdg_data_home = os.getenv('XDG_DATA_HOME',
os.path.expanduser('~/.local/share'))
@@ -267,11 +372,27 @@ class ActivityBundle(Bundle):
os.symlink(info_file,
os.path.join(installed_icons_dir,
os.path.basename(info_file)))
+ return install_path
+
+ def install(self):
+ from sugar import activity
+ from sugar import env
+
+ activities_path = env.get_user_activities_path()
+ act = activity.get_registry().get_activity(self._bundle_id)
+ if act is not None and act.path.startswith(activities_path):
+ raise AlreadyInstalledException
+ install_dir = env.get_user_activities_path()
+ install_path = self.unpack(install_dir)
+
if not activity.get_registry().add_bundle(install_path):
raise RegistrationException
def uninstall(self, force=False):
+ from sugar import activity
+ from sugar import env
+
if self._unpacked:
install_path = self._path
else:
@@ -316,6 +437,9 @@ class ActivityBundle(Bundle):
raise RegistrationException
def upgrade(self):
+ from sugar import activity
+ from sugar import env
+
act = activity.get_registry().get_activity(self._bundle_id)
if act is None:
logging.warning('Activity not installed')
diff --git a/src/sugar/bundle/bundle.py b/src/sugar/bundle/bundle.py
index 47d08b2..95756d4 100644
--- a/src/sugar/bundle/bundle.py
+++ b/src/sugar/bundle/bundle.py
@@ -114,6 +114,30 @@ class Bundle:
zip_file.close()
return f
+
+ def _is_dir(self, filename):
+ if self._unpacked:
+ path = os.path.join(self._path, filename)
+ return os.path.isdir(path)
+ else:
+ return True #zip files contain all dirs you care about!
+
+ def _is_file(self, filename):
+ if self._unpacked:
+ path = os.path.join(self._path, filename)
+ return os.path.isfile(path)
+ else:
+ zip_file = zipfile.ZipFile(self._path)
+ path = os.path.join(self._zip_root_dir, filename)
+ try:
+ zip_file.getinfo(path)
+ #no exception above, so:
+ return True
+ except KeyError:
+ return False
+ finally:
+ zip_file.close()
+
def get_path(self):
"""Get the bundle path."""
_______________________________________________
Sugar mailing list
Sugar@lists.laptop.org
http://lists.laptop.org/listinfo/sugar
