BryanDavis has uploaded a new change for review.
https://gerrit.wikimedia.org/r/130560
Change subject: Precompute data for GitInfo
......................................................................
Precompute data for GitInfo
Create JSON cache files of git version information during a full scap.
These files will be read by GitInfo.php once I66e058a is merged to
provide version information for mw-core and extensions.
Bug: 53972
Change-Id: I16cf49edaa11bfaeb82be987df22badc44109966
---
M scap/main.py
M scap/tasks.py
M scap/utils.py
3 files changed, 126 insertions(+), 5 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/tools/scap
refs/changes/60/130560/1
diff --git a/scap/main.py b/scap/main.py
index 1bb7ada..082b5be 100644
--- a/scap/main.py
+++ b/scap/main.py
@@ -140,11 +140,12 @@
2. Sync deploy directory on localhost with staging area
3. Compile wikiversions.json to cdb in deploy directory
4. Update l10n files in staging area
- 5. Ask scap proxies to sync with master server
- 6. Ask apaches to sync with fastest rsync server
- 7. Ask apaches to rebuild l10n CDB files
- 8. Update wikiversions.cdb on localhost
- 9. Ask apaches to sync wikiversions.cdb
+ 5. Compute git version information
+ 6. Ask scap proxies to sync with master server
+ 7. Ask apaches to sync with fastest rsync server
+ 8. Ask apaches to rebuild l10n CDB files
+ 9. Update wikiversions.cdb on localhost
+ 10. Ask apaches to sync wikiversions.cdb
"""
self._assert_auth_sock()
@@ -172,6 +173,11 @@
tasks.update_localization_cache(
version, wikidb, self.arguments.verbose, self.config)
+ # Compute git version information
+ with log.Timer('cache_git_info', self.stats):
+ for version, wikidb in self.active_wikiversions().items():
+ tasks.cache_git_info(version, self.config)
+
# Update rsync proxies
scap_proxies = utils.read_dsh_hosts_file('scap-proxies')
with log.Timer('sync-common to proxies', self.stats):
diff --git a/scap/tasks.py b/scap/tasks.py
index f2c6052..f54badc 100644
--- a/scap/tasks.py
+++ b/scap/tasks.py
@@ -36,6 +36,40 @@
'--no-perms')
+def cache_git_info(version, cfg):
+ """Create JSON cache files of git branch information.
+
+ :param version: MediaWiki version (eg '1.23wmf15')
+ :param cfg: Dict of global configuration values
+ :raises: :class:`IOError` if version directory is not found
+ """
+ branch_dir = os.path.join(cfg['stage_dir'], 'php-%s' % version)
+
+ if not os.path.isdir(branch_dir):
+ raise IOError(errno.ENOENT, 'Invalid branch directory', branch_dir)
+
+ # Create cache directory if needed
+ cache_dir = os.path.join(branch_dir, 'cache', 'gitinfo')
+ if not os.path.isdir(cache_dir):
+ os.mkdir(cache_dir)
+
+ # Create cache for branch
+ info = utils.git_info(branch_dir)
+ cache_file = utils.git_info_filename(branch_dir, branch_dir, cache_dir)
+ with open(cache_file, 'w') as f:
+ json.dump(info, f)
+
+ # Create cache for each extension
+ extensions_dir = os.path.join(branch_dir, 'extensions')
+ for subdir in utils.iterate_subdirectories(extensions_dir):
+ if os.path.exists(os.path.join(subdir, '.git')):
+ info = utils.git_info(subdir)
+ cache_file = utils.git_info_filename(
+ subdir, branch_dir, cache_dir)
+ with open(cache_file, 'w') as f:
+ json.dump(info, f)
+
+
def check_php_syntax(*paths):
"""Run lint.php on `paths`; raise CalledProcessError if nonzero exit."""
cmd = '/usr/bin/php -n -dextension=parsekit.so /usr/local/bin/lint.php'
diff --git a/scap/utils.py b/scap/utils.py
index 52e759d..d171938 100644
--- a/scap/utils.py
+++ b/scap/utils.py
@@ -207,3 +207,84 @@
if proc.returncode:
raise subprocess.CalledProcessError(proc.returncode, cmd)
+
+
+def iterate_subdirectories(root):
+ for name in os.listdir(root):
+ subdir = os.path.join(root, name)
+ if os.path.isdir(subdir):
+ yield subdir
+
+
+def git_info_filename(directory, install_path, cache_path):
+ """Compute the path for a git_info cache file related to a given
+ directory."""
+ path = directory
+ if path.startswith(install_path):
+ path = path[len(install_path):]
+ return os.path.join(cache_path, 'info%s.json' % path.replace('/', '-'))
+
+
+def git_info(directory):
+ """Compute git version information for a given directory that is
+ compatible with MediaWiki's GitInfo class.
+
+ :param directory: Directory to scan for git information
+ :returns: Dict of information about current repository state
+ """
+ git_dir = os.path.join(directory, '.git')
+ if not os.path.exists(git_dir):
+ raise IOError(errno.ENOENT, '.git not found', directory)
+
+ if os.path.isfile(git_dir):
+ # submodules
+ with open(git_dir, 'r') as f:
+ git_ref = f.read().strip()
+
+ if not git_ref.startswith('gitdir: '):
+ raise IOError(errno.EINVAL, 'Unexpected .git contents', git_dir)
+ git_ref = git_ref[8:]
+ if git_ref[0] != '/':
+ git_ref = os.path.abspath(os.path.join(directory, git_ref))
+ git_dir = git_ref
+
+ head_file = os.path.join(git_dir, 'HEAD')
+ with open(head_file, 'r') as f:
+ head = f.read().strip()
+ if head.startswith('ref: '):
+ head = head[5:]
+
+ if re.match(r'^[0-9a-f]{40}$', head, re.IGNORECASE):
+ # Working copy is a detached head, so we can't check for upstream
+ # intersection easily.
+ head_sha1 = head
+ else:
+ # Find first commit shared with the upstream branch. This keeps us
+ # from leaking information about locally committed changes such as
+ # security patches.
+ head_sha1 = subprocess.check_output(
+ ('/usr/bin/git', 'rev-list', '-1', '@{upstream}'),
+ cwd=git_dir).strip()
+
+ commit_date = subprocess.check_output(
+ ('/usr/bin/git', 'show', '-s', '--format=%ct', head_sha1),
+ cwd=git_dir).strip()
+
+ if head.startswith('refs/heads/'):
+ branch = head[11:]
+ else:
+ branch = head
+
+ # Requires git v1.7.5+
+ remote_url = subprocess.check_output(
+ ('/usr/bin/git', 'ls-remote', '--get-url'),
+ cwd=git_dir).strip()
+
+ return {
+ '@directory': directory,
+ 'head': head,
+ 'headSHA1': head_sha1,
+ 'headCommitDate': commit_date,
+ 'branch': head,
+ 'remoteURL': remote_url,
+ }
--
To view, visit https://gerrit.wikimedia.org/r/130560
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I16cf49edaa11bfaeb82be987df22badc44109966
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/tools/scap
Gerrit-Branch: master
Gerrit-Owner: BryanDavis <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
