jenkins-bot has submitted this change and it was merged.
Change subject: Allow rendering from unzipped bundle
......................................................................
Allow rendering from unzipped bundle
Allow the renderer to consume an unzipped bundle file. This saves CPU
cycles when the bundle was created on the same machine. Due to the
use of hardlinks, the bundle directory in this case must be on the
same file system as the temp dir -- use the system environment
variable TMPDIR to specify an appropriate directory if necessary.
Change-Id: Iad77b766a12654876a7c140220df9a37dc25f6c6
---
M bin/mw-ocg-latexer
M lib/index.js
2 files changed, 76 insertions(+), 16 deletions(-)
Approvals:
Cscott: Looks good to me, approved
Mwalker: Looks good to me, approved
jenkins-bot: Verified
diff --git a/bin/mw-ocg-latexer b/bin/mw-ocg-latexer
index 7e57a1c..b8935b0 100755
--- a/bin/mw-ocg-latexer
+++ b/bin/mw-ocg-latexer
@@ -13,7 +13,7 @@
program
.version(latexer.version)
- .usage('[options] <bundle.zip>')
+ .usage('[options] <bundle_dir or bundle.zip>')
.option('-o, --output <filename>',
'Save PDF to the given <filename>', null)
.option('-s, --size <letter|a4>',
@@ -36,7 +36,7 @@
program.parse(process.argv);
if (program.args.length === 0) {
- console.error('A bundle filename is required.');
+ console.error('A bundle filename or directory is required.');
return 1;
}
if (program.args.length > 1) {
diff --git a/lib/index.js b/lib/index.js
index 9e3bcc2..7351d87 100644
--- a/lib/index.js
+++ b/lib/index.js
@@ -1119,10 +1119,29 @@
// We also use `P`, a set of helpers for promises that make it easier
// to work with methods which accept node-style callbacks.
-// Step 1: unpack a bundle, and return a promise for the builddir
+// Helper: hard link a directory, recursively.
+var cprl = function(from, to) {
+ return P.call(fs.mkdir, fs, to).then(function() {
+ return P.call(fs.readdir, fs, from);
+ }).map(function(file) {
+ var pathfrom = path.join(from, file);
+ var pathto = path.join(to, file);
+ return P.call(fs.lstat, fs, pathfrom).then(function(stats) {
+ if (stats.isFile()) {
+ return P.call(fs.link, fs, pathfrom, pathto);
+ }
+ if (stats.isDirectory()) {
+ return cprl(pathfrom, pathto);
+ }
+ // ignore other file types (symlink, block device, etc)
+ });
+ });
+};
+
+// Step 1a: unpack a bundle, and return a promise for the builddir
// and control file contents.
var unpackBundle = function(options) {
- var metabook, builddir, status = options.status;
+ var builddir, status = options.status;
status.createStage(0, 'Unpacking content bundle');
@@ -1144,14 +1163,42 @@
cwd: bundledir
});
}).then(function() {
- // now read in the main metabook.json file
- return P.call(
- fs.readFile, fs, path.join(builddir, 'bundle',
'metabook.json')
- ).then(function(data) {
- metabook = JSON.parse(data);
- });
+ return builddir;
+ });
+};
+
+// Step 1b: we were given a bundle directory. Create a tmpdir and then
+// hard link the bundle directory into it. Be sure your TMPDIR is
+// on the same filesystem as the provided bundle directory if you
+// want this to be fast.
+var hardlinkBundle = function(options) {
+ var builddir, status = options.status;
+
+ status.createStage(0, 'Creating work space');
+ // first create a temporary directory
+ return P.call(tmp.dir, tmp, {
+ prefix: json.name,
+ unsafeCleanup: !(options.debug || options.latex)
+ }).then(function(_builddir) {
+ builddir = _builddir;
+ // make latex subdir
+ return Promise.join(
+ // make latex subdir
+ P.call(fs.mkdir, fs, path.join(builddir, 'latex')),
+ // hardlink bundledir into 'bundle'
+ cprl(path.resolve( options.bundle ),
path.join(builddir, 'bundle')).
+ catch(function(e) {
+ // slightly helpful diagnostics
+ if (e.code === 'EXDEV') {
+ throw new Error(
+ "TMPDIR must be on same
filesystem as bundle dir"
+ );
+ }
+ throw e;
+ })
+ );
}).then(function() {
- return { metabook: metabook, builddir: builddir };
+ return builddir;
});
};
@@ -1595,11 +1642,24 @@
});
var metabook, builddir, imagemap;
return Promise.resolve().then(function() {
- // unpack the bundle
- return unpackBundle(options);
- }).then(function(args) {
- metabook = args.metabook;
- builddir = args.builddir;
+ // were we given a zip file or a directory?
+ return P.call(fs.stat, fs, options.bundle);
+ }).then(function(stat) {
+ if (stat.isDirectory()) {
+ // create a workspace and hard link the provided
directory
+ return hardlinkBundle(options);
+ } else {
+ // unpack the bundle
+ return unpackBundle(options);
+ }
+ }).then(function(_builddir) {
+ builddir = _builddir;
+ // read the main metabook.json file
+ return P.call(
+ fs.readFile, fs, path.join(builddir, 'bundle',
'metabook.json')
+ ).then(function(data) {
+ metabook = JSON.parse(data);
+ });
}).then(function() {
// process images
return processImages(metabook, builddir, options);
--
To view, visit https://gerrit.wikimedia.org/r/149220
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Iad77b766a12654876a7c140220df9a37dc25f6c6
Gerrit-PatchSet: 3
Gerrit-Project:
mediawiki/extensions/Collection/OfflineContentGenerator/latex_renderer
Gerrit-Branch: master
Gerrit-Owner: Cscott <[email protected]>
Gerrit-Reviewer: Cscott <[email protected]>
Gerrit-Reviewer: Mwalker <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
