Author: [email protected]
Date: Fri May 15 16:23:53 2009
New Revision: 5406
Modified:
trunk/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardLinkerContext.java
trunk/dev/core/src/com/google/gwt/dev/HostedMode.java
trunk/dev/core/src/com/google/gwt/dev/Link.java
trunk/dev/oophm/overlay/com/google/gwt/dev/HostedMode.java
Log:
Implement link-to-zip archive, which can be faster for big links.
When a user specifies a -war or -extra param that ends with ".zip", ".jar",
or ".war", produce output as an archive instead of a filesystem folder.
This feature is intended for build systems which Link, then immediately zip
up the output directory. Doing this within the Link phase, is actually
much faster, since it bypasses the need to create a whole directory tree on
disk.
Still TODO: figure out a way to parallelize the compression across multiple
processors.
Review by: spoon, fabbott
Modified:
trunk/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardLinkerContext.java
==============================================================================
---
trunk/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardLinkerContext.java
(original)
+++
trunk/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardLinkerContext.java
Fri May 15 16:23:53 2009
@@ -53,8 +53,11 @@
import com.google.gwt.dev.util.DefaultTextOutput;
import com.google.gwt.dev.util.FileBackedObject;
import com.google.gwt.dev.util.Util;
+import com.google.gwt.util.tools.Utility;
+import java.io.BufferedOutputStream;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.Reader;
@@ -69,6 +72,8 @@
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
/**
* An implementation of {...@link LinkerContext} that is initialized from a
@@ -114,6 +119,16 @@
};
/**
+ * Returns the parent path of forward-slash based partial path. Assumes
the
+ * given path does not end with a trailing slash.
+ */
+ private static String getParentPath(String path) {
+ assert !path.endsWith("/");
+ int pos = path.lastIndexOf('/');
+ return (pos >= 0) ? path.substring(0, pos) : null;
+ }
+
+ /**
* A faster bulk version of {...@link File#mkdirs()} that takes advantage of
* cached state to avoid a lot of file system access.
*/
@@ -132,6 +147,33 @@
createdDirs.add(path);
}
+ /**
+ * Creates directory entries within a zip archive. This is consistent
with how
+ * most tools operate.
+ *
+ * @param path the path of a directory within the archive to create
+ * @param zipOutputStream the archive we're creating
+ * @param createdDirs the set of already-created directories to avoid
+ * duplication
+ */
+ private static void mkzipDirs(String path, ZipOutputStream
zipOutputStream,
+ Set<String> createdDirs) throws IOException {
+ if (path == null) {
+ return;
+ }
+ if (createdDirs.contains(path)) {
+ return;
+ }
+ mkzipDirs(getParentPath(path), zipOutputStream, createdDirs);
+ ZipEntry entry = new ZipEntry(path + '/');
+ entry.setSize(0);
+ entry.setCompressedSize(0);
+ entry.setCrc(0);
+ entry.setMethod(ZipOutputStream.STORED);
+ zipOutputStream.putNextEntry(entry);
+ createdDirs.add(path);
+ }
+
private final ArtifactSet artifacts = new ArtifactSet();
private final SortedSet<ConfigurationProperty> configurationProperties;
@@ -452,22 +494,86 @@
}
/**
- * Writes artifacts into output directories in the standard way.
+ * Writes artifacts into the extra directory in the standard way.
*
* @param logger logs the operation
* @param artifacts the set of artifacts to write
- * @param outputPath the output path for deployable artifacts
* @param extraPath optional extra path for non-deployable artifacts
* @throws UnableToCompleteException
*/
- public void produceOutputDirectory(TreeLogger logger, ArtifactSet
artifacts,
- File outputPath, File extraPath) throws UnableToCompleteException {
+ public void produceExtraDirectory(TreeLogger logger, ArtifactSet
artifacts,
+ File extraPath) throws UnableToCompleteException {
+ extraPath = extraPath.getAbsoluteFile();
+ logger = logger.branch(TreeLogger.TRACE, "Writing extras into "
+ + extraPath.getPath(), null);
- outputPath = outputPath.getAbsoluteFile();
- if (extraPath != null) {
- extraPath = extraPath.getAbsoluteFile();
+ Set<String> createdDirs = new HashSet<String>();
+ for (EmittedArtifact artifact : artifacts.find(EmittedArtifact.class))
{
+ TreeLogger artifactLogger = logger.branch(TreeLogger.DEBUG,
+ "Emitting resource " + artifact.getPartialPath(), null);
+
+ if (!artifact.isPrivate()) {
+ continue;
+ }
+
+ File outFile = new File(extraPath, getExtraPathForLinker(
+ artifact.getLinker(), artifact.getPartialPath()));
+ writeArtifactToFile(artifactLogger, artifact, outFile, createdDirs);
}
+ }
+ /**
+ * Writes artifacts into an extra zip in the standard way.
+ *
+ * @param logger logs the operation
+ * @param artifacts the set of artifacts to write
+ * @param extraZip the output zip for deployable artifacts
+ * @param pathPrefix path within the zip to write into; if non-empty
must end
+ * with a trailing slash
+ * @throws UnableToCompleteException
+ */
+ public void produceExtraZip(TreeLogger logger, ArtifactSet artifacts,
+ File extraZip, String pathPrefix) throws UnableToCompleteException {
+ extraZip = extraZip.getAbsoluteFile();
+ logger = logger.branch(TreeLogger.TRACE, "Linking compilation into "
+ + extraZip.getPath(), null);
+
+ try {
+ Set<String> createdDirs = new HashSet<String>();
+ ZipOutputStream zipOutputStream = new ZipOutputStream(
+ new BufferedOutputStream(new FileOutputStream(extraZip)));
+ for (EmittedArtifact artifact :
artifacts.find(EmittedArtifact.class)) {
+ TreeLogger artifactLogger = logger.branch(TreeLogger.DEBUG,
+ "Emitting resource " + artifact.getPartialPath(), null);
+
+ if (!artifact.isPrivate()) {
+ continue;
+ }
+ String path = pathPrefix
+ + getExtraPathForLinker(artifact.getLinker(),
+ artifact.getPartialPath());
+ writeArtifactToZip(artifactLogger, artifact, path, zipOutputStream,
+ createdDirs);
+ }
+ Utility.close(zipOutputStream);
+ } catch (FileNotFoundException e) {
+ logger.log(TreeLogger.ERROR, "Unable to create extra archive "
+ + extraZip.getPath(), e);
+ throw new UnableToCompleteException();
+ }
+ }
+
+ /**
+ * Writes artifacts into output directory in the standard way.
+ *
+ * @param logger logs the operation
+ * @param artifacts the set of artifacts to write
+ * @param outputPath the output path for deployable artifacts
+ * @throws UnableToCompleteException
+ */
+ public void produceOutputDirectory(TreeLogger logger, ArtifactSet
artifacts,
+ File outputPath) throws UnableToCompleteException {
+ outputPath = outputPath.getAbsoluteFile();
logger = logger.branch(TreeLogger.TRACE, "Linking compilation into "
+ outputPath.getPath(), null);
@@ -476,31 +582,50 @@
TreeLogger artifactLogger = logger.branch(TreeLogger.DEBUG,
"Emitting resource " + artifact.getPartialPath(), null);
- File outFile;
if (artifact.isPrivate()) {
- if (extraPath == null) {
+ continue;
+ }
+ File outFile = new File(outputPath, artifact.getPartialPath());
+ writeArtifactToFile(artifactLogger, artifact, outFile, createdDirs);
+ }
+ }
+
+ /**
+ * Writes artifacts into an output zip in the standard way.
+ *
+ * @param logger logs the operation
+ * @param artifacts the set of artifacts to write
+ * @param outZip the output zip for deployable artifacts
+ * @param pathPrefix path within the zip to write into; if non-empty
must end
+ * with a trailing slash
+ * @throws UnableToCompleteException
+ */
+ public void produceOutputZip(TreeLogger logger, ArtifactSet artifacts,
+ File outZip, String pathPrefix) throws UnableToCompleteException {
+ outZip = outZip.getAbsoluteFile();
+ logger = logger.branch(TreeLogger.TRACE, "Linking compilation into "
+ + outZip.getPath(), null);
+
+ try {
+ ZipOutputStream zipOutputStream = new ZipOutputStream(
+ new BufferedOutputStream(new FileOutputStream(outZip)));
+ Set<String> createdDirs = new HashSet<String>();
+ for (EmittedArtifact artifact :
artifacts.find(EmittedArtifact.class)) {
+ TreeLogger artifactLogger = logger.branch(TreeLogger.DEBUG,
+ "Emitting resource " + artifact.getPartialPath(), null);
+
+ if (artifact.isPrivate()) {
continue;
}
- outFile = new File(getExtraPathForLinker(extraPath,
- artifact.getLinker()), artifact.getPartialPath());
- } else {
- outFile = new File(outputPath, artifact.getPartialPath());
- }
-
- if (!outFile.exists()
- || (outFile.lastModified() <= artifact.getLastModified())) {
- mkdirs(outFile.getParentFile(), createdDirs);
- try {
- FileOutputStream out = new FileOutputStream(outFile);
- artifact.writeTo(artifactLogger, out);
- out.close();
- } catch (IOException e) {
- artifactLogger.log(TreeLogger.ERROR, "Unable to create file '"
- + outFile.getAbsolutePath() + "'", e);
- throw new UnableToCompleteException();
- }
- outFile.setLastModified(artifact.getLastModified());
- }
+ String path = pathPrefix + artifact.getPartialPath();
+ writeArtifactToZip(artifactLogger, artifact, path, zipOutputStream,
+ createdDirs);
+ }
+ Utility.close(zipOutputStream);
+ } catch (FileNotFoundException e) {
+ logger.log(TreeLogger.ERROR, "Unable to create output archive "
+ + outZip.getPath(), e);
+ throw new UnableToCompleteException();
}
}
@@ -508,10 +633,45 @@
* Creates a linker-specific subdirectory in the module's auxiliary
output
* directory.
*/
- private File getExtraPathForLinker(File extraPath,
- Class<? extends Linker> linkerType) {
+ private String getExtraPathForLinker(Class<? extends Linker> linkerType,
+ String partialPath) {
assert linkerShortNames.containsKey(linkerType) : linkerType.getName()
+ " unknown";
- return new File(extraPath, linkerShortNames.get(linkerType));
+ return linkerShortNames.get(linkerType) + '/' + partialPath;
+ }
+
+ private void writeArtifactToFile(TreeLogger logger, EmittedArtifact
artifact,
+ File outFile, Set<String> createdDirs) throws
UnableToCompleteException {
+ if (!outFile.exists()
+ || (outFile.lastModified() <= artifact.getLastModified())) {
+ mkdirs(outFile.getParentFile(), createdDirs);
+ try {
+ FileOutputStream out = new FileOutputStream(outFile);
+ artifact.writeTo(logger, out);
+ out.close();
+ } catch (IOException e) {
+ logger.log(TreeLogger.ERROR, "Unable to create file '"
+ + outFile.getAbsolutePath() + "'", e);
+ throw new UnableToCompleteException();
+ }
+ outFile.setLastModified(artifact.getLastModified());
+ }
+ }
+
+ private void writeArtifactToZip(TreeLogger logger, EmittedArtifact
artifact,
+ String path, ZipOutputStream zipOutputStream, Set<String>
createdDirs)
+ throws UnableToCompleteException {
+ try {
+ mkzipDirs(getParentPath(path), zipOutputStream, createdDirs);
+ ZipEntry zipEntry = new ZipEntry(path);
+ zipEntry.setTime(artifact.getLastModified());
+ zipOutputStream.putNextEntry(zipEntry);
+ artifact.writeTo(logger, zipOutputStream);
+ zipOutputStream.closeEntry();
+ } catch (IOException e) {
+ logger.log(TreeLogger.ERROR, "Unable to write out artifact '"
+ + artifact.getPartialPath() + "'", e);
+ throw new UnableToCompleteException();
+ }
}
}
Modified: trunk/dev/core/src/com/google/gwt/dev/HostedMode.java
==============================================================================
--- trunk/dev/core/src/com/google/gwt/dev/HostedMode.java (original)
+++ trunk/dev/core/src/com/google/gwt/dev/HostedMode.java Fri May 15
16:23:53 2009
@@ -219,7 +219,7 @@
@Deprecated
public void setOutDir(File outDir) {
- this.outDir = outDir;
+ this.outDir = outDir;
}
public void setServletContainerLauncher(ServletContainerLauncher scl) {
@@ -457,19 +457,24 @@
TreeLogger linkLogger = logger.branch(TreeLogger.DEBUG, "Linking
module '"
+ module.getName() + "'");
- // TODO: move the module-specific computations to a helper function.
- File moduleOutDir = new File(options.getWarDir(), module.getName());
- File moduleExtraDir = (options.getExtraDir() == null) ? null : new
File(
- options.getExtraDir(), module.getName());
-
// Create a new active linker stack for the fresh link.
StandardLinkerContext linkerStack = new
StandardLinkerContext(linkLogger,
module, options);
linkerStacks.put(module.getName(), linkerStack);
ArtifactSet artifacts = linkerStack.invokeLink(linkLogger);
- linkerStack.produceOutputDirectory(linkLogger, artifacts, moduleOutDir,
- moduleExtraDir);
+ produceOutput(linkLogger, linkerStack, artifacts, module);
+ }
+
+ private void produceOutput(TreeLogger logger,
+ StandardLinkerContext linkerStack, ArtifactSet artifacts, ModuleDef
module)
+ throws UnableToCompleteException {
+ File moduleOutDir = new File(options.getWarDir(), module.getName());
+ linkerStack.produceOutputDirectory(logger, artifacts, moduleOutDir);
+ if (options.getExtraDir() != null) {
+ File moduleExtraDir = new File(options.getExtraDir(),
module.getName());
+ linkerStack.produceExtraDirectory(logger, artifacts, moduleExtraDir);
+ }
}
/**
@@ -486,19 +491,13 @@
TreeLogger linkLogger = logger.branch(TreeLogger.DEBUG,
"Relinking module '" + module.getName() + "'");
- // TODO: move the module-specific computations to a helper function.
- File moduleOutDir = new File(options.getWarDir(), module.getName());
- File moduleExtraDir = (options.getExtraDir() == null) ? null : new
File(
- options.getExtraDir(), module.getName());
-
// Find the existing linker stack.
StandardLinkerContext linkerStack = linkerStacks.get(module.getName());
assert linkerStack != null;
ArtifactSet artifacts = linkerStack.invokeRelink(linkLogger,
newlyGeneratedArtifacts);
- linkerStack.produceOutputDirectory(linkLogger, artifacts, moduleOutDir,
- moduleExtraDir);
+ produceOutput(linkLogger, linkerStack, artifacts, module);
}
private void validateServletTags(TreeLogger logger,
Modified: trunk/dev/core/src/com/google/gwt/dev/Link.java
==============================================================================
--- trunk/dev/core/src/com/google/gwt/dev/Link.java (original)
+++ trunk/dev/core/src/com/google/gwt/dev/Link.java Fri May 15 16:23:53 2009
@@ -200,24 +200,37 @@
File moduleExtraDir = new File(outDir, module.getName() + "-aux");
Util.recursiveDelete(moduleOutDir, true);
Util.recursiveDelete(moduleExtraDir, true);
- linkerContext.produceOutputDirectory(logger, artifacts, moduleOutDir,
- moduleExtraDir);
+ linkerContext.produceOutputDirectory(logger, artifacts, moduleOutDir);
+ linkerContext.produceExtraDirectory(logger, artifacts, moduleExtraDir);
logger.log(TreeLogger.INFO, "Link succeeded");
}
private static void doProduceOutput(TreeLogger logger, ArtifactSet
artifacts,
StandardLinkerContext linkerContext, ModuleDef module, File outDir,
File extraDir) throws UnableToCompleteException {
- // TODO: move the module-specific computations to a helper function.
- File moduleOutDir = new File(outDir, module.getName());
- File moduleExtraDir = (extraDir == null) ? null : new File(extraDir,
- module.getName());
- Util.recursiveDelete(moduleOutDir, true);
- if (moduleExtraDir != null) {
- Util.recursiveDelete(moduleExtraDir, true);
+ String outPath = outDir.getPath();
+ if (!outDir.isDirectory()
+ && (outPath.endsWith(".war") || outPath.endsWith(".jar") ||
outPath.endsWith(".zip"))) {
+ linkerContext.produceOutputZip(logger, artifacts, outDir,
+ module.getName() + '/');
+ } else {
+ File moduleOutDir = new File(outDir, module.getName());
+ Util.recursiveDelete(moduleOutDir, true);
+ linkerContext.produceOutputDirectory(logger, artifacts,
moduleOutDir);
+ }
+
+ if (extraDir != null) {
+ String extraPath = extraDir.getPath();
+ if (!extraDir.isDirectory()
+ && (extraPath.endsWith(".war") || extraPath.endsWith(".jar") ||
extraPath.endsWith(".zip"))) {
+ linkerContext.produceExtraZip(logger, artifacts, extraDir,
+ module.getName() + '/');
+ } else {
+ File moduleExtraDir = new File(extraDir, module.getName());
+ Util.recursiveDelete(moduleExtraDir, true);
+ linkerContext.produceExtraDirectory(logger, artifacts,
moduleExtraDir);
+ }
}
- linkerContext.produceOutputDirectory(logger, artifacts, moduleOutDir,
- moduleExtraDir);
logger.log(TreeLogger.INFO, "Link succeeded");
}
Modified: trunk/dev/oophm/overlay/com/google/gwt/dev/HostedMode.java
==============================================================================
--- trunk/dev/oophm/overlay/com/google/gwt/dev/HostedMode.java (original)
+++ trunk/dev/oophm/overlay/com/google/gwt/dev/HostedMode.java Fri May 15
16:23:53 2009
@@ -470,19 +470,24 @@
TreeLogger linkLogger = logger.branch(TreeLogger.DEBUG, "Linking
module '"
+ module.getName() + "'");
- // TODO: move the module-specific computations to a helper function.
- File moduleOutDir = new File(options.getWarDir(), module.getName());
- File moduleExtraDir = (options.getExtraDir() == null) ? null : new
File(
- options.getExtraDir(), module.getName());
-
// Create a new active linker stack for the fresh link.
StandardLinkerContext linkerStack = new
StandardLinkerContext(linkLogger,
module, options);
linkerStacks.put(module.getName(), linkerStack);
ArtifactSet artifacts = linkerStack.invokeLink(linkLogger);
- linkerStack.produceOutputDirectory(linkLogger, artifacts, moduleOutDir,
- moduleExtraDir);
+ produceOutput(linkLogger, linkerStack, artifacts, module);
+ }
+
+ private void produceOutput(TreeLogger logger,
+ StandardLinkerContext linkerStack, ArtifactSet artifacts, ModuleDef
module)
+ throws UnableToCompleteException {
+ File moduleOutDir = new File(options.getWarDir(), module.getName());
+ linkerStack.produceOutputDirectory(logger, artifacts, moduleOutDir);
+ if (options.getExtraDir() != null) {
+ File moduleExtraDir = new File(options.getExtraDir(),
module.getName());
+ linkerStack.produceExtraDirectory(logger, artifacts, moduleExtraDir);
+ }
}
/**
@@ -499,19 +504,13 @@
TreeLogger linkLogger = logger.branch(TreeLogger.DEBUG,
"Relinking module '" + module.getName() + "'");
- // TODO: move the module-specific computations to a helper function.
- File moduleOutDir = new File(options.getWarDir(), module.getName());
- File moduleExtraDir = (options.getExtraDir() == null) ? null : new
File(
- options.getExtraDir(), module.getName());
-
// Find the existing linker stack.
StandardLinkerContext linkerStack = linkerStacks.get(module.getName());
assert linkerStack != null;
ArtifactSet artifacts = linkerStack.invokeRelink(linkLogger,
newlyGeneratedArtifacts);
- linkerStack.produceOutputDirectory(linkLogger, artifacts, moduleOutDir,
- moduleExtraDir);
+ produceOutput(linkLogger, linkerStack, artifacts, module);
}
private void validateServletTags(TreeLogger logger,
--~--~---------~--~----~------------~-------~--~----~
http://groups.google.com/group/Google-Web-Toolkit-Contributors
-~----------~----~----~----~------~----~------~--~---
