Revision: 6501 Author: [email protected] Date: Tue Oct 27 23:15:10 2009 Log: Implements "-XshardPrecompile" for sharding the precompilation.
When enabled, Precompile merely counts the number of permutations, and all the real work happens during CompilePerms. This net result is more work is done, but if enough processors can run perms in parallel, a compile can finish faster in wall time. Traditionally, peak memory use (and therefore OutOfMemoryError) tends occur while Precompiling many permutations; enabling precompile sharding will greatly reduce peak memory requirements. Patch by: spoon Review by: me http://code.google.com/p/google-web-toolkit/source/detail?r=6501 Added: /trunk/dev/core/src/com/google/gwt/dev/PrecompilationResult.java /trunk/dev/core/src/com/google/gwt/dev/util/arg/ArgHandlerShardPrecompile.java /trunk/dev/core/src/com/google/gwt/dev/util/arg/OptionEnableGeneratingOnShards.java Deleted: /trunk/dev/core/src/com/google/gwt/dev/PrecompilationFile.java Modified: /trunk/dev/core/src/com/google/gwt/dev/CompilePerms.java /trunk/dev/core/src/com/google/gwt/dev/Compiler.java /trunk/dev/core/src/com/google/gwt/dev/GWTCompiler.java /trunk/dev/core/src/com/google/gwt/dev/Link.java /trunk/dev/core/src/com/google/gwt/dev/Precompilation.java /trunk/dev/core/src/com/google/gwt/dev/Precompile.java /trunk/dev/core/src/com/google/gwt/dev/cfg/Properties.java /trunk/dev/core/src/com/google/gwt/dev/jjs/JavaToJavaScriptCompiler.java /trunk/dev/core/src/com/google/gwt/dev/jjs/PermutationResult.java ======================================= --- /dev/null +++ /trunk/dev/core/src/com/google/gwt/dev/PrecompilationResult.java Tue Oct 27 23:15:10 2009 @@ -0,0 +1,26 @@ +/* + * Copyright 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.gwt.dev; + +import java.io.Serializable; + +/** + * A result of running {...@link Precompile}. It's either a {...@link Precompilation} + * or, if precompiles are going to happen on compile shards, an instance of + * {...@link com.google.gwt.dev.Precompile.PrecompileOptions}. + */ +public interface PrecompilationResult extends Serializable { +} ======================================= --- /dev/null +++ /trunk/dev/core/src/com/google/gwt/dev/util/arg/ArgHandlerShardPrecompile.java Tue Oct 27 23:15:10 2009 @@ -0,0 +1,45 @@ +/* + * Copyright 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.gwt.dev.util.arg; + +import com.google.gwt.util.tools.ArgHandlerFlag; + +/** + * An argument handler that enables running generators on shards. + */ +public class ArgHandlerShardPrecompile extends ArgHandlerFlag { + private OptionEnableGeneratingOnShards options; + + public ArgHandlerShardPrecompile(OptionEnableGeneratingOnShards options) { + this.options = options; + } + + @Override + public String getPurpose() { + return "Enables running generators on CompilePerms shards"; + } + + @Override + public String getTag() { + return "-XshardPrecompile"; + } + + @Override + public boolean setFlag() { + options.setEnabledGeneratingOnShards(true); + return true; + } +} ======================================= --- /dev/null +++ /trunk/dev/core/src/com/google/gwt/dev/util/arg/OptionEnableGeneratingOnShards.java Tue Oct 27 23:15:10 2009 @@ -0,0 +1,32 @@ +/* + * Copyright 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.gwt.dev.util.arg; + +/** + * Enables running generators on compile shards rather than during the + * Precompile stage. + */ +public interface OptionEnableGeneratingOnShards { + /** + * Returns <code>true</code> if generation is allowed to happen on shards. + */ + boolean isEnabledGeneratingOnShards(); + + /** + * Sets whether generation may happen on shards. + */ + void setEnabledGeneratingOnShards(boolean allowed); +} ======================================= --- /trunk/dev/core/src/com/google/gwt/dev/PrecompilationFile.java Fri May 15 16:23:01 2009 +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright 2009 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package com.google.gwt.dev; - -import com.google.gwt.core.ext.TreeLogger; -import com.google.gwt.core.ext.UnableToCompleteException; -import com.google.gwt.dev.util.Util; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Enumeration; -import java.util.List; -import java.util.jar.JarEntry; -import java.util.jar.JarFile; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.zip.ZipEntry; - -/** - * The backing file for a {...@link Precompilation} for a subset of the - * permutations for a compilation. The permutations this one is for are in a - * consecutive range described by {...@link #getFirstPerm()} and - * {...@link #getNumPerms()}. - */ -public class PrecompilationFile { - /** - * Suffix for precompilation files stored in the work directory. - */ - private static final String FILENAME_PREFIX = "precompilation"; - - /** - * Prefix for precompilation files stored in the work directory. - */ - private static final String FILENAME_SUFFIX = ".ser"; - - public static String fileNameForPermutations(int firstPerm, int numPerms) { - return FILENAME_PREFIX + firstPerm + "-" + (firstPerm + numPerms - 1) - + FILENAME_SUFFIX; - } - - public static Collection<PrecompilationFile> scanJarFile(File file) - throws IOException { - Pattern pattern = Pattern.compile("precompilation([0-9]+)-([0-9]+)\\.ser"); - - List<PrecompilationFile> precomps = new ArrayList<PrecompilationFile>(); - - JarFile jarFile = new JarFile(file); - Enumeration<JarEntry> entries = jarFile.entries(); - while (entries.hasMoreElements()) { - JarEntry entry = entries.nextElement(); - Matcher matcher = pattern.matcher(entry.getName()); - if (matcher.matches()) { - int start = Integer.parseInt(matcher.group(1)); - int end = Integer.parseInt(matcher.group(2)); - int numPerms = end - start + 1; - - precomps.add(new PrecompilationFile(jarFile, entry, start, numPerms)); - } - } - - return precomps; - } - - private final JarFile jarFile; - private final ZipEntry zipEntry; - - private final int firstPerm; - - private final int numPerms; - - public PrecompilationFile(JarFile jarFile, ZipEntry zipEntry, int firstPerm, - int numPerms) { - this.firstPerm = firstPerm; - this.numPerms = numPerms; - - this.jarFile = jarFile; - this.zipEntry = zipEntry; - } - - /** - * Return the first permutation this {...@link Precompilation} is for. - */ - public int getFirstPerm() { - return firstPerm; - } - - /** - * Get the number of the highest permutation included in this - * {...@link Precompilation}. - */ - public int getLastPerm() { - return getFirstPerm() + getNumPerms() - 1; - } - - /** - * Return the number of permutations in this {...@link Precompilation}. - */ - public int getNumPerms() { - return numPerms; - } - - public boolean isForPermutation(int perm) { - return perm >= getFirstPerm() && perm <= getLastPerm(); - } - - public Precompilation newInstance(TreeLogger logger) - throws UnableToCompleteException { - try { - return Util.readStreamAsObject(jarFile.getInputStream(zipEntry), - Precompilation.class); - } catch (IOException e) { - logger.log(TreeLogger.ERROR, "Unable to instantiate object", e); - throw new UnableToCompleteException(); - } catch (ClassNotFoundException e) { - logger.log(TreeLogger.ERROR, "Missing class definition", e); - throw new UnableToCompleteException(); - } - } -} ======================================= --- /trunk/dev/core/src/com/google/gwt/dev/CompilePerms.java Tue Oct 27 19:12:04 2009 +++ /trunk/dev/core/src/com/google/gwt/dev/CompilePerms.java Tue Oct 27 23:15:10 2009 @@ -18,9 +18,14 @@ import com.google.gwt.core.ext.TreeLogger; import com.google.gwt.core.ext.UnableToCompleteException; import com.google.gwt.dev.CompileTaskRunner.CompileTask; +import com.google.gwt.dev.Precompile.PrecompileOptions; +import com.google.gwt.dev.cfg.ModuleDef; +import com.google.gwt.dev.cfg.ModuleDefLoader; +import com.google.gwt.dev.cfg.PropertyPermutations; import com.google.gwt.dev.jjs.PermutationResult; import com.google.gwt.dev.jjs.UnifiedAst; import com.google.gwt.dev.util.FileBackedObject; +import com.google.gwt.dev.util.Util; import com.google.gwt.dev.util.arg.ArgHandlerLocalWorkers; import com.google.gwt.dev.util.arg.OptionLocalWorkers; import com.google.gwt.util.tools.ArgHandlerString; @@ -28,7 +33,6 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; import java.util.List; import java.util.SortedSet; import java.util.TreeSet; @@ -243,40 +247,23 @@ static File makePermFilename(File compilerWorkDir, int permNumber) { return new File(compilerWorkDir, "permutation-" + permNumber + ".js"); } - - /** - * Check whether any of the listed permutations have their precompilation in - * the supplied precompilation file. - */ - private static boolean isPrecompileForAnyOf( - PrecompilationFile precompilationFile, int[] permutations) { - if (permutations == null) { - // Special case: compile everything. - return true; - } - for (int perm : permutations) { - if (precompilationFile.isForPermutation(perm)) { - return true; - } - } - return false; - } /** * Choose the subset of requested permutations that correspond to the * indicated precompilation. */ private static Permutation[] selectPermutationsForPrecompilation( - int[] permsToRun, PrecompilationFile precompilationFile, - Precompilation precompilation) { + int[] permsToRun, Precompilation precompilation) { if (permsToRun == null) { // Special case: compile everything. return precompilation.getPermutations(); } ArrayList<Permutation> subPermsList = new ArrayList<Permutation>(); - for (int perm : permsToRun) { - if (precompilationFile.isForPermutation(perm)) { - subPermsList.add(precompilation.getPermutation(perm)); + for (int id : permsToRun) { + for (Permutation perm : precompilation.getPermutations()) { + if (perm.getId() == id) { + subPermsList.add(perm); + } } } return subPermsList.toArray(new Permutation[subPermsList.size()]); @@ -296,56 +283,39 @@ int[] permsToRun = options.getPermsToCompile(); File compilerWorkDir = options.getCompilerWorkDir(moduleName); - Collection<PrecompilationFile> precomps; + File precompilationFile = new File(compilerWorkDir, + Precompile.PRECOMPILE_FILENAME); + + PrecompilationResult precompileResults; try { - precomps = PrecompilationFile.scanJarFile(new File(compilerWorkDir, - Precompile.PRECOMPILE_FILENAME)); + precompileResults = Util.readFileAsObject(precompilationFile, + PrecompilationResult.class); } catch (IOException e) { - logger.log(TreeLogger.ERROR, "Failed to scan " - + Precompile.PRECOMPILE_FILENAME + "; has Precompile been run?", e); + logger.log(TreeLogger.ERROR, "Failed to read " + + Precompile.PRECOMPILE_FILENAME + "; has Precompile been run?"); return false; + } catch (ClassNotFoundException e) { + logger.log(TreeLogger.ERROR, "Failed to read " + + Precompile.PRECOMPILE_FILENAME, e); + return false; } /* - * Check that all requested permutations actually have a Precompile - * available + * TODO(spoon) Check that all requested permutations have a permutation + * available before starting. probably needs two branches. */ - if (permsToRun != null) { - checking_perms : for (int perm : permsToRun) { - for (PrecompilationFile precomp : precomps) { - if (precomp.isForPermutation(perm)) { - continue checking_perms; - } - } - logger.log(TreeLogger.ERROR, - "No precompilation file found for permutation " + perm); - return false; - } - } else { - // TODO: validate that a contiguous set of all perms exists. - } - - /* - * Perform the compiles one file at a time, to minimize the number of - * times a Precompilation needs to be deserialized. - */ - for (PrecompilationFile precompilationFile : precomps) { - if (!isPrecompileForAnyOf(precompilationFile, permsToRun)) { - continue; - } - Precompilation precompilation; - try { - /* - * TODO: don't bother deserializing the generated artifacts. - */ - precompilation = precompilationFile.newInstance(logger); - } catch (UnableToCompleteException e) { + + if (precompileResults instanceof PrecompileOptions) { + PrecompileOptions precompilationOptions = (PrecompileOptions) precompileResults; + if (!precompileAndCompile(logger, moduleName, compilerWorkDir, + precompilationOptions)) { return false; } - + } else { + Precompilation precompilation = (Precompilation) precompileResults; // Choose which permutations go with this permutation Permutation[] subPerms = selectPermutationsForPrecompilation( - permsToRun, precompilationFile, precompilation); + permsToRun, precompilation); List<FileBackedObject<PermutationResult>> resultFiles = makeResultFiles( compilerWorkDir, subPerms); @@ -356,4 +326,63 @@ return true; } -} + + /** + * Run both a precompile and a compile with the given precompilation options. + */ + private boolean precompileAndCompile(TreeLogger logger, String moduleName, + File compilerWorkDir, PrecompileOptions precompilationOptions) + throws UnableToCompleteException { + precompilationOptions.setOptimizePrecompile(false); + precompilationOptions.setCompilationStateRetained(true); + precompilationOptions.setGenDir(null); + + ModuleDef module = ModuleDefLoader.loadFromClassPath(logger, moduleName); + PropertyPermutations allPermutations = new PropertyPermutations( + module.getProperties()); + int[] perms = options.getPermsToCompile(); + if (perms == null) { + perms = new int[allPermutations.size()]; + for (int i = 0; i < perms.length; ++i) { + perms[i] = i; + } + } + + logger = logger.branch(TreeLogger.INFO, "Compiling " + perms.length + + " permutation" + (perms.length > 1 ? "s" : "")); + for (int permId : perms) { + /* + * TODO(spoon,scottb): move Precompile out of the loop to run only once + * per shard. We'll need a new PropertyPermutations constructor that can + * take a precise list. Then figure out a way to avoid copying the + * generated artifacts into every perm result on a shard. + */ + module.getCompilationState(logger).refresh(logger); + PropertyPermutations onePerm = new PropertyPermutations(allPermutations, + permId, 1); + + assert (precompilationOptions.getDumpSignatureFile() == null); + Precompilation precompilation = Precompile.precompile(logger, + precompilationOptions, module, permId, onePerm, + precompilationOptions.getGenDir(), compilerWorkDir, null); + if (precompilation == null) { + return false; + } + + // Choose which permutations go with this precompilation + Permutation[] subPerms = selectPermutationsForPrecompilation( + new int[] {permId}, precompilation); + assert subPerms.length == 1; + + PermutationResult permResult = compile(logger, subPerms[0], + precompilation.getUnifiedAst()); + FileBackedObject<PermutationResult> resultFile = new FileBackedObject<PermutationResult>( + PermutationResult.class, makePermFilename(compilerWorkDir, permId)); + permResult.addArtifacts(precompilation.getGeneratedArtifacts()); + resultFile.set(logger, permResult); + } + + logger.log(TreeLogger.INFO, "Compile of permutations succeeded"); + return true; + } +} ======================================= --- /trunk/dev/core/src/com/google/gwt/dev/Compiler.java Fri Oct 23 13:15:54 2009 +++ /trunk/dev/core/src/com/google/gwt/dev/Compiler.java Tue Oct 27 23:15:10 2009 @@ -213,8 +213,8 @@ File absPath = new File(options.getWarDir(), module.getName()); absPath = absPath.getAbsoluteFile(); Link.link(logger.branch(TreeLogger.TRACE, "Linking into " + absPath), - module, generatedArtifacts, allPerms, resultFiles, - options.getWarDir(), options.getExtraDir(), precompileOptions); + module, generatedArtifacts, resultFiles, options.getWarDir(), + options.getExtraDir(), precompileOptions); long compileDone = System.currentTimeMillis(); long delta = compileDone - compileStart; ======================================= --- /trunk/dev/core/src/com/google/gwt/dev/GWTCompiler.java Fri Oct 23 13:15:54 2009 +++ /trunk/dev/core/src/com/google/gwt/dev/GWTCompiler.java Tue Oct 27 23:15:10 2009 @@ -204,7 +204,7 @@ Link.legacyLink(logger.branch(TreeLogger.TRACE, "Linking into " + options.getOutDir().getPath()), module, generatedArtifacts, - allPerms, resultFiles, options.getOutDir(), precompileOptions); + resultFiles, options.getOutDir(), precompileOptions); long compileDone = System.currentTimeMillis(); long delta = compileDone - compileStart; ======================================= --- /trunk/dev/core/src/com/google/gwt/dev/Link.java Tue Oct 27 18:11:59 2009 +++ /trunk/dev/core/src/com/google/gwt/dev/Link.java Tue Oct 27 23:15:10 2009 @@ -22,6 +22,7 @@ import com.google.gwt.core.ext.linker.impl.StandardCompilationResult; import com.google.gwt.core.ext.linker.impl.StandardLinkerContext; import com.google.gwt.dev.CompileTaskRunner.CompileTask; +import com.google.gwt.dev.Precompile.PrecompileOptions; import com.google.gwt.dev.cfg.BindingProperty; import com.google.gwt.dev.cfg.ModuleDef; import com.google.gwt.dev.cfg.ModuleDefLoader; @@ -45,7 +46,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -135,14 +135,14 @@ } public static void legacyLink(TreeLogger logger, ModuleDef module, - ArtifactSet generatedArtifacts, Permutation[] permutations, + ArtifactSet generatedArtifacts, List<FileBackedObject<PermutationResult>> resultFiles, File outDir, JJSOptions precompileOptions) throws UnableToCompleteException, IOException { StandardLinkerContext linkerContext = new StandardLinkerContext(logger, module, precompileOptions); ArtifactSet artifacts = doLink(logger, linkerContext, generatedArtifacts, - permutations, resultFiles); + resultFiles); OutputFileSet outFileSet = new OutputFileSetOnDirectory(outDir, module.getName() + "/"); OutputFileSet extraFileSet = new OutputFileSetOnDirectory(outDir, @@ -151,14 +151,14 @@ } public static void link(TreeLogger logger, ModuleDef module, - ArtifactSet generatedArtifacts, Permutation[] permutations, + ArtifactSet generatedArtifacts, List<FileBackedObject<PermutationResult>> resultFiles, File outDir, File extrasDir, JJSOptions precompileOptions) throws UnableToCompleteException, IOException { StandardLinkerContext linkerContext = new StandardLinkerContext(logger, module, precompileOptions); ArtifactSet artifacts = doLink(logger, linkerContext, generatedArtifacts, - permutations, resultFiles); + resultFiles); doProduceOutput(logger, artifacts, linkerContext, chooseOutputFileSet( outDir, module.getName() + "/"), chooseOutputFileSet(extrasDir, module.getName() + "/")); @@ -187,6 +187,19 @@ // Exit w/ non-success code. System.exit(1); } + + /** + * Add to a compilation result all of the selection permutations from its + * associated permutation. + */ + private static void addSelectionPermutations( + StandardCompilationResult compilation, Permutation perm, + StandardLinkerContext linkerContext) { + for (StaticPropertyOracle propOracle : perm.getPropertyOracles()) { + compilation.addSelectionPermutation(computeSelectionPermutation( + linkerContext, propOracle)); + } + } /** * Choose an output file set for the given <code>dirOrJar</code> based on @@ -218,21 +231,44 @@ return new OutputFileSetOnDirectory(dirOrJar, dirPathPrefix); } } + + /** + * Return a map giving the value of each non-trivial selection property. + */ + private static Map<SelectionProperty, String> computeSelectionPermutation( + StandardLinkerContext linkerContext, StaticPropertyOracle propOracle) { + BindingProperty[] orderedProps = propOracle.getOrderedProps(); + String[] orderedPropValues = propOracle.getOrderedPropValues(); + Map<SelectionProperty, String> unboundProperties = new HashMap<SelectionProperty, String>(); + for (int i = 0; i < orderedProps.length; i++) { + SelectionProperty key = linkerContext.getProperty(orderedProps[i].getName()); + if (key.tryGetValue() != null) { + /* + * The view of the Permutation doesn't include properties with defined + * values. + */ + continue; + } else if (key.isDerived()) { + /* + * The property provider does not need to be invoked, because the value + * is determined entirely by other properties. + */ + continue; + } + unboundProperties.put(key, orderedPropValues[i]); + } + return unboundProperties; + } private static ArtifactSet doLink(TreeLogger logger, StandardLinkerContext linkerContext, ArtifactSet generatedArtifacts, - Permutation[] perms, List<FileBackedObject<PermutationResult>> resultFiles) + List<FileBackedObject<PermutationResult>> resultFiles) throws UnableToCompleteException { - if (perms.length != resultFiles.size()) { - throw new IllegalArgumentException( - "Mismatched resultFiles.length and permutation count"); - } - - for (int i = 0; i < perms.length; ++i) { - finishPermuation(logger, perms[i], resultFiles.get(i), linkerContext); - } - linkerContext.addOrReplaceArtifacts(generatedArtifacts); + for (FileBackedObject<PermutationResult> resultFile : resultFiles) { + PermutationResult result = resultFile.newInstance(logger); + finishPermutation(logger, result.getPermutation(), result, linkerContext); + } return linkerContext.invokeLink(logger); } @@ -251,35 +287,13 @@ logger.log(TreeLogger.INFO, "Link succeeded"); } - private static void finishPermuation(TreeLogger logger, Permutation perm, - FileBackedObject<PermutationResult> resultFile, - StandardLinkerContext linkerContext) throws UnableToCompleteException { - PermutationResult permutationResult = resultFile.newInstance(logger); - StandardCompilationResult compilation = linkerContext.getCompilation(permutationResult); - StaticPropertyOracle[] propOracles = perm.getPropertyOracles(); - for (StaticPropertyOracle propOracle : propOracles) { - BindingProperty[] orderedProps = propOracle.getOrderedProps(); - String[] orderedPropValues = propOracle.getOrderedPropValues(); - Map<SelectionProperty, String> unboundProperties = new HashMap<SelectionProperty, String>(); - for (int i = 0; i < orderedProps.length; i++) { - SelectionProperty key = linkerContext.getProperty(orderedProps[i].getName()); - if (key.tryGetValue() != null) { - /* - * The view of the Permutation doesn't include properties with defined - * values. - */ - continue; - } else if (key.isDerived()) { - /* - * The property provider does not need to be invoked, because the - * value is determined entirely by other properties. - */ - continue; - } - unboundProperties.put(key, orderedPropValues[i]); - } - compilation.addSelectionPermutation(unboundProperties); - } + /** + * Add a compilation to a linker context. + */ + private static void finishPermutation(TreeLogger logger, Permutation perm, + PermutationResult permResult, StandardLinkerContext linkerContext) { + StandardCompilationResult compilation = linkerContext.getCompilation(permResult); + addSelectionPermutations(compilation, perm, linkerContext); logScriptSize(logger, perm.getId(), compilation); } @@ -315,17 +329,6 @@ public boolean run(TreeLogger logger) throws UnableToCompleteException { for (String moduleName : options.getModuleNames()) { - File compilerWorkDir = options.getCompilerWorkDir(moduleName); - Collection<PrecompilationFile> precomps; - try { - precomps = PrecompilationFile.scanJarFile(new File(compilerWorkDir, - Precompile.PRECOMPILE_FILENAME)); - } catch (IOException e) { - logger.log(TreeLogger.ERROR, "Failed to scan " - + Precompile.PRECOMPILE_FILENAME, e); - return false; - } - ModuleDef module = ModuleDefLoader.loadFromClassPath(logger, moduleName); OutputFileSet outFileSet; @@ -356,37 +359,54 @@ "Unexpected exception while producing output", e); throw new UnableToCompleteException(); } - - if (precomps.isEmpty()) { - logger.log(TreeLogger.ERROR, "No precompilation files found in '" - + compilerWorkDir.getAbsolutePath() - + "'; please run Precompile first"); - return false; - } List<Permutation> permsList = new ArrayList<Permutation>(); ArtifactSet generatedArtifacts = new ArtifactSet(); JJSOptions precompileOptions = null; - for (PrecompilationFile precompilationFile : precomps) { - Precompilation precompilation; - try { - precompilation = precompilationFile.newInstance(logger); - } catch (UnableToCompleteException e) { - return false; - } + File compilerWorkDir = options.getCompilerWorkDir(moduleName); + List<Integer> permutationIds = new ArrayList<Integer>(); + PrecompilationResult precompileResults; + try { + precompileResults = Util.readFileAsObject(new File(compilerWorkDir, + Precompile.PRECOMPILE_FILENAME), PrecompilationResult.class); + } catch (ClassNotFoundException e) { + logger.log(TreeLogger.ERROR, "Error reading " + + Precompile.PRECOMPILE_FILENAME); + return false; + } catch (IOException e) { + logger.log(TreeLogger.ERROR, "Error reading " + + Precompile.PRECOMPILE_FILENAME); + return false; + } + + if (precompileResults instanceof PrecompileOptions) { + /** + * Precompiling happened on the shards. + */ + precompileOptions = (JJSOptions) precompileResults; + int numPermutations = module.getProperties().numPermutations(); + for (int i = 0; i < numPermutations; ++i) { + permutationIds.add(i); + } + } else { + /** + * Precompiling happened on the start node. + */ + Precompilation precompilation = (Precompilation) precompileResults; permsList.addAll(Arrays.asList(precompilation.getPermutations())); generatedArtifacts.addAll(precompilation.getGeneratedArtifacts()); precompileOptions = precompilation.getUnifiedAst().getOptions(); - } - - Permutation[] perms = permsList.toArray(new Permutation[permsList.size()]); + + for (Permutation perm : precompilation.getPermutations()) { + permutationIds.add(perm.getId()); + } + } List<FileBackedObject<PermutationResult>> resultFiles = new ArrayList<FileBackedObject<PermutationResult>>( - perms.length); - for (int i = 0; i < perms.length; ++i) { - File f = CompilePerms.makePermFilename(compilerWorkDir, - perms[i].getId()); + permutationIds.size()); + for (int id : permutationIds) { + File f = CompilePerms.makePermFilename(compilerWorkDir, id); if (!f.exists()) { logger.log(TreeLogger.ERROR, "File not found '" + f.getAbsolutePath() + "'; please compile all permutations"); @@ -402,8 +422,7 @@ module, precompileOptions); ArtifactSet artifacts = doLink(branch, linkerContext, generatedArtifacts, - perms, resultFiles); - + resultFiles); try { doProduceOutput(branch, artifacts, linkerContext, outFileSet, extraFileSet); ======================================= --- /trunk/dev/core/src/com/google/gwt/dev/Precompilation.java Thu Mar 26 09:27:13 2009 +++ /trunk/dev/core/src/com/google/gwt/dev/Precompilation.java Tue Oct 27 23:15:10 2009 @@ -17,7 +17,13 @@ import com.google.gwt.core.ext.linker.ArtifactSet; import com.google.gwt.dev.jjs.UnifiedAst; - +import com.google.gwt.dev.util.Util; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.io.Serializable; import java.util.Collection; @@ -25,13 +31,14 @@ * The result of compilation phase 1, includes a unified AST and metadata * relevant to each permutation. */ -public class Precompilation implements Serializable { +public class Precompilation implements Serializable, PrecompilationResult { /* * TODO: don't make this whole class serializable, instead dump the * independent members out to a file so that the generated artifacts are * optional to deserialize. */ - private ArtifactSet generatedArtifacts; + private transient ArtifactSet generatedArtifacts; + private transient byte[] generatedArtifactsSerialized; private final Permutation[] permutations; private final UnifiedAst unifiedAst; @@ -68,6 +75,20 @@ * Returns the set of generated artifacts from the precompile phase. */ public ArtifactSet getGeneratedArtifacts() { + if (generatedArtifacts == null) { + try { + assert generatedArtifactsSerialized != null; + generatedArtifacts = Util.readStreamAsObject(new ByteArrayInputStream( + generatedArtifactsSerialized), ArtifactSet.class); + generatedArtifactsSerialized = null; + } catch (ClassNotFoundException e) { + throw new RuntimeException( + "Unexpected exception deserializing from memory stream", e); + } catch (IOException e) { + throw new RuntimeException( + "Unexpected exception deserializing from memory stream", e); + } + } return generatedArtifacts; } @@ -94,4 +115,17 @@ public UnifiedAst getUnifiedAst() { return unifiedAst; } -} + + private void readObject(ObjectInputStream stream) throws IOException, + ClassNotFoundException { + stream.defaultReadObject(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + Util.copyNoClose(stream, baos); + generatedArtifactsSerialized = baos.toByteArray(); + } + + private void writeObject(ObjectOutputStream stream) throws IOException { + stream.defaultWriteObject(); + Util.writeObjectToStream(stream, generatedArtifacts); + } +} ======================================= --- /trunk/dev/core/src/com/google/gwt/dev/Precompile.java Fri Oct 23 11:09:22 2009 +++ /trunk/dev/core/src/com/google/gwt/dev/Precompile.java Tue Oct 27 23:15:10 2009 @@ -54,19 +54,20 @@ import com.google.gwt.dev.util.arg.ArgHandlerEnableAssertions; import com.google.gwt.dev.util.arg.ArgHandlerGenDir; import com.google.gwt.dev.util.arg.ArgHandlerMaxPermsPerPrecompile; +import com.google.gwt.dev.util.arg.ArgHandlerShardPrecompile; import com.google.gwt.dev.util.arg.ArgHandlerScriptStyle; import com.google.gwt.dev.util.arg.ArgHandlerSoyc; import com.google.gwt.dev.util.arg.ArgHandlerSoycDetailed; import com.google.gwt.dev.util.arg.ArgHandlerValidateOnlyFlag; import com.google.gwt.dev.util.arg.OptionDisableUpdateCheck; import com.google.gwt.dev.util.arg.OptionDumpSignatures; +import com.google.gwt.dev.util.arg.OptionEnableGeneratingOnShards; import com.google.gwt.dev.util.arg.OptionGenDir; import com.google.gwt.dev.util.arg.OptionMaxPermsPerPrecompile; import com.google.gwt.dev.util.arg.OptionValidateOnly; import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; +import java.io.Serializable; import java.util.Collection; import java.util.HashSet; import java.util.Set; @@ -74,8 +75,6 @@ import java.util.SortedSet; import java.util.TreeMap; import java.util.concurrent.FutureTask; -import java.util.jar.JarOutputStream; -import java.util.zip.ZipEntry; /** * Performs the first phase of compilation, generating the set of permutations @@ -88,7 +87,8 @@ */ public interface PrecompileOptions extends JJSOptions, CompileTaskOptions, OptionGenDir, OptionValidateOnly, OptionDisableUpdateCheck, - OptionDumpSignatures, OptionMaxPermsPerPrecompile { + OptionDumpSignatures, OptionEnableGeneratingOnShards, + OptionMaxPermsPerPrecompile, PrecompilationResult { } static class ArgProcessor extends CompileArgProcessor { @@ -97,6 +97,7 @@ registerHandler(new ArgHandlerGenDir(options)); registerHandler(new ArgHandlerScriptStyle(options)); registerHandler(new ArgHandlerEnableAssertions(options)); + registerHandler(new ArgHandlerShardPrecompile(options)); registerHandler(new ArgHandlerDisableAggressiveOptimization(options)); registerHandler(new ArgHandlerDisableClassMetadata(options)); registerHandler(new ArgHandlerDisableCastChecking(options)); @@ -118,9 +119,10 @@ } static class PrecompileOptionsImpl extends CompileTaskOptionsImpl implements - PrecompileOptions { + PrecompileOptions, Serializable { private boolean disableUpdateCheck; private File dumpFile; + private boolean enableGeneratingOnShards; private File genDir; private final JJSOptionsImpl jjsOptions = new JJSOptionsImpl(); private int maxPermsPerPrecompile; @@ -143,6 +145,7 @@ setGenDir(other.getGenDir()); setMaxPermsPerPrecompile(other.getMaxPermsPerPrecompile()); setValidateOnly(other.isValidateOnly()); + setEnabledGeneratingOnShards(other.isEnabledGeneratingOnShards()); } public File getDumpSignatureFile() { @@ -184,6 +187,10 @@ public boolean isEnableAssertions() { return jjsOptions.isEnableAssertions(); } + + public boolean isEnabledGeneratingOnShards() { + return enableGeneratingOnShards; + } public boolean isOptimizePrecompile() { return jjsOptions.isOptimizePrecompile(); @@ -240,6 +247,10 @@ public void setEnableAssertions(boolean enableAssertions) { jjsOptions.setEnableAssertions(enableAssertions); } + + public void setEnabledGeneratingOnShards(boolean enabled) { + enableGeneratingOnShards = enabled; + } public void setGenDir(File genDir) { this.genDir = genDir; @@ -460,30 +471,8 @@ } } - private static AbstractCompiler getCompiler(ModuleDef module) { - ConfigurationProperty compilerClassProp = module.getProperties().createConfiguration( - "x.compiler.class", false); - String compilerClassName = compilerClassProp.getValue(); - if (compilerClassName == null || compilerClassName.length() == 0) { - return new JavaScriptCompiler(); - } - Throwable caught; - try { - Class<?> compilerClass = Class.forName(compilerClassName); - return (AbstractCompiler) compilerClass.newInstance(); - } catch (ClassNotFoundException e) { - caught = e; - } catch (InstantiationException e) { - caught = e; - } catch (IllegalAccessException e) { - caught = e; - } - throw new RuntimeException("Unable to instantiate compiler class '" - + compilerClassName + "'", caught); - } - - private static Precompilation precompile(TreeLogger logger, - JJSOptions jjsOptions, ModuleDef module, int permutationBase, + static Precompilation precompile(TreeLogger logger, JJSOptions jjsOptions, + ModuleDef module, int permutationBase, PropertyPermutations allPermutations, File genDir, File generatorResourcesDir, File dumpSignatureFile) { @@ -506,9 +495,8 @@ module, compilationState, generatedArtifacts, allPermutations, genDir, generatorResourcesDir); PerfLogger.start("Precompile"); - UnifiedAst unifiedAst = getCompiler(module).precompile(logger, - module, rpo, declEntryPts, null, jjsOptions, - rpo.getPermuationCount() == 1); + UnifiedAst unifiedAst = getCompiler(module).precompile(logger, module, + rpo, declEntryPts, null, jjsOptions, rpo.getPermuationCount() == 1); PerfLogger.end(); // Merge all identical permutations together. @@ -537,6 +525,28 @@ return null; } } + + private static AbstractCompiler getCompiler(ModuleDef module) { + ConfigurationProperty compilerClassProp = module.getProperties().createConfiguration( + "x.compiler.class", false); + String compilerClassName = compilerClassProp.getValue(); + if (compilerClassName == null || compilerClassName.length() == 0) { + return new JavaScriptCompiler(); + } + Throwable caught; + try { + Class<?> compilerClass = Class.forName(compilerClassName); + return (AbstractCompiler) compilerClass.newInstance(); + } catch (ClassNotFoundException e) { + caught = e; + } catch (InstantiationException e) { + caught = e; + } catch (IllegalAccessException e) { + caught = e; + } + throw new RuntimeException("Unable to instantiate compiler class '" + + compilerClassName + "'", caught); + } private final PrecompileOptionsImpl options; @@ -545,7 +555,6 @@ } public boolean run(TreeLogger logger) throws UnableToCompleteException { - boolean originalCompilationStateRetained = options.isCompilationStateRetained(); // Avoid early optimizations since permutation compiles will run separately. options.setOptimizePrecompile(false); @@ -555,116 +564,76 @@ // No need to check mkdirs result because an IOException will occur anyway compilerWorkDir.mkdirs(); - JarOutputStream precompilationJar; - try { - precompilationJar = new JarOutputStream(new FileOutputStream(new File( - compilerWorkDir, PRECOMPILE_FILENAME))); - } catch (IOException e) { - logger.log(TreeLogger.ERROR, "Could not create " + PRECOMPILE_FILENAME, - e); - return false; - } + File precompilationFile = new File(compilerWorkDir, PRECOMPILE_FILENAME); ModuleDef module = ModuleDefLoader.loadFromClassPath(logger, moduleName); - // TODO: All JDT checks now before even building TypeOracle? - module.getCompilationState(logger); - + boolean generateOnShards = options.isEnabledGeneratingOnShards(); if (options.isValidateOnly()) { - TreeLogger branch = logger.branch(TreeLogger.INFO, - "Validating compilation " + module.getName()); - if (!validate(branch, options, module, options.getGenDir(), - compilerWorkDir, options.getDumpSignatureFile())) { - branch.log(TreeLogger.ERROR, "Validation failed"); - return false; - } - branch.log(TreeLogger.INFO, "Validation succeeded"); - } else { - TreeLogger branch = logger.branch(TreeLogger.INFO, - "Precompiling module " + module.getName()); - PropertyPermutations allPermutations = new PropertyPermutations( - module.getProperties()); - int potentialPermutations = allPermutations.size(); - int permutationsPerIteration = options.getMaxPermsPerPrecompile(); - - if (permutationsPerIteration <= 0) { - permutationsPerIteration = potentialPermutations; - } + // Don't bother running on shards for just a validation run + generateOnShards = false; + } else if (options.getDumpSignatureFile() != null) { + logger.log(TreeLogger.INFO, + "Precompiling on the start node, because a dump signature file was specified"); + /* + * It would be possible to shard in this case, too. However, each + * permutation would have its own signatures dumped. Either the output + * would need to be multiple dump files, or those files would need to be + * combined back into one. + */ + generateOnShards = false; + } + + if (generateOnShards) { /* - * The potential number of permutations to precompile >= the actual - * number of permutations that end up being precompiled, because some of - * the permutations might collapse due to identical rebind results. So - * we have to track these two counts and ids separately. + * Pre-precompile. Count the permutations and plan to do a real + * precompile in the CompilePerms shards. */ - int actualPermutations = 0; - for (int potentialFirstPerm = 0; potentialFirstPerm < potentialPermutations; potentialFirstPerm += permutationsPerIteration) { - int numPermsToPrecompile = Math.min(potentialPermutations - - potentialFirstPerm, permutationsPerIteration); - - /* - * After the first precompile, we need to forcibly refresh - * CompilationState to clear out generated units from previous - * precompilations. - */ - if (potentialFirstPerm != 0) { - module.getCompilationState(branch).refresh(branch); - } - - if (potentialFirstPerm + numPermsToPrecompile < potentialPermutations) { - /* - * On all iterations but the last, force retainCompilationState to - * be true. Otherwise, state will be discarded that is needed on - * later iterations. - */ - options.setCompilationStateRetained(true); - } else { - // On the last iteration, use whatever the original setting was - options.setCompilationStateRetained(originalCompilationStateRetained); - } - - // Select only the range of property permutations that we want - PropertyPermutations localPermutations = new PropertyPermutations( - allPermutations, potentialFirstPerm, numPermsToPrecompile); + TreeLogger branch = logger.branch(TreeLogger.INFO, + "Precompiling (minimal) module " + module.getName()); + Util.writeObjectAsFile(logger, precompilationFile, options); + int numPermutations = module.getProperties().numPermutations(); + Util.writeStringAsFile(logger, new File(compilerWorkDir, + PERM_COUNT_FILENAME), String.valueOf(numPermutations)); + branch.log(TreeLogger.INFO, + "Precompilation (minimal) succeeded, number of permutations: " + + numPermutations); + } else { + // TODO: All JDT checks now before even building TypeOracle? + module.getCompilationState(logger); + + if (options.isValidateOnly()) { + TreeLogger branch = logger.branch(TreeLogger.INFO, + "Validating compilation " + module.getName()); + if (!validate(branch, options, module, options.getGenDir(), + compilerWorkDir, options.getDumpSignatureFile())) { + branch.log(TreeLogger.ERROR, "Validation failed"); + return false; + } + branch.log(TreeLogger.INFO, "Validation succeeded"); + } else { + TreeLogger branch = logger.branch(TreeLogger.INFO, + "Precompiling module " + module.getName()); Precompilation precompilation = precompile(branch, options, module, - actualPermutations, localPermutations, options.getGenDir(), - compilerWorkDir, options.getDumpSignatureFile()); + options.getGenDir(), compilerWorkDir, + options.getDumpSignatureFile()); if (precompilation == null) { branch.log(TreeLogger.ERROR, "Precompilation failed"); return false; } - int actualNumPermsPrecompiled = precompilation.getPermutations().length; - String precompilationFilename = PrecompilationFile.fileNameForPermutations( - actualPermutations, actualNumPermsPrecompiled); - try { - precompilationJar.putNextEntry(new ZipEntry(precompilationFilename)); - Util.writeObjectToStream(precompilationJar, precompilation); - } catch (IOException e) { - branch.log(TreeLogger.ERROR, - "Failed to write a precompilation result", e); - return false; - } - - actualPermutations += actualNumPermsPrecompiled; - branch.log(TreeLogger.DEBUG, "Compiled " + actualNumPermsPrecompiled - + " permutations starting from " + potentialFirstPerm); - } - - try { - precompilationJar.close(); - } catch (IOException e) { - branch.log(TreeLogger.ERROR, "Failed to finalize " - + PRECOMPILE_FILENAME, e); - return false; - } - - Util.writeStringAsFile(branch, new File(compilerWorkDir, - PERM_COUNT_FILENAME), String.valueOf(actualPermutations)); - branch.log(TreeLogger.INFO, - "Precompilation succeeded, number of permutations: " - + actualPermutations); + Util.writeObjectAsFile(logger, precompilationFile, precompilation); + + int permsPrecompiled = precompilation.getPermutations().length; + Util.writeStringAsFile(logger, new File(compilerWorkDir, + PERM_COUNT_FILENAME), String.valueOf(permsPrecompiled)); + branch.log(TreeLogger.INFO, + "Precompilation succeeded, number of permutations: " + + permsPrecompiled); + } } } + return true; } } ======================================= --- /trunk/dev/core/src/com/google/gwt/dev/cfg/Properties.java Tue Oct 27 15:28:45 2009 +++ /trunk/dev/core/src/com/google/gwt/dev/cfg/Properties.java Tue Oct 27 23:15:10 2009 @@ -73,6 +73,15 @@ public Iterator<Property> iterator() { return map.values().iterator(); } + + /** + * Count the total number of permutations that this property set supports. + * This method can be expensive because it always recalculates the answer + * based on the current set of properties and values. + */ + public int numPermutations() { + return new PropertyPermutations(this).size(); + } private <T extends Property> T create(String name, boolean flag, boolean useFlagArgument, Class<T> clazz) { ======================================= --- /trunk/dev/core/src/com/google/gwt/dev/jjs/JavaToJavaScriptCompiler.java Tue Oct 27 19:12:04 2009 +++ /trunk/dev/core/src/com/google/gwt/dev/jjs/JavaToJavaScriptCompiler.java Tue Oct 27 23:15:10 2009 @@ -171,6 +171,10 @@ } this.statementRanges = statementRanges; } + + public void addArtifacts(Collection<? extends Artifact<?>> newArtifacts) { + this.artifacts.addAll(newArtifacts); + } public ArtifactSet getArtifacts() { return artifacts; @@ -356,9 +360,8 @@ PermutationResult toReturn = new PermutationResultImpl(js, permutation, makeSymbolMap(symbolTable), ranges); - toReturn.getArtifacts().addAll( - makeSoycArtifacts(logger, permutationId, jprogram, js, - sizeBreakdowns, sourceInfoMaps, dependencies, map, obfuscateMap)); + toReturn.addArtifacts(makeSoycArtifacts(logger, permutationId, jprogram, + js, sizeBreakdowns, sourceInfoMaps, dependencies, map, obfuscateMap)); logger.log(TreeLogger.TRACE, "Permutation took " + (System.currentTimeMillis() - permStart) + " ms"); ======================================= --- /trunk/dev/core/src/com/google/gwt/dev/jjs/PermutationResult.java Tue Oct 27 19:12:04 2009 +++ /trunk/dev/core/src/com/google/gwt/dev/jjs/PermutationResult.java Tue Oct 27 23:15:10 2009 @@ -15,17 +15,24 @@ */ package com.google.gwt.dev.jjs; +import com.google.gwt.core.ext.linker.Artifact; import com.google.gwt.core.ext.linker.ArtifactSet; import com.google.gwt.core.ext.linker.StatementRanges; import com.google.gwt.dev.Permutation; import java.io.Serializable; +import java.util.Collection; /** * An extensible return type for the results of compiling a single permutation. */ public interface PermutationResult extends Serializable { + /** + * Adds additional artifacts to this permutation result. + */ + void addArtifacts(Collection<? extends Artifact<?>> newArtifacts); + /** * Returns any Artifacts that may have been created as a result of compiling * the permutation. --~--~---------~--~----~------------~-------~--~----~ http://groups.google.com/group/Google-Web-Toolkit-Contributors -~----------~----~----~----~------~----~------~--~---
