Revision: 10224
Author: [email protected]
Date: Wed May 25 07:29:58 2011
Log: Implemented expression range shifting for IE block size
transformation. Moved expression range shifting for both transformations
inside the transformation classes. Modified constructors/instance variables
for the JsAbstractTextTransformer to support updating the expression ranges
in the SourceInfo map.
Review at http://gwt-code-reviews.appspot.com/1451801
Review by: [email protected]
http://code.google.com/p/google-web-toolkit/source/detail?r=10224
Modified:
/trunk/dev/core/src/com/google/gwt/dev/jjs/JavaToJavaScriptCompiler.java
/trunk/dev/core/src/com/google/gwt/dev/jjs/impl/JsAbstractTextTransformer.java
/trunk/dev/core/src/com/google/gwt/dev/jjs/impl/JsFunctionClusterer.java
/trunk/dev/core/src/com/google/gwt/dev/jjs/impl/JsIEBlockTextTransformer.java
/trunk/dev/core/src/com/google/gwt/dev/js/JsReportGenerationVisitor.java
/trunk/dev/core/src/com/google/gwt/dev/js/JsSourceGenerationVisitorWithSizeBreakdown.java
=======================================
---
/trunk/dev/core/src/com/google/gwt/dev/jjs/JavaToJavaScriptCompiler.java
Tue May 24 12:34:23 2011
+++
/trunk/dev/core/src/com/google/gwt/dev/jjs/JavaToJavaScriptCompiler.java
Wed May 25 07:29:58 2011
@@ -86,6 +86,7 @@
import com.google.gwt.dev.jjs.impl.HandleCrossFragmentReferences;
import com.google.gwt.dev.jjs.impl.ImplementClassLiteralsAsFields;
import com.google.gwt.dev.jjs.impl.JavaToJavaScriptMap;
+import com.google.gwt.dev.jjs.impl.JsAbstractTextTransformer;
import com.google.gwt.dev.jjs.impl.JsFunctionClusterer;
import com.google.gwt.dev.jjs.impl.JsIEBlockTextTransformer;
import com.google.gwt.dev.jjs.impl.JsoDevirtualizer;
@@ -1034,41 +1035,53 @@
for (int i = 0; i < js.length; i++) {
DefaultTextOutput out = new
DefaultTextOutput(options.getOutput().shouldMinimize());
JsSourceGenerationVisitorWithSizeBreakdown v;
+
if (sourceInfoMaps != null) {
v = new JsReportGenerationVisitor(out, jjsMap);
} else {
v = new JsSourceGenerationVisitorWithSizeBreakdown(out, jjsMap);
}
v.accept(jsProgram.getFragmentBlock(i));
-
+
+ StatementRanges statementRanges = v.getStatementRanges();
+ String code = out.toString();
+ Map<Range, SourceInfo> infoMap = (sourceInfoMaps != null) ?
v.getSourceInfoMap() : null;
+
+ JsAbstractTextTransformer transformer =
+ new JsAbstractTextTransformer(code, statementRanges, infoMap) {
+ @Override public void exec() { }
+ @Override protected void updateSourceInfoMap() { }
+ };
+
/**
* Reorder function decls to improve compression ratios. Also
restructures
* the top level blocks into sub-blocks if they exceed 32767
statements.
*/
Event functionClusterEvent =
SpeedTracerLogger.start(CompilerEventType.FUNCTION_CLUSTER);
- JsFunctionClusterer clusterer =
- new JsFunctionClusterer(out.toString(), v.getStatementRanges());
// only cluster for obfuscated mode
if (options.isAggressivelyOptimize() && options.getOutput() ==
JsOutputOption.OBFUSCATED) {
- clusterer.exec();
+ transformer = new JsFunctionClusterer(transformer);
+ transformer.exec();
}
functionClusterEvent.end();
+
// rewrite top-level blocks to limit the number of statements
- JsIEBlockTextTransformer ieXformer = new
JsIEBlockTextTransformer(clusterer);
if (splitBlocks) {
- ieXformer.exec();
- }
- js[i] = ieXformer.getJs();
+ transformer = new JsIEBlockTextTransformer(transformer);
+ transformer.exec();
+ }
+
+ js[i] = transformer.getJs();
+ ranges[i] = transformer.getStatementRanges();
if (sizeBreakdowns != null) {
sizeBreakdowns[i] = v.getSizeBreakdown();
}
if (sourceInfoMaps != null) {
- sourceInfoMaps.add(((JsReportGenerationVisitor)
v).getSourceInfoMap());
- }
- ranges[i] = ieXformer.getStatementRanges();
+ sourceInfoMaps.add(transformer.getSourceInfoMap());
+ }
}
}
-
+
/**
* This method can be used to fetch the list of referenced classs.
*
=======================================
---
/trunk/dev/core/src/com/google/gwt/dev/jjs/impl/JsAbstractTextTransformer.java
Tue Apr 19 10:10:18 2011
+++
/trunk/dev/core/src/com/google/gwt/dev/jjs/impl/JsAbstractTextTransformer.java
Wed May 25 07:29:58 2011
@@ -17,8 +17,11 @@
import com.google.gwt.core.ext.linker.StatementRanges;
import com.google.gwt.core.ext.linker.impl.StandardStatementRanges;
+import com.google.gwt.core.ext.soyc.Range;
+import com.google.gwt.dev.jjs.SourceInfo;
import java.util.ArrayList;
+import java.util.Map;
/**
* Base class for transforming program text.
@@ -26,17 +29,23 @@
public abstract class JsAbstractTextTransformer {
protected String js;
+
+ protected StatementRanges originalStatementRanges;
protected StatementRanges statementRanges;
+
+ protected Map<Range, SourceInfo> sourceInfoMap;
public JsAbstractTextTransformer(JsAbstractTextTransformer xformer) {
- this.js = xformer.getJs();
- this.statementRanges = xformer.getStatementRanges();
+ this(xformer.getJs(), xformer.getStatementRanges(),
xformer.getSourceInfoMap());
}
- public JsAbstractTextTransformer(String js, StatementRanges
statementRanges) {
+ public JsAbstractTextTransformer(String js, StatementRanges
statementRanges,
+ Map<Range, SourceInfo> sourceInfoMap) {
this.js = js;
this.statementRanges = statementRanges;
+ this.originalStatementRanges = statementRanges;
+ this.sourceInfoMap = sourceInfoMap;
}
public abstract void exec();
@@ -44,23 +53,26 @@
public String getJs() {
return js;
}
+
+ public Map<Range, SourceInfo> getSourceInfoMap() {
+ return sourceInfoMap;
+ }
public StatementRanges getStatementRanges() {
return statementRanges;
}
- protected void addStatement(String code, StringBuilder newJs,
ArrayList<Integer> starts,
- ArrayList<Integer> ends) {
- beginStatement(newJs, starts);
+ protected void addStatement(int index, String code, StringBuilder newJs,
+ ArrayList<Integer> starts, ArrayList<Integer> ends) {
+ beginStatement(index, newJs, starts);
newJs.append(code);
- endStatement(newJs, ends);
+ endStatement(index, newJs, ends);
}
- protected void beginStatement(StringBuilder newJs, ArrayList<Integer>
starts) {
+ protected void beginStatement(int index, StringBuilder newJs,
ArrayList<Integer> starts) {
starts.add(newJs.length());
}
- // FIXME document parameters
/**
* Called if any operations need to be performed before all statements
have
* been processed.
@@ -73,16 +85,10 @@
ArrayList<Integer> ends) {
}
- // FIXME document
- /**
- * @param newJs
- * @param ends
- */
- protected void endStatement(StringBuilder newJs, ArrayList<Integer>
ends) {
+ protected void endStatement(int index, StringBuilder newJs,
ArrayList<Integer> ends) {
ends.add(newJs.length());
}
- // FIXME document parameters
/**
* Called if any operations need to be performed after all statements
have
* been processed.
@@ -110,18 +116,26 @@
ArrayList<Integer> ends = new ArrayList<Integer>();
beginStatements(newJs, starts, ends);
- for (int stmtIndice : stmtIndices) {
- String code = getJsForRange(stmtIndice);
- addStatement(code, newJs, starts, ends);
+ for (int stmtIndex : stmtIndices) {
+ String code = getJsForRange(stmtIndex);
+ addStatement(stmtIndex, code, newJs, starts, ends);
}
endStatements(newJs, starts, ends);
assert starts.size() == ends.size() : "Size mismatch between start and"
+ " end statement ranges.";
- assert starts.get(0) == 0 && ends.get(ends.size() - 1) ==
newJs.length() : "statement ranges don't cover entire JS output string.";
-
- StandardStatementRanges newRanges = new
StandardStatementRanges(starts, ends);
+ assert starts.get(0) == 0 && ends.get(ends.size() - 1) ==
newJs.length() :
+ "statement ranges don't cover entire JS output string.";
+
js = newJs.toString();
- statementRanges = newRanges;
- }
-}
+ statementRanges = new StandardStatementRanges(starts, ends);
+ updateSourceInfoMap();
+ }
+
+ /**
+ * Update the expression ranges in the SourceInfo map after the
+ * transformer has manipulated the statements.
+ */
+ protected abstract void updateSourceInfoMap();
+
+}
=======================================
---
/trunk/dev/core/src/com/google/gwt/dev/jjs/impl/JsFunctionClusterer.java
Tue May 24 12:34:23 2011
+++
/trunk/dev/core/src/com/google/gwt/dev/jjs/impl/JsFunctionClusterer.java
Wed May 25 07:29:58 2011
@@ -16,13 +16,18 @@
package com.google.gwt.dev.jjs.impl;
import com.google.gwt.core.ext.linker.StatementRanges;
+import com.google.gwt.core.ext.soyc.Range;
+import com.google.gwt.dev.jjs.SourceInfo;
import com.google.gwt.dev.util.editdistance.GeneralEditDistance;
import com.google.gwt.dev.util.editdistance.GeneralEditDistances;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.LinkedList;
+import java.util.Map;
import java.util.regex.Pattern;
/**
@@ -60,19 +65,32 @@
private static boolean isFunctionDeclaration(String code) {
return functionDeclarationPattern.matcher(code).lookingAt();
}
+
+ /**
+ * Number of function declarations found.
+ */
+ private int numFunctions;
+
+ /**
+ * The statement indices after clustering. The element at index j
represents
+ * the index of the statement in the original code that is moved to
index j
+ * in the new code after clustering.
+ */
+ private int[] reorderedIndices;
public JsFunctionClusterer(JsAbstractTextTransformer xformer) {
super(xformer);
}
- public JsFunctionClusterer(String js, StatementRanges statementRanges) {
- super(js, statementRanges);
+ public JsFunctionClusterer(String js, StatementRanges statementRanges,
+ Map<Range, SourceInfo> sourceInfoMap) {
+ super(js, statementRanges, sourceInfoMap);
}
@Override
public void exec() {
LinkedList<Integer> functionIndices = new LinkedList<Integer>();
-
+
// gather up all of the indices of function decl statements
for (int i = 0; i < statementRanges.numStatements(); i++) {
String code = getJsForRange(i);
@@ -80,6 +98,8 @@
functionIndices.add(i);
}
}
+
+ numFunctions = functionIndices.size();
// sort the indices according to size of statement range
Collections.sort(functionIndices, new Comparator<Integer>() {
@@ -125,21 +145,93 @@
functionIndices.remove(bestIndex);
}
+ reorderedIndices = Arrays.copyOf(clusteredIndices,
statementRanges.numStatements());
recomputeJsAndStatementRanges(clusteredIndices);
}
+ /**
+ * Returns the array of reordered statement indices after clustering.
+ * @return The array of indices, where the element at index j represents
+ * the index of the statement in the original code that is moved to
index j
+ * in the new code after clustering.
+ */
+ public int[] getReorderedIndices() {
+ return reorderedIndices;
+ }
@Override
protected void endStatements(StringBuilder newJs, ArrayList<Integer>
starts,
ArrayList<Integer> ends) {
+ int j = numFunctions;
// Then output everything else that is not a function.
for (int i = 0; i < statementRanges.numStatements(); i++) {
String code = getJsForRange(i);
if (!isFunctionDeclaration(code)) {
- addStatement(code, newJs, starts, ends);
+ addStatement(j, code, newJs, starts, ends);
+ reorderedIndices[j] = i;
+ j++;
}
}
super.endStatements(newJs, starts, ends);
}
+
+ /**
+ * Fixes the index ranges of individual expressions in the generated
+ * JS after function clustering has reordered statements. Loops over
+ * each expression, determines which statement it falls in, and shifts
+ * the indices according to where that statement moved.
+ */
+ @Override
+ protected void updateSourceInfoMap() {
+ if (sourceInfoMap != null) {
+ // create mapping of statement ranges
+ Map<Range, Range> statementShifts = new HashMap<Range, Range>();
+ for (int j = 0; j < statementRanges.numStatements(); j++) {
+ int permutedStart = statementRanges.start(j);
+ int permutedEnd = statementRanges.end(j);
+ int originalStart =
originalStatementRanges.start(reorderedIndices[j]);
+ int originalEnd = originalStatementRanges.end(reorderedIndices[j]);
+
+ statementShifts.put(new Range(originalStart, originalEnd),
+ new Range(permutedStart, permutedEnd));
+ }
+
+ Range[] oldStatementRanges = statementShifts.keySet().toArray(new
Range[0]);
+ Arrays.sort(oldStatementRanges, Range.SOURCE_ORDER_COMPARATOR);
+
+ Range[] oldExpressionRanges = sourceInfoMap.keySet().toArray(new
Range[0]);
+ Arrays.sort(oldExpressionRanges, Range.SOURCE_ORDER_COMPARATOR);
+
+
+ // iterate over expression ranges and shift
+ Map<Range, SourceInfo> updatedInfoMap = new HashMap<Range,
SourceInfo>();
+ Range entireProgram =
+ new Range(0, oldStatementRanges[oldStatementRanges.length -
1].getEnd());
+ for (int i = 0, j = 0; j < oldExpressionRanges.length; j++) {
+ Range oldExpression = oldExpressionRanges[j];
+ if (oldExpression.equals(entireProgram)) {
+ updatedInfoMap.put(oldExpression,
sourceInfoMap.get(oldExpression));
+ continue;
+ }
+
+ if (!oldStatementRanges[i].contains(oldExpressionRanges[j])) {
+ // expression should fall in the next statement
+ i++;
+ assert oldStatementRanges[i].contains(oldExpressionRanges[j]);
+ }
+
+ Range oldStatement = oldStatementRanges[i];
+ Range newStatement = statementShifts.get(oldStatement);
+ int shift = newStatement.getStart() - oldStatement.getStart();
+
+ Range oldExpressionRange = oldExpressionRanges[j];
+ Range newExpressionRange = new Range(oldExpressionRange.getStart()
+ shift,
+ oldExpressionRange.getEnd() + shift);
+ updatedInfoMap.put(newExpressionRange,
sourceInfoMap.get(oldExpressionRange));
+ }
+
+ sourceInfoMap = updatedInfoMap;
+ }
+ }
private int stmtSize(int index1) {
return statementRanges.end(index1) - statementRanges.start(index1);
=======================================
---
/trunk/dev/core/src/com/google/gwt/dev/jjs/impl/JsIEBlockTextTransformer.java
Tue Apr 19 10:10:18 2011
+++
/trunk/dev/core/src/com/google/gwt/dev/jjs/impl/JsIEBlockTextTransformer.java
Wed May 25 07:29:58 2011
@@ -16,8 +16,15 @@
package com.google.gwt.dev.jjs.impl;
import com.google.gwt.core.ext.linker.StatementRanges;
+import com.google.gwt.core.ext.soyc.Range;
+import com.google.gwt.dev.jjs.SourceInfo;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
/**
* Limits top-level blocks to MAX_BLOCK_SIZE statements.
@@ -32,13 +39,18 @@
private int currentStatementCount;
private boolean doSplits;
+
+ private Set<Integer> statementsAddedBlockClose = new HashSet<Integer>();
+
+ private Set<Integer> statementsAddedBlockOpen = new HashSet<Integer>();
public JsIEBlockTextTransformer(JsAbstractTextTransformer xformer) {
super(xformer);
}
- public JsIEBlockTextTransformer(String js, StatementRanges
statementRanges) {
- super(js, statementRanges);
+ public JsIEBlockTextTransformer(String js, StatementRanges
statementRanges,
+ Map<Range, SourceInfo> sourceInfoMap) {
+ super(js, statementRanges, sourceInfoMap);
}
/**
@@ -55,17 +67,26 @@
recomputeJsAndStatementRanges(statementIndices);
}
}
+
+ public Set<Integer> getStatementsAddedBlockClose() {
+ return statementsAddedBlockClose;
+ }
+
+ public Set<Integer> getStatementsAddedBlockOpen() {
+ return statementsAddedBlockOpen;
+ }
/**
* Record start of statement, and optionally inject new open block.
*/
@Override
- protected void beginStatement(StringBuilder newJs, ArrayList<Integer>
starts) {
+ protected void beginStatement(int index, StringBuilder newJs,
ArrayList<Integer> starts) {
if (doSplits && currentStatementCount == 0) {
- super.beginStatement(newJs, starts);
+ super.beginStatement(index, newJs, starts);
newJs.append('{');
+ statementsAddedBlockOpen.add(Integer.valueOf(index));
} else if (!doSplits) {
- super.beginStatement(newJs, starts);
+ super.beginStatement(index, newJs, starts);
}
}
@@ -81,14 +102,15 @@
* full.
*/
@Override
- protected void endStatement(StringBuilder newJs, ArrayList<Integer>
ends) {
+ protected void endStatement(int index, StringBuilder newJs,
ArrayList<Integer> ends) {
currentStatementCount++;
if (doSplits && currentStatementCount == MAX_BLOCK_SIZE) {
newJs.append('}');
- super.endStatement(newJs, ends);
+ super.endStatement(index, newJs, ends);
currentStatementCount = 0;
+ statementsAddedBlockClose.add(Integer.valueOf(index));
} else if (!doSplits) {
- super.endStatement(newJs, ends);
+ super.endStatement(index, newJs, ends);
}
}
@@ -101,14 +123,91 @@
optionallyCloseLastBlock(newJs, ends);
super.endStatements(newJs, starts, ends);
}
+
+ /**
+ * Fixes the index ranges of individual expressions in the generated
+ * JS after chunking statements into blocks that satisfy the IE block
+ * size problem. Loops over each expression, determines whether the
+ * statement in which it falls has a brace inserted before/after, and
+ * shifts forward according to where it falls in the block.
+ */
+ @Override
+ protected void updateSourceInfoMap() {
+ if (sourceInfoMap != null) {
+ Range[] oldExpressionRanges = sourceInfoMap.keySet().toArray(new
Range[0]);
+ Arrays.sort(oldExpressionRanges, Range.SOURCE_ORDER_COMPARATOR);
+
+ // iterate over expression ranges and shift
+ Map<Range, SourceInfo> updatedInfoMap = new HashMap<Range,
SourceInfo>();
+ Range entireProgram =
+ new Range(0,
originalStatementRanges.end(originalStatementRanges.numStatements() - 1));
+ int shift = 0;
+
+ // set to keep track of which statements have already shifted.
+ // need to account for when a shift has already been added for the
extra
+ // open brace in a statement--sometimes there are multiple
expressions
+ // that all start at the same place a the beginning of a statement in
+ // the expression list
+ // ex: _.gC=function x()... yields the expressions _, _.gC, _.gC
= ...
+ Set<Integer> shiftAdded = new HashSet<Integer>();
+
+ for (int i = 0, j = 0; j < oldExpressionRanges.length; j++) {
+ Range oldExpression = oldExpressionRanges[j];
+ if (oldExpression.equals(entireProgram)) {
+ continue;
+ }
+
+ if (originalStatementRanges.start(i) > oldExpression.getStart()
+ || oldExpression.getEnd() > originalStatementRanges.end(i)) {
+
+ // expression should fall in the next statement
+ i++;
+ assert originalStatementRanges.start(i) <=
oldExpression.getStart()
+ && oldExpression.getEnd() <= originalStatementRanges.end(i);
+
+ if (statementsAddedBlockClose.contains(Integer.valueOf(i - 1))) {
+ // there's an extra statement index in the addedBlockClose
list,
+ // which corresponds to the extra closing brace at the end of
the
+ // program. but this index doesn't match up to the indices in
the
+ // old statement ranges--it's equal to the # of statements in
the
+ // original code divided by the IE block size
+ if (i != statementRanges.numStatements()) {
+ shift++;
+ }
+ }
+ }
+
+ if (statementsAddedBlockOpen.contains(Integer.valueOf(i))
+ && oldExpression.getStart() == originalStatementRanges.start(i)
+ && !shiftAdded.contains(i)) {
+
+ shift++;
+ shiftAdded.add(Integer.valueOf(i));
+ }
+
+ int newStart = oldExpression.getStart() + shift;
+ int newEnd = oldExpression.getEnd() + shift;
+
+ Range newExpression = new Range(newStart, newEnd);
+ updatedInfoMap.put(newExpression,
sourceInfoMap.get(oldExpression));
+ }
+
+ updatedInfoMap.put(new Range(0, entireProgram.getEnd() + shift),
+ sourceInfoMap.get(entireProgram));
+
+ sourceInfoMap = updatedInfoMap;
+ }
+ }
/**
* Close last block if it never filled.
*/
private void optionallyCloseLastBlock(StringBuilder newJs,
ArrayList<Integer> ends) {
- if (doSplits && currentStatementCount > 1 && currentStatementCount <
MAX_BLOCK_SIZE) {
+ if (doSplits && currentStatementCount > 0 && currentStatementCount <
MAX_BLOCK_SIZE) {
newJs.append("}");
ends.add(newJs.length());
+ statementsAddedBlockClose.add(Integer.valueOf(ends.size() - 1));
}
}
-}
+
+}
=======================================
---
/trunk/dev/core/src/com/google/gwt/dev/js/JsReportGenerationVisitor.java
Wed Feb 2 13:13:58 2011
+++
/trunk/dev/core/src/com/google/gwt/dev/js/JsReportGenerationVisitor.java
Wed May 25 07:29:58 2011
@@ -41,6 +41,7 @@
this.out = out;
}
+ @Override
public Map<Range, SourceInfo> getSourceInfoMap() {
return Collections.unmodifiableMap(sourceInfoMap);
}
=======================================
---
/trunk/dev/core/src/com/google/gwt/dev/js/JsSourceGenerationVisitorWithSizeBreakdown.java
Wed Feb 2 13:13:58 2011
+++
/trunk/dev/core/src/com/google/gwt/dev/js/JsSourceGenerationVisitorWithSizeBreakdown.java
Wed May 25 07:29:58 2011
@@ -15,6 +15,8 @@
*/
package com.google.gwt.dev.js;
+import com.google.gwt.core.ext.soyc.Range;
+import com.google.gwt.dev.jjs.SourceInfo;
import com.google.gwt.dev.jjs.ast.JClassType;
import com.google.gwt.dev.jjs.ast.JMethod;
import com.google.gwt.dev.jjs.impl.FragmentExtractor;
@@ -55,6 +57,11 @@
public SizeBreakdown getSizeBreakdown() {
return new SizeBreakdown(out.getPosition(), sizeMap);
}
+
+ public Map<Range, SourceInfo> getSourceInfoMap() {
+ // override if your child class creates sourceinfo
+ return null;
+ }
@Override
public boolean visit(JsBlock x, JsContext ctx) {
--
http://groups.google.com/group/Google-Web-Toolkit-Contributors
