Revision: 4678
Author: [email protected]
Date: Mon Dec 12 16:56:03 2011
Log: more cajoler speedups
http://codereview.appspot.com/5486048
For my current testcase, this is about a 10% speedup to the cajoler.
- The rewriter relies extensively on a marker "synthetic", which
is stored as a key-value pair in a HashMap. The map lookups
have a noticeable cost.
This change adds a boolean "synthetic" to AbstractParseTreeNode
and uses that instead of the HashMap. This is the largest
speedup in this change.
- Rewriter#expand will pre-filter rules in a ruleset based on
specific nodetype. So, it knows that when it's examining a
Declaration node, it doesn't have to try running a rule that
can only match AssignOperation. It determines a rule's
matchable nodetype by parsing the rule's "matches" attribute.
Some rules do not have a parseable "matches", because the match
condition is not expressible with quasi syntax, so these rules
are always checked against every nodetype, even though they
might apply to just a specific nodetype.
This change adds another attribute "matchNode", which lets us
specify a nodetype that the rule must match.
- The nodetype SpecialOperation represents several semantically
different javascript operators, and this interferes with the
rule filtering in Rewriter#expand. For instance, a rule that
matches "@v.@p" is a SpecialOperation nodetype, which means the
rule is also tried against the "delete" operator, the comma
operator, etc.
This change adds another type of rule pre-filter that looks at
the specific operator being matched.
[email protected]
http://code.google.com/p/google-caja/source/detail?r=4678
Added:
/trunk/src/com/google/caja/parser/quasiliteral/NodeTypeFilter.java
/trunk/src/com/google/caja/parser/quasiliteral/QuasiCache.java
/trunk/src/com/google/caja/parser/quasiliteral/RuleFilter.java
/trunk/src/com/google/caja/parser/quasiliteral/SpecialOpFilter.java
Modified:
/trunk/build.xml
/trunk/src/com/google/caja/lang/css/Partitions.java
/trunk/src/com/google/caja/parser/AbstractParseTreeNode.java
/trunk/src/com/google/caja/parser/js/Operator.java
/trunk/src/com/google/caja/parser/js/ReturnStmt.java
/trunk/src/com/google/caja/parser/js/SyntheticNodes.java
/trunk/src/com/google/caja/parser/quasiliteral/ES53Rewriter.java
/trunk/src/com/google/caja/parser/quasiliteral/IllegalReferenceCheckRewriter.java
/trunk/src/com/google/caja/parser/quasiliteral/NonAsciiCheckVisitor.java
/trunk/src/com/google/caja/parser/quasiliteral/QuasiBuilder.java
/trunk/src/com/google/caja/parser/quasiliteral/Rewriter.java
/trunk/src/com/google/caja/parser/quasiliteral/Rule.java
/trunk/src/com/google/caja/parser/quasiliteral/RuleChain.java
/trunk/src/com/google/caja/parser/quasiliteral/RuleDescription.java
/trunk/src/com/google/caja/parser/quasiliteral/Scope.java
/trunk/src/com/google/caja/parser/quasiliteral/SimpleQuasiNode.java
/trunk/src/com/google/caja/parser/quasiliteral/SyntheticRuleSet.java
/trunk/src/com/google/caja/parser/quasiliteral/TrailingUnderscoresHole.java
/trunk/tests/com/google/caja/parser/ParseTreeNodeTest.java
=======================================
--- /dev/null
+++ /trunk/src/com/google/caja/parser/quasiliteral/NodeTypeFilter.java Mon
Dec 12 16:56:03 2011
@@ -0,0 +1,34 @@
+// Copyright (C) 2011 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.package
com.google.caja.parser.quasiliteral;
+
+package com.google.caja.parser.quasiliteral;
+
+import com.google.caja.parser.ParseTreeNode;
+
+public class NodeTypeFilter extends RuleFilter<Class<? extends
ParseTreeNode>> {
+
+ public NodeTypeFilter(RuleChain ruleChain) {
+ super(ruleChain);
+ }
+
+ @Override
+ Class<? extends ParseTreeNode> getKey(ParseTreeNode node) {
+ return QuasiBuilder.fuzzType(node.getClass());
+ }
+
+ @Override
+ boolean canMatch(Rule rule, Class<? extends ParseTreeNode> key) {
+ return rule.canMatch(key);
+ }
+}
=======================================
--- /dev/null
+++ /trunk/src/com/google/caja/parser/quasiliteral/QuasiCache.java Mon Dec
12 16:56:03 2011
@@ -0,0 +1,46 @@
+// Copyright (C) 2011 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.package
com.google.caja.parser.quasiliteral;
+
+package com.google.caja.parser.quasiliteral;
+
+import com.google.caja.lexer.ParseException;
+import com.google.caja.util.Maps;
+
+import java.util.Map;
+
+/**
+ * {@code QuasiCache} caches the quasi trees generated by parsing strings.
+ * This is a separate class rather than built into {@code QuasiBuilder}
+ * because QuasiBuilder returns mutable quasi trees.
+ *
+ * <p>{@code QuasiCache} assumes callers will not mutate the returned quasi
+ * trees, but does not enforce immutability.
+ */
+public class QuasiCache {
+ private static Map<String, QuasiNode> cache = Maps.newHashMap();
+
+ public static QuasiNode parse(String pattern) {
+ if (cache.containsKey(pattern)) {
+ return cache.get(pattern);
+ }
+ try {
+ QuasiNode q = QuasiBuilder.parseQuasiNode(pattern);
+ cache.put(pattern, q);
+ return q;
+ } catch (ParseException e) {
+ cache.put(pattern, null);
+ return null;
+ }
+ }
+}
=======================================
--- /dev/null
+++ /trunk/src/com/google/caja/parser/quasiliteral/RuleFilter.java Mon Dec
12 16:56:03 2011
@@ -0,0 +1,81 @@
+// Copyright (C) 2011 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.package
com.google.caja.parser.quasiliteral;
+
+package com.google.caja.parser.quasiliteral;
+
+import com.google.caja.parser.ParseTreeNode;
+import com.google.caja.util.Maps;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public abstract class RuleFilter<KeyT> {
+ private final RuleChain ruleChain;
+ private final Map<KeyT, List<Rule>> cache = Maps.newHashMap();
+
+ private static boolean debug = false;
+
+ public RuleFilter(RuleChain ruleChain) {
+ this.ruleChain = ruleChain;
+ }
+
+ public void reset() {
+ cache.clear();
+ }
+
+ public List<Rule> rulesFor(ParseTreeNode node) {
+ KeyT key = getKey(node);
+ if (key == null) {
+ return ruleChain.getAllRules();
+ }
+ List<Rule> someRules = cache.get(key);
+ if (someRules == null) {
+ someRules = computeRulesFor(node, key);
+ cache.put(key, someRules);
+ if (debug) { debugShowRules(someRules, node); }
+ }
+ return someRules;
+ }
+
+ private List<Rule> computeRulesFor(ParseTreeNode node, KeyT key) {
+ List<Rule> someRules = new ArrayList<Rule>();
+ for (Rule rule : ruleChain.getAllRules()) {
+ if (canMatch(rule, key)) {
+ someRules.add(rule);
+ }
+ }
+ return Collections.unmodifiableList(someRules);
+ }
+
+ private void debugShowRules(List<Rule> someRules, ParseTreeNode node) {
+ System.err.println(someRules.size() + " rules for " + node);
+ StringBuilder names = new StringBuilder(" ");
+ int ll = 2;
+ for (Rule rule: someRules) {
+ String name = rule.getName();
+ ll += 1 + name.length();
+ if (75 < ll) {
+ names.append("\n ");
+ ll = 2 + 1 + name.length();
+ }
+ names.append(" ").append(name);
+ }
+ System.err.println(names);
+ }
+
+ abstract KeyT getKey(ParseTreeNode node);
+ abstract boolean canMatch(Rule rule, KeyT key);
+}
=======================================
--- /dev/null
+++ /trunk/src/com/google/caja/parser/quasiliteral/SpecialOpFilter.java Mon
Dec 12 16:56:03 2011
@@ -0,0 +1,48 @@
+// Copyright (C) 2011 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.package
com.google.caja.parser.quasiliteral;
+
+package com.google.caja.parser.quasiliteral;
+
+import com.google.caja.parser.ParseTreeNode;
+import com.google.caja.parser.js.Operation;
+import com.google.caja.parser.js.Operator;
+
+public class SpecialOpFilter extends RuleFilter<Operator> {
+
+ public SpecialOpFilter (RuleChain ruleChain) {
+ super(ruleChain);
+ }
+
+ @Override
+ Operator getKey(ParseTreeNode node) {
+ return ((Operation) node).getOperator();
+ }
+
+ @Override
+ boolean canMatch(Rule rule, Operator op) {
+ return ruleOperator(rule) == op || rule.canMatch(Operation.class);
+ }
+
+ private Operator ruleOperator(Rule rule) {
+ QuasiNode qp = QuasiCache.parse(rule.getRuleDescription().matches());
+ if (qp != null && qp instanceof SimpleQuasiNode) {
+ SimpleQuasiNode sqp = (SimpleQuasiNode) qp;
+ if (Operation.class.isAssignableFrom(sqp.getMatchedClass())) {
+ return (Operator) sqp.getValue();
+ }
+ }
+ return null;
+ }
+
+}
=======================================
--- /trunk/build.xml Mon Dec 12 11:44:56 2011
+++ /trunk/build.xml Mon Dec 12 16:56:03 2011
@@ -1770,7 +1770,7 @@
classpathref="classpath.run" />
</target>
- <target name="precajole" depends="Precajoler">
+ <target name="precajole" depends="Precajoler" unless="precajole.skip">
<precajole to="${lib.caja}/precajole/data">
<source file="${3pp.jquery}/1.6.4/jquery-1.6.4.js">
http://ajax.googleapis.com/ajax/libs/jquery/1.6.4/jquery.js
=======================================
--- /trunk/src/com/google/caja/lang/css/Partitions.java Fri Nov 11 15:28:32
2011
+++ /trunk/src/com/google/caja/lang/css/Partitions.java Mon Dec 12 16:56:03
2011
@@ -14,6 +14,10 @@
package com.google.caja.lang.css;
+import com.google.caja.util.Bag;
+import com.google.caja.util.Lists;
+import com.google.caja.util.Maps;
+
import java.lang.reflect.Array;
import java.util.Arrays;
import java.util.BitSet;
@@ -22,10 +26,6 @@
import java.util.Map;
import java.util.Set;
-import com.google.caja.util.Bag;
-import com.google.caja.util.Lists;
-import com.google.caja.util.Maps;
-
class Partitions {
static class Partition<T> {
/**
@@ -171,7 +171,6 @@
for (int i = 0; i < nonZeroCols.count; ++i) { nonZeroCols.els[i] = i; }
ISlice[] matrix = new ISlice[nSets];
- int matrixSum = 0;
{
int i = 0;
BitSet rowBits = new BitSet(universe.length);
@@ -181,7 +180,6 @@
int index = elToIndex.get(el);
if (!rowBits.get(index)) {
rowBits.set(index);
- ++matrixSum;
}
}
ISlice row = new ISlice();
=======================================
--- /trunk/src/com/google/caja/parser/AbstractParseTreeNode.java Mon Nov 28
01:31:08 2011
+++ /trunk/src/com/google/caja/parser/AbstractParseTreeNode.java Mon Dec 12
16:56:03 2011
@@ -45,6 +45,7 @@
private List<Token<?>> comments = Collections.<Token<?>>emptyList();
private SyntheticAttributes attributes;
private boolean immutable = false;
+ private boolean synthetic = false;
@Override
public boolean makeImmutable() {
@@ -57,6 +58,15 @@
@Override
public boolean isImmutable() { return immutable; }
+
+ public boolean isSynthetic() { return synthetic; }
+
+ public void setSynthetic(boolean value) {
+ if (immutable) {
+ throw new UnsupportedOperationException();
+ }
+ synthetic = value;
+ }
/**
* The list of children. This can be appended to for efficient
initialization
@@ -419,6 +429,7 @@
if (attributes != null) {
cloned.attributes = new SyntheticAttributes(attributes);
}
+ cloned.synthetic = synthetic;
return cloned;
}
=======================================
--- /trunk/src/com/google/caja/parser/js/Operator.java Thu Dec 10 17:39:38
2009
+++ /trunk/src/com/google/caja/parser/js/Operator.java Mon Dec 12 16:56:03
2011
@@ -27,7 +27,6 @@
* @author [email protected]
*/
public enum Operator {
-
// from precedence tables at
// http://www.codehouse.com/javascript/precedence/
SQUARE_BRACKET(OperatorType.BRACKET, OperatorCategory.SPECIAL,
=======================================
--- /trunk/src/com/google/caja/parser/js/ReturnStmt.java Fri Nov 18
12:29:44 2011
+++ /trunk/src/com/google/caja/parser/js/ReturnStmt.java Mon Dec 12
16:56:03 2011
@@ -15,7 +15,6 @@
package com.google.caja.parser.js;
import com.google.caja.lexer.FilePosition;
-import com.google.caja.parser.ParseTreeNode;
import com.google.caja.reporting.RenderContext;
import com.google.javascript.jscomp.jsonml.JsonML;
import com.google.javascript.jscomp.jsonml.TagType;
=======================================
--- /trunk/src/com/google/caja/parser/js/SyntheticNodes.java Thu Jul 22
10:23:41 2010
+++ /trunk/src/com/google/caja/parser/js/SyntheticNodes.java Mon Dec 12
16:56:03 2011
@@ -14,8 +14,8 @@
package com.google.caja.parser.js;
+import com.google.caja.parser.AbstractParseTreeNode;
import com.google.caja.parser.ParseTreeNode;
-import com.google.caja.util.SyntheticAttributeKey;
// TODO(erights,ihab): Remove the concept of "synthetic" entirely.
/**
@@ -34,37 +34,36 @@
* @author [email protected]
*/
public final class SyntheticNodes {
- /**
- * Indicates that a node breaks the caja rules in some way, but that this
- * is OK since it comes from a privileged source, not user code.
- */
- public static final SyntheticAttributeKey<Boolean> SYNTHETIC
- = new SyntheticAttributeKey<Boolean>(Boolean.class, "synthetic");
public static boolean isSynthesizable(ParseTreeNode node) {
return (node instanceof Identifier && node.getValue() != null)
|| node instanceof FunctionConstructor
|| node instanceof UncajoledModule;
}
+
+ public static boolean is(ParseTreeNode t) {
+ return t instanceof AbstractParseTreeNode
+ && ((AbstractParseTreeNode) t).isSynthetic();
+ }
/**
* A convenience function used to mark {@link #isSynthesizable
synthesizable}
* nodes created during source->javascript translation as
- * {@link #SYNTHETIC synthetic}. Nodes corresponsing to javascript
+ * {@link #SYNTHETIC synthetic}. Nodes corresponding to javascript
* embedded in the original will not be synthetic.
* <p>
* This is meant to be imported statically.
*/
public static <T extends ParseTreeNode> T s(T t) {
if (isSynthesizable(t)) {
- t.getAttributes().set(SYNTHETIC, Boolean.TRUE);
+ ((AbstractParseTreeNode) t).setSynthetic(true);
}
return t;
}
/** A synthetic identifier may occupy the <code>*__</code> namespace. */
public static Identifier s(Identifier t) {
- t.getAttributes().set(SYNTHETIC, Boolean.TRUE);
+ t.setSynthetic(true);
return t;
}
@@ -74,7 +73,12 @@
* function.
*/
public static FunctionConstructor s(FunctionConstructor t) {
- t.getAttributes().set(SYNTHETIC, Boolean.TRUE);
+ t.setSynthetic(true);
+ return t;
+ }
+
+ public static Block s(Block t) {
+ t.setSynthetic(true);
return t;
}
=======================================
--- /trunk/src/com/google/caja/parser/quasiliteral/ES53Rewriter.java Fri
Nov 18 12:29:44 2011
+++ /trunk/src/com/google/caja/parser/quasiliteral/ES53Rewriter.java Mon
Dec 12 16:56:03 2011
@@ -14,6 +14,8 @@
package com.google.caja.parser.quasiliteral;
+import static com.google.caja.parser.js.SyntheticNodes.s;
+
import com.google.caja.lexer.FilePosition;
import com.google.caja.parser.ParseTreeNode;
import com.google.caja.parser.ParseTreeNodeContainer;
@@ -70,8 +72,6 @@
import com.google.caja.util.Pair;
import com.google.caja.util.Sets;
-import static com.google.caja.parser.js.SyntheticNodes.s;
-
import java.net.URI;
import java.util.Arrays;
import java.util.Collection;
@@ -231,7 +231,8 @@
name="translatedCode",
synopsis="Allow code received from a *->JS translator",
reason="Translated code should not be treated as user supplied
JS.",
- matches="<TranslatedCode>")
+ matches="<TranslatedCode>",
+ matchNode=TranslatedCode.class)
public ParseTreeNode fire(ParseTreeNode node, Scope scope) {
if (node instanceof TranslatedCode) {
Statement rewritten
@@ -322,6 +323,7 @@
+ "contents of the 'substitutes' of this rule.",
reason="So that the module loader can be invoked to load a
module.",
matches="<an UncajoledModule>",
+ matchNode=UncajoledModule.class,
substitutes=(
""
+ "(/*@synthetic*/{"
@@ -709,7 +711,8 @@
name="varBadSuffixDeclaration",
synopsis="Statically reject if a variable with `__` suffix is
found.",
reason="Caja reserves the `__` suffix for internal use.",
- matches="<approx>(var|function) @v__ ...", // TODO(mikesamuel):
limit
+ matches="<approx> var @v__ ...",
+ matchNode=Declaration.class,
substitutes="<reject>")
public ParseTreeNode fire(ParseTreeNode node, Scope scope) {
if (node instanceof Declaration) {
@@ -725,6 +728,33 @@
}
},
+
+ // This rule is separate from varBadSuffixDeclaration, because
+ // although FunctionDeclaration is a subclass of Declaration,
+ // it fuzzes as a FunctionConstructor.
+ new Rule() {
+ @Override
+ @RuleDescription(
+ name="functionBadSuffixDeclaration",
+ synopsis="Statically reject if a variable with `__` suffix is
found.",
+ reason="Caja reserves the `__` suffix for internal use.",
+ matches="<approx> function @v__ ...",
+ matchNode=FunctionDeclaration.class,
+ substitutes="<reject>")
+ public ParseTreeNode fire(ParseTreeNode node, Scope scope) {
+ if (node instanceof FunctionDeclaration) {
+ Identifier name = ((FunctionDeclaration) node).getIdentifier();
+ if (name.getValue().endsWith("__")) {
+ mq.addMessage(
+
RewriterMessageType.VARIABLES_CANNOT_END_IN_DOUBLE_UNDERSCORE,
+ node.getFilePosition(), this, node);
+ return node;
+ }
+ }
+ return NONE;
+ }
+ },
+
// TODO(metaweta): Use fastpath
new Rule() {
@Override
@@ -733,6 +763,7 @@
synopsis="Global vars are rewritten to be properties of
IMPORTS___.",
reason="",
matches="@v",
+ matchNode=Reference.class,
substitutes="IMPORTS___.@fp ?" +
"IMPORTS___.@v :" +
"___.ri(IMPORTS___, @vname)")
@@ -762,7 +793,8 @@
name="varDefault",
synopsis="Any remaining uses of a variable name are preserved.",
reason="",
- matches="@v", // TODO(mikesamuel): limit further
+ matches="@v",
+ matchNode=Reference.class,
substitutes="@v")
public ParseTreeNode fire(ParseTreeNode node, Scope scope) {
Map<String, ParseTreeNode> bindings = match(node);
@@ -1300,8 +1332,8 @@
name="setReadModifyWriteLocalVar",
synopsis="",
reason="",
- // TODO(mikesamuel): better lower limit
matches="<approx> @x @op= @y",
+ matchNode=AssignOperation.class,
substitutes="<approx> @x = @x @op @y")
// Handle x += 3 and similar ops by rewriting them using the
assignment
// delegate, "x += y" => "x = x + y", with
deconstructReadAssignOperand
@@ -1334,8 +1366,8 @@
@RuleDescription(
name="setIncrDecr",
synopsis="Handle pre and post ++ and --.",
- // TODO(mikesamuel): better lower bound
matches="<approx> ++@x but any {pre,post}{in,de}crement will do",
+ matchNode=AssignOperation.class,
reason="")
public ParseTreeNode fire(ParseTreeNode node, Scope scope) {
if (!(node instanceof AssignOperation)) { return NONE; }
@@ -1853,7 +1885,8 @@
name="mapBadKeySuffix",
synopsis="Statically reject a property whose name ends with
`__`",
reason="",
- matches="\"@k__\": @v",
+ matches="<approx> \"@k__\": @v",
+ matchNode=ObjProperty.class,
substitutes="<reject>")
public ParseTreeNode fire(ParseTreeNode node, Scope scope) {
if (node instanceof ObjProperty) {
@@ -1876,7 +1909,8 @@
name="objectProperty",
synopsis="nymize object properties",
reason="",
- matches="\"@k\": @v",
+ matches="<approx> \"@k\": @v",
+ matchNode=ObjProperty.class,
substitutes="<nymized>")
public ParseTreeNode fire(ParseTreeNode node, Scope scope) {
if (node instanceof ObjProperty) {
@@ -2064,6 +2098,7 @@
reason="So that every use of a regex literal creates a new
instance"
+ " to prevent state from leaking via interned literals.
This"
+ " is consistent with the way ES4 treates regex literals.",
+ matchNode=RegexpLiteral.class,
substitutes="new RegExp.new___(@pattern, @modifiers?)")
public ParseTreeNode fire(ParseTreeNode node, Scope scope) {
if (node instanceof RegexpLiteral) {
@@ -2091,7 +2126,8 @@
+ " could lead to a directive appearing in an illegal
position"
+ " since directives must appear at the beginning of a
program"
+ " or function body, not in an arbitrary block",
- matches="'use';",
+ matches="<approx> 'use';",
+ matchNode=DirectivePrologue.class,
substitutes=";")
public ParseTreeNode fire(ParseTreeNode node, Scope scope) {
if (node instanceof DirectivePrologue) {
=======================================
---
/trunk/src/com/google/caja/parser/quasiliteral/IllegalReferenceCheckRewriter.java
Mon Aug 10 14:00:55 2009
+++
/trunk/src/com/google/caja/parser/quasiliteral/IllegalReferenceCheckRewriter.java
Mon Dec 12 16:56:03 2011
@@ -38,8 +38,7 @@
reason="Double underscore identifiers may not be mentioned by
Caja "
+ "code")
public ParseTreeNode fire(ParseTreeNode node, Scope scope) {
- if (node instanceof Identifier
- && !node.getAttributes().is(SyntheticNodes.SYNTHETIC)) {
+ if (node instanceof Identifier && !SyntheticNodes.is(node)) {
String name = ((Identifier)node).getValue();
if (name != null && name.endsWith("__")) {
mq.addMessage(
=======================================
---
/trunk/src/com/google/caja/parser/quasiliteral/NonAsciiCheckVisitor.java
Mon Aug 4 18:06:15 2008
+++
/trunk/src/com/google/caja/parser/quasiliteral/NonAsciiCheckVisitor.java
Mon Dec 12 16:56:03 2011
@@ -40,7 +40,7 @@
if (!(ac.node instanceof Identifier)) { return true; }
Identifier ident = (Identifier) ac.node;
String name = ident.getName();
- if (!ident.getAttributes().is(SyntheticNodes.SYNTHETIC)
+ if (!SyntheticNodes.is(ident)
&& name != null && !name.matches("^[a-zA-Z_$][a-zA-Z0-9_$]*$")) {
mq.addMessage(
RewriterMessageType.NONASCII_IDENTIFIER,
=======================================
--- /trunk/src/com/google/caja/parser/quasiliteral/QuasiBuilder.java Mon
Nov 28 01:31:08 2011
+++ /trunk/src/com/google/caja/parser/quasiliteral/QuasiBuilder.java Mon
Dec 12 16:56:03 2011
@@ -29,8 +29,11 @@
import com.google.caja.parser.js.Expression;
import com.google.caja.parser.js.ExpressionStmt;
import com.google.caja.parser.js.FormalParam;
+import com.google.caja.parser.js.FunctionConstructor;
import com.google.caja.parser.js.FunctionDeclaration;
import com.google.caja.parser.js.Identifier;
+import com.google.caja.parser.js.LabeledStatement;
+import com.google.caja.parser.js.LabeledStmtWrapper;
import com.google.caja.parser.js.ObjProperty;
import com.google.caja.parser.js.ObjectConstructor;
import com.google.caja.parser.js.Parser;
@@ -194,6 +197,26 @@
public static QuasiNode parseQuasiNode(String pattern) throws
ParseException {
return parseQuasiNode(FilePosition.UNKNOWN.source(), pattern);
}
+
+ /** This parallels the fuzzing done in
+ * {@link QuasiBuilder#parseQuasiNode(InputSource, String)} */
+ // TODO(felix8a): why is this comment a lie?
+ public static Class<? extends ParseTreeNode> fuzzType(
+ Class<? extends ParseTreeNode> nodeClass) {
+ if (nodeClass == FunctionDeclaration.class) {
+ return FunctionConstructor.class;
+ }
+ if (nodeClass == Expression.class) {
+ return ExpressionStmt.class;
+ }
+ if (nodeClass == Reference.class) {
+ return Identifier.class;
+ }
+ if (nodeClass == LabeledStmtWrapper.class) {
+ return LabeledStatement.class;
+ }
+ return nodeClass;
+ }
private static QuasiNode getPatternNode(String patternText) {
if (!patternCache.containsKey(patternText)) {
=======================================
--- /trunk/src/com/google/caja/parser/quasiliteral/Rewriter.java Mon Nov 28
01:31:08 2011
+++ /trunk/src/com/google/caja/parser/quasiliteral/Rewriter.java Mon Dec 12
16:56:03 2011
@@ -29,7 +29,6 @@
import com.google.caja.reporting.MessageLevel;
import com.google.caja.reporting.MessageQueue;
import com.google.caja.reporting.RenderContext;
-import com.google.caja.util.SyntheticAttributes;
import java.util.HashSet;
import java.util.Set;
@@ -218,8 +217,7 @@
// Returns true if check passed.
private boolean checkTainted(ParseTreeNode node, MessageQueue mq) {
if (tainted.contains(node)) {
- SyntheticAttributes attrs = node.getAttributes();
- if (!attrs.is(SyntheticNodes.SYNTHETIC)) {
+ if (!SyntheticNodes.is(node)) {
mq.addMessage(
RewriterMessageType.UNSEEN_NODE_LEFT_OVER, node,
node.getFilePosition());
=======================================
--- /trunk/src/com/google/caja/parser/quasiliteral/Rule.java Fri Nov 18
12:29:44 2011
+++ /trunk/src/com/google/caja/parser/quasiliteral/Rule.java Mon Dec 12
16:56:03 2011
@@ -14,6 +14,8 @@
package com.google.caja.parser.quasiliteral;
+import static com.google.caja.parser.js.SyntheticNodes.s;
+
import com.google.caja.lexer.FilePosition;
import com.google.caja.lexer.Keyword;
import com.google.caja.lexer.TokenConsumer;
@@ -31,12 +33,12 @@
import com.google.caja.parser.js.Identifier;
import com.google.caja.parser.js.Literal;
import com.google.caja.parser.js.Noop;
+import com.google.caja.parser.js.ObjectConstructor;
import com.google.caja.parser.js.Operation;
import com.google.caja.parser.js.Operator;
import com.google.caja.parser.js.Reference;
import com.google.caja.parser.js.StringLiteral;
import com.google.caja.parser.js.SyntheticNodes;
-import static com.google.caja.parser.js.SyntheticNodes.s;
import com.google.caja.reporting.MessageContext;
import com.google.caja.reporting.MessagePart;
import com.google.caja.reporting.RenderContext;
@@ -139,6 +141,35 @@
}
return description;
}
+
+ public boolean canMatch(Class<? extends ParseTreeNode> nodeType) {
+ RuleDescription desc = getRuleDescription();
+ Class<? extends ParseTreeNode> bound = desc.matchNode();
+ if (bound != ParseTreeNode.class) {
+ // If the rule has an explicit matchNode=, use it.
+ bound = QuasiBuilder.fuzzType(bound);
+
+ } else {
+ // Otherwise, try parsing the matches= pattern
+ String pattern = desc.matches();
+ bound = quasiLowerBound(QuasiCache.parse(pattern));
+ }
+ return bound.isAssignableFrom(nodeType);
+ }
+
+ private static Class<? extends ParseTreeNode> quasiLowerBound(QuasiNode
p) {
+ if (p == null) {
+ return ParseTreeNode.class;
+ } else if (p instanceof SimpleQuasiNode) {
+ return QuasiBuilder.fuzzType(((SimpleQuasiNode)
p).getMatchedClass());
+ } else if (p instanceof ObjectCtorQuasiNode) {
+ return ObjectConstructor.class;
+ } else if (p instanceof StringLiteralQuasiNode) {
+ return StringLiteral.class;
+ } else {
+ return ParseTreeNode.class;
+ }
+ }
/**
* Process the given input, returning a rewritten node.
@@ -184,6 +215,9 @@
node.getValue(),
rewrittenChildren);
result.getAttributes().putAll(node.getAttributes());
+ if (SyntheticNodes.is(node)) {
+ SyntheticNodes.s(result);
+ }
result.makeImmutable();
return result;
@@ -363,7 +397,7 @@
}
protected static boolean isSynthetic(Identifier node) {
- return node.getAttributes().is(SyntheticNodes.SYNTHETIC);
+ return node.isSynthetic();
}
protected static boolean isSynthetic(Reference node) {
@@ -371,7 +405,7 @@
}
protected static boolean isSynthetic(FunctionConstructor node) {
- return node.getAttributes().is(SyntheticNodes.SYNTHETIC);
+ return node.isSynthetic();
}
protected static String getReferenceName(ParseTreeNode ref) {
=======================================
--- /trunk/src/com/google/caja/parser/quasiliteral/RuleChain.java Fri Feb
26 16:15:29 2010
+++ /trunk/src/com/google/caja/parser/quasiliteral/RuleChain.java Mon Dec
12 16:56:03 2011
@@ -14,24 +14,12 @@
package com.google.caja.parser.quasiliteral;
-import com.google.caja.lexer.ParseException;
import com.google.caja.parser.ParseTreeNode;
-import com.google.caja.parser.js.Expression;
-import com.google.caja.parser.js.ExpressionStmt;
-import com.google.caja.parser.js.FunctionConstructor;
-import com.google.caja.parser.js.FunctionDeclaration;
-import com.google.caja.parser.js.Identifier;
-import com.google.caja.parser.js.LabeledStatement;
-import com.google.caja.parser.js.LabeledStmtWrapper;
-import com.google.caja.parser.js.ObjectConstructor;
-import com.google.caja.parser.js.Reference;
-import com.google.caja.parser.js.StringLiteral;
+import com.google.caja.parser.js.SpecialOperation;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
/**
* An order-significant series of rules.
@@ -40,91 +28,28 @@
*/
public final class RuleChain {
private final List<Rule> rules = new ArrayList<Rule>();
- private final Map<Class<? extends ParseTreeNode>, List<Rule>> filtered
- = new HashMap<Class<? extends ParseTreeNode>, List<Rule>>();
+
+ private final NodeTypeFilter nodeTypeFilter = new NodeTypeFilter(this);
+ private final SpecialOpFilter specOpFilter = new SpecialOpFilter(this);
public void add(Rule r) {
rules.add(r);
- filtered.clear();
+ nodeTypeFilter.reset();
+ specOpFilter.reset();
}
- /**
- * Return at least the rules applicable to the given node, but possibly
more.
- */
- public List<Rule> applicableTo(ParseTreeNode node) {
- Class<? extends ParseTreeNode> nodeType = toNodeType(node.getClass());
- List<Rule> filteredRules = filtered.get(nodeType);
- if (filteredRules == null) {
- computeRulesFor(nodeType);
- filteredRules = filtered.get(nodeType);
- }
- return filteredRules;
- }
-
- public Iterable<Rule> getAllRules() {
+ public List<Rule> getAllRules() {
return Collections.unmodifiableList(rules);
}
/**
- * Caches quasi text to a conservative lower bound for ParseTreeNode
types
- * that it might match.
- * This does not assume that quasi-strings parse to a valid parse tree.
- * Some of the quasi-strings look like "<Approximately> @foo = @bar" and
- * so on parse failures we return a lower bound of ParseTreeNode.
+ * Return at least the rules applicable to the given node, but possibly
more.
*/
- private static final Map<String, Class<? extends ParseTreeNode>>
lowerBounds
- = Collections.synchronizedMap(
- new HashMap<String, Class<? extends ParseTreeNode>>());
- static {
- lowerBounds.put(null, ParseTreeNode.class); // lower bound for no
pattern
- }
- private void computeRulesFor(Class<? extends ParseTreeNode> nodeType) {
- List<Rule> applicableRules = new ArrayList<Rule>();
- for (Rule rule : rules) {
- String pattern = rule.getRuleDescription().matches();
- Class<? extends ParseTreeNode> lowerBound = lowerBounds.get(pattern);
- if (lowerBound == null) {
- try {
- QuasiNode p = QuasiBuilder.parseQuasiNode(pattern);
- if (p instanceof SimpleQuasiNode) {
- lowerBound = toNodeType(((SimpleQuasiNode)
p).getMatchedClass());
- } else if (p instanceof ObjectCtorQuasiNode) {
- lowerBound = ObjectConstructor.class;
- } else if (p instanceof StringLiteralQuasiNode) {
- lowerBound = StringLiteral.class;
- } else {
- lowerBound = ParseTreeNode.class;
- }
- } catch (ParseException ex) {
- // If the match pattern can't be parsed then assume the lowest
lower
- // bound. This may happen if the match string is documentation,
not
- // a real pattern.
- lowerBound = ParseTreeNode.class;
- }
- lowerBounds.put(pattern, lowerBound);
- }
- if (lowerBound.isAssignableFrom(nodeType)) {
- applicableRules.add(rule);
- }
- }
- filtered.put(nodeType, Collections.unmodifiableList(applicableRules));
- }
-
- /** Parallels fuzzing done in QuasiBuilder.parseQuasiNode */
- private static Class<? extends ParseTreeNode> toNodeType(
- Class<? extends ParseTreeNode> nodeClass) {
- if (nodeClass == FunctionDeclaration.class) {
- return FunctionConstructor.class;
- }
- if (nodeClass == Expression.class) {
- return ExpressionStmt.class;
- }
- if (nodeClass == Reference.class) {
- return Identifier.class;
- }
- if (nodeClass == LabeledStmtWrapper.class) {
- return LabeledStatement.class;
- }
- return nodeClass;
+ public List<Rule> applicableTo(ParseTreeNode node) {
+ if (node instanceof SpecialOperation) {
+ return specOpFilter.rulesFor(node);
+ } else {
+ return nodeTypeFilter.rulesFor(node);
+ }
}
}
=======================================
--- /trunk/src/com/google/caja/parser/quasiliteral/RuleDescription.java Wed
Sep 24 19:39:44 2008
+++ /trunk/src/com/google/caja/parser/quasiliteral/RuleDescription.java Mon
Dec 12 16:56:03 2011
@@ -14,6 +14,8 @@
package com.google.caja.parser.quasiliteral;
+import com.google.caja.parser.ParseTreeNode;
+
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
@@ -22,7 +24,7 @@
/**
* Describes each transformation rule
- *
+ *
* @author [email protected] (Jasvir Nagra)
*
*/
@@ -34,27 +36,33 @@
* @return Name of the transformation {@code Rule}
*/
String name();
-
+
/**
* @return Summary of the effect of {@code Rule}
*/
String synopsis();
-
+
/**
* @return Reason that necessitates {@code Rule}
- */
+ */
String reason();
/**
* @return Quasiliteral pattern matched by {@code Rule}
- */
+ */
String matches() default "<UNKNOWN>";
/**
* @return Code generated by {@code Rule}
- */
+ */
String substitutes() default "<UNKNOWN>";
+ /**
+ * @return Type of node this rule applies to. This could go away
+ * if {@code matches} were more expressive.
+ */
+ Class<? extends ParseTreeNode> matchNode() default ParseTreeNode.class;
+
// TODO(ihab.awad): Add metadata for a rule indicating that it always
fails,
// and perhaps in this case add a sanity check that "substitutes" is
empty.
}
=======================================
--- /trunk/src/com/google/caja/parser/quasiliteral/Scope.java Fri Nov 18
12:29:44 2011
+++ /trunk/src/com/google/caja/parser/quasiliteral/Scope.java Mon Dec 12
16:56:03 2011
@@ -14,39 +14,37 @@
package com.google.caja.parser.quasiliteral;
+import static com.google.caja.parser.js.SyntheticNodes.s;
+import static com.google.caja.parser.quasiliteral.QuasiBuilder.substV;
+
import com.google.caja.lexer.FilePosition;
import com.google.caja.lexer.Keyword;
import com.google.caja.parser.ParseTreeNode;
import com.google.caja.parser.ParseTreeNodeContainer;
-import com.google.caja.parser.js.DirectivePrologue;
-import com.google.caja.parser.js.Expression;
-import com.google.caja.parser.js.MultiDeclaration;
-import com.google.caja.parser.js.SyntheticNodes;
+import com.google.caja.parser.js.Block;
import com.google.caja.parser.js.CatchStmt;
import com.google.caja.parser.js.Declaration;
+import com.google.caja.parser.js.DirectivePrologue;
+import com.google.caja.parser.js.Expression;
import com.google.caja.parser.js.FunctionConstructor;
import com.google.caja.parser.js.FunctionDeclaration;
import com.google.caja.parser.js.Identifier;
-import com.google.caja.parser.js.Reference;
-import com.google.caja.parser.js.Block;
-import com.google.caja.parser.js.Statement;
+import com.google.caja.parser.js.MultiDeclaration;
import com.google.caja.parser.js.Operation;
import com.google.caja.parser.js.Operator;
+import com.google.caja.parser.js.Reference;
+import com.google.caja.parser.js.Statement;
import com.google.caja.parser.js.UncajoledModule;
import com.google.caja.parser.js.scope.ScopeType;
import com.google.caja.reporting.Message;
import com.google.caja.reporting.MessageLevel;
import com.google.caja.reporting.MessagePart;
-import com.google.caja.reporting.MessageQueue;
import com.google.caja.reporting.MessageType;
import com.google.caja.util.Lists;
import com.google.caja.util.Maps;
import com.google.caja.util.Pair;
import com.google.caja.util.Sets;
-import static com.google.caja.parser.js.SyntheticNodes.s;
-import static com.google.caja.parser.quasiliteral.QuasiBuilder.substV;
-
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@@ -569,7 +567,7 @@
}
private void visitFunctionConstructor(FunctionConstructor node) {
- if (node.getAttributes().is(SyntheticNodes.SYNTHETIC)) {
+ if (node.isSynthetic()) {
// Synthetic function definitions are treated as "transparent"; our
// scope analysis should "see through" them as though they were
just
// part of the surrounding code.
@@ -609,7 +607,7 @@
}
private void visitReference(Reference node) {
- if
(!node.getIdentifier().getAttributes().is(SyntheticNodes.SYNTHETIC) &&
+ if (!node.getIdentifier().isSynthetic() &&
!exceptionVariables.contains(node.getIdentifierName())) {
references.add(node);
}
@@ -686,7 +684,7 @@
|| maskedType == LocalType.CAUGHT_EXCEPTION)
? MessageLevel.ERROR
: MessageLevel.LINT);
- if (!ident.getAttributes().is(SyntheticNodes.SYNTHETIC) &&
+ if (!ident.isSynthetic() &&
ident.getFilePosition() != null) {
s.rewriter.mq.getMessages().add(new Message(
MessageType.MASKING_SYMBOL,
=======================================
--- /trunk/src/com/google/caja/parser/quasiliteral/SimpleQuasiNode.java Fri
Feb 26 16:15:29 2010
+++ /trunk/src/com/google/caja/parser/quasiliteral/SimpleQuasiNode.java Mon
Dec 12 16:56:03 2011
@@ -34,7 +34,7 @@
public class SimpleQuasiNode extends QuasiNode {
private final Class<? extends ParseTreeNode> clazz;
private final Object value;
- private Equivalence valueComparator;
+ private final Equivalence valueComparator;
protected SimpleQuasiNode(
Class<? extends ParseTreeNode> clazz, Object value,
@@ -124,6 +124,8 @@
/** The class of node matched by this quasi node. */
public Class<? extends ParseTreeNode> getMatchedClass() { return clazz; }
+
+ public Object getValue() { return value; }
@Override
public String toString() {
=======================================
--- /trunk/src/com/google/caja/parser/quasiliteral/SyntheticRuleSet.java
Fri Nov 18 12:29:44 2011
+++ /trunk/src/com/google/caja/parser/quasiliteral/SyntheticRuleSet.java
Mon Dec 12 16:56:03 2011
@@ -53,6 +53,7 @@
synopsis="Pass through synthetic references.",
reason="A variable may not be mentionable otherwise.",
matches="/* synthetic */ @ref",
+ matchNode=Reference.class,
substitutes="<expanded>")
public ParseTreeNode fire(ParseTreeNode node, Scope scope) {
if (node instanceof Reference) {
@@ -211,6 +212,7 @@
synopsis="Pass through synthetic formals which are
unmentionable.",
reason="Synthetic code might need local variables for
safe-keeping.",
matches="/* synthetic */ @x in a parameter list",
+ matchNode=FormalParam.class,
substitutes="<expanded>")
public ParseTreeNode fire(ParseTreeNode node, Scope scope) {
if (node instanceof FormalParam
=======================================
---
/trunk/src/com/google/caja/parser/quasiliteral/TrailingUnderscoresHole.java
Wed Feb 11 20:17:12 2009
+++
/trunk/src/com/google/caja/parser/quasiliteral/TrailingUnderscoresHole.java
Mon Dec 12 16:56:03 2011
@@ -47,6 +47,7 @@
specimen.getFilePosition(),
value.substring(0, value.length() - trailing.length()));
shortIdentifier.getAttributes().putAll(specimen.getAttributes());
+ shortIdentifier.setSynthetic(specimen.isSynthetic());
return putIfDeepEquals(
bindings,
@@ -66,6 +67,7 @@
Identifier withSuffix = new Identifier(
withoutSuffix.getFilePosition(), n.getValue() + trailing);
withSuffix.getAttributes().putAll(withoutSuffix.getAttributes());
+ withSuffix.setSynthetic(withoutSuffix.isSynthetic());
substitutes.add(withSuffix);
return true;
}
=======================================
--- /trunk/tests/com/google/caja/parser/ParseTreeNodeTest.java Mon Nov 28
01:31:08 2011
+++ /trunk/tests/com/google/caja/parser/ParseTreeNodeTest.java Mon Dec 12
16:56:03 2011
@@ -698,6 +698,8 @@
assertSerializable(p);
}
+ @SuppressWarnings("unchecked")
+ // setComments causes a warning about generic array for varargs
public final void testImmutability() throws Exception {
Block n = new Block(
FilePosition.UNKNOWN,
@@ -720,7 +722,7 @@
fail();
} catch (UnsupportedOperationException e) {}
try {
- n.getAttributes().set(SyntheticNodes.SYNTHETIC, true);
+ SyntheticNodes.s(n);
fail();
} catch (UnsupportedOperationException e) {}
}
