Revision: 3611
Author: jasvir
Date: Sun Jul 26 13:18:08 2009
Log: Updates the benchmarking code to including sizes and times
http://codereview.appspot.com/96158
* Added measurement of sizes of plain and cajoled code, rendered
pretty or minified and encoded plain or gzipped.
* Changed the measurement of performance to actual time taken rather
than a V8 "score". V8 scores do not give useful information over
history nor are comparable to other benchmarks.
[email protected]
http://code.google.com/p/google-caja/source/detail?r=3611
Added:
/trunk/tests/com/google/caja/demos/benchmarks/BenchmarkSize.java
Modified:
/trunk/tests/com/google/caja/demos/benchmarks/BenchmarkRunner.java
/trunk/tests/com/google/caja/util/CajaTestCase.java
/trunk/tests/com/google/caja/util/RhinoTestBed.java
=======================================
--- /dev/null
+++ /trunk/tests/com/google/caja/demos/benchmarks/BenchmarkSize.java Sun
Jul 26 13:18:08 2009
@@ -0,0 +1,122 @@
+// Copyright (C) 2008 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.caja.demos.benchmarks;
+
+import com.google.caja.lexer.ParseException;
+import com.google.caja.parser.AncestorChain;
+import com.google.caja.parser.js.Block;
+import com.google.caja.parser.js.CajoledModule;
+import com.google.caja.plugin.PluginCompiler;
+import com.google.caja.plugin.PluginMeta;
+import com.google.caja.reporting.TestBuildInfo;
+import com.google.caja.util.CajaTestCase;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.util.zip.GZIPOutputStream;
+
+/**
+ * Unit test which measures the size of cajoled javascript
+ *
+ * @author Jasvir Nagra ([email protected])
+ */
+public class BenchmarkSize extends CajaTestCase {
+
+ // Javascript files to benchmark
+ // TODO(jasvir): Find a nice collection of "typical" html files!
+ String[] pureJs = {
+ "richards.js",
+ "deltablue.js",
+ "crypto.js",
+ "raytrace.js"
+ };
+
+ /**
+ * Measures the size of cajoled vs original javascript
+ * Accumulates the result and formats it for consumption by varz
+ * Format:
+ * VarZ:benchmark.<benchmark>.size.<html|js>.<original|cajita|valija>
+ * .<pretty|minified>.<plain|gzip>
+ */
+ public void testJavascript () throws ParseException, IOException {
+ for (String js : pureJs) {
+ String name = stripExt(js, ".js");
+ varz(name, "original", "pretty", "plain",
+ size(charset(plain(fromResource(js)))));
+ varz(name, "original", "pretty", "gzip",
+ size(gzip(charset((plain(fromResource(js)))))));
+ varz(name, "original", "minified", "plain",
+ size(charset(minify(js(fromResource(js))))));
+ varz(name, "original", "minified", "gzip",
+ size(gzip(charset(minify(js(fromResource(js)))))));
+
+ varz(name, "valija", "pretty", "plain",
+ size(charset(render(valija(js(fromResource(js)))))));
+ varz(name, "valija", "pretty", "gzip",
+ size(gzip(charset(render(valija(js(fromResource(js))))))));
+ varz(name, "valija", "minify", "plain",
+ size(charset(minify(valija(js(fromResource(js)))))));
+ varz(name, "valija", "minify", "gzip",
+ size(gzip(charset(minify(valija(js(fromResource(js))))))));
+ }
+ }
+
+ private int size(byte[] data) {
+ return data.length;
+ }
+
+ private void varz(String name, String lang, String rendering, String enc,
+ long value) {
+ System.out.println("VarZ:benchmark." + name + ".size.js." +
+ lang + "." + rendering +"." + enc + "=" + value);
+ }
+
+ private String stripExt(String filename, String extension) {
+ if (filename.endsWith(extension)) {
+ return filename.substring(0, filename.length() - extension.length());
+ }
+ return filename;
+ }
+
+ public byte[] charset(String v) throws UnsupportedEncodingException {
+ return v.getBytes("UTF-8");
+ }
+
+ public byte[] gzip(byte[] data) throws IOException {
+ ByteArrayOutputStream stream = new ByteArrayOutputStream();
+ GZIPOutputStream gzipper = new GZIPOutputStream(stream);
+ gzipper.write(data);
+ gzipper.finish();
+ return stream.toByteArray();
+ }
+
+ public CajoledModule valija(Block plain) {
+ return cajole(plain, true);
+ }
+
+ public CajoledModule cajita(Block plain) {
+ return cajole(plain, false);
+ }
+
+ public CajoledModule cajole(Block js, boolean valija) {
+ PluginMeta meta = new PluginMeta();
+ meta.setValijaMode(valija);
+ PluginCompiler pc = new PluginCompiler(new TestBuildInfo(), meta, mq);
+ pc.addInput(AncestorChain.instance(js));
+ assertTrue(pc.run());
+ return pc.getJavascript();
+ }
+}
=======================================
--- /trunk/tests/com/google/caja/demos/benchmarks/BenchmarkRunner.java Tue
Jul 7 09:39:02 2009
+++ /trunk/tests/com/google/caja/demos/benchmarks/BenchmarkRunner.java Sun
Jul 26 13:18:08 2009
@@ -22,7 +22,8 @@
import com.google.caja.reporting.TestBuildInfo;
/**
- * Unit test which executes the V8 benchmark and collates the result for
rendering with varz
+ * Unit test which executes the V8 benchmark
+ * and collates the result for rendering with varz
*/
public class BenchmarkRunner extends CajaTestCase {
public void testRichards() throws Exception {
runBenchmark("richards.js"); }
@@ -32,23 +33,28 @@
public void testEarleyBoyer() throws Exception {
runBenchmark("earley-boyer.js");
}
+
/**
* Runs the given benchmark
* Accumulates the result and formats it for consumption by varz
* Format:
- * VarZ:benchmark.<benchmark name>.<speed|
size>.<language>.<debug?>.<engine>.<primed?>
+ * VarZ:benchmark.<benchmark name>.<speed|size>.<language>
+ * .<debug?>.<engine>.<primed?>
*/
private void runBenchmark(String filename) throws Exception {
- double scoreUncajoled = runUncajoled(filename);
- double scoreCajoled = runCajoled(filename);
+ double timeTakenUncajoled = runUncajoled(filename);
+ double timeTakenCajoled = runCajoled(filename);
+
System.out.println(
- "VarZ:benchmark." + getName()
+ ".speed.uncajoled.nodebug.rhino.cold=" + scoreUncajoled);
+ "VarZ:benchmark." + getName()
+ ".time.uncajoled.nodebug.rhino.cold="
+ + timeTakenUncajoled);
System.out.println(
- "VarZ:benchmark." + getName()
+ ".speed.valija.nodebug.rhino.cold=" + scoreCajoled);
+ "VarZ:benchmark." + getName() + ".time.valija.nodebug.rhino.cold="
+ + timeTakenCajoled);
System.out.println(
- "VarZ:benchmark." + getName()
+ ".speeddiff.valija.nodebug.rhino.cold="
- + (scoreCajoled / scoreUncajoled));
+ "VarZ:benchmark." + getName()
+ ".timeratio.valija.nodebug.rhino.cold="
+ + (timeTakenCajoled / timeTakenUncajoled));
}
// Like run.js but outputs the result differently.
@@ -65,15 +71,16 @@
+ " NotifyScore: function (s) { benchmark.score = s; }\n"
+ " });"
);
-
+
private double runUncajoled(String filename) throws Exception {
- Number score = (Number) RhinoTestBed.runJs(
+ Number elapsed = (Number) RhinoTestBed.runJs(
new RhinoTestBed.Input("var benchmark = {};", "setup"),
+ new RhinoTestBed.Input("benchmark.startTime = new
Date();", "clock"),
new RhinoTestBed.Input(getClass(), "base.js"),
new RhinoTestBed.Input(getClass(), filename),
new RhinoTestBed.Input(RUN_SCRIPT, getName()),
- new RhinoTestBed.Input("benchmark.score", "score"));
- return score.doubleValue();
+ new RhinoTestBed.Input("(new Date() -
benchmark.startTime)", "elapsed"));
+ return elapsed.doubleValue();
}
private double runCajoled(String filename) throws Exception {
@@ -85,7 +92,9 @@
pc.addInput(AncestorChain.instance(js(fromString(RUN_SCRIPT))));
assertTrue(pc.run());
String cajoledJs = render(pc.getJavascript());
- Number score = (Number) RhinoTestBed.runJs(
+ Number elapsed = (Number) RhinoTestBed.runJs(
+ new RhinoTestBed.Input(getClass(),
+ "../../../../../js/json_sans_eval/json_sans_eval.js"),
new RhinoTestBed.Input(getClass(), "../../cajita.js"),
new RhinoTestBed.Input(
""
@@ -103,13 +112,14 @@
""
+ "testImports = ___.copy(___.sharedImports);\n"
+ "testImports.benchmark = {};\n"
+ + "testImports.benchmark.startTime = new Date();"
+ "testImports.$v = valijaMaker.CALL___(testImports);\n"
+ "___.getNewModuleHandler().setImports(testImports);",
"benchmark-container"),
new RhinoTestBed.Input(cajoledJs, getName()),
new RhinoTestBed.Input(
- "testImports.benchmark.score",
- "score"));
- return score.doubleValue();
+ "(new Date() - testImports.benchmark.startTime)",
+ "elapsed"));
+ return elapsed.doubleValue();
}
}
=======================================
--- /trunk/tests/com/google/caja/util/CajaTestCase.java Thu Jul 16 12:48:13
2009
+++ /trunk/tests/com/google/caja/util/CajaTestCase.java Sun Jul 26 13:18:08
2009
@@ -108,6 +108,10 @@
return cp;
}
+ protected String plain(CharProducer cp) {
+ return new String(cp.getBuffer());
+ }
+
protected Block js(CharProducer cp) throws ParseException {
return js(cp, false);
}
=======================================
--- /trunk/tests/com/google/caja/util/RhinoTestBed.java Mon May 4 15:13:42
2009
+++ /trunk/tests/com/google/caja/util/RhinoTestBed.java Sun Jul 26 13:18:08
2009
@@ -72,7 +72,6 @@
* @author [email protected]
*/
public class RhinoTestBed {
-
/**
* Runs the javascript from the given inputs in order, and returns the
* result.
