[
https://issues.apache.org/jira/browse/STORM-2148?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15601152#comment-15601152
]
Jungtaek Lim commented on STORM-2148:
-------------------------------------
I've done simple benchmark between script evaluation vs native instance method
call.
Below is test code block, which is borrowed from TestPlanCompiler.testNested():
{code}
{
final Object[] current = context.values;
final java.util.Map inp2_ = (java.util.Map) current[2];
final java.util.List inp3_ = (java.util.List) current[3];
final boolean v4 = (Integer)
org.apache.calcite.runtime.SqlFunctions.arrayItemOptional(inp3_, 2) == null;
outputValues[0] = ((Integer)
org.apache.calcite.runtime.SqlFunctions.mapItemOptional(inp2_ == null ?
(java.util.Map) null : (java.util.Map)
org.apache.calcite.runtime.SqlFunctions.mapItemOptional(inp2_, "a"), "b") ==
null || ((Integer)
org.apache.calcite.runtime.SqlFunctions.mapItemOptional(inp2_ == null ?
(java.util.Map) null : (java.util.Map)
org.apache.calcite.runtime.SqlFunctions.mapItemOptional(inp2_, "a"),
"b")).intValue() == 2) && (v4 || ((Integer)
org.apache.calcite.runtime.SqlFunctions.arrayItemOptional(inp3_, 2)).intValue()
== 200) ? ((Integer)
org.apache.calcite.runtime.SqlFunctions.mapItemOptional(inp2_ == null ?
(java.util.Map) null : (java.util.Map)
org.apache.calcite.runtime.SqlFunctions.mapItemOptional(inp2_, "a"), "b") ==
null || v4 ? (Boolean) null : Boolean.TRUE) : Boolean.FALSE;
}
if (outputValues[0] == null || !((Boolean) outputValues[0])) { return 0; }
{
final Object[] current = context.values;
final java.util.Map inp1_ = (java.util.Map) current[1];
outputValues[0] = org.apache.calcite.runtime.SqlFunctions.toInt(current[0]);
outputValues[1] = inp1_ == null ? (Integer) null : (Integer)
org.apache.calcite.runtime.SqlFunctions.mapItemOptional(inp1_, "c");
outputValues[2] = (java.util.Map) current[2];
outputValues[3] = (java.util.List) current[3];
}
return 1;
{code}
and below is test benchmark code I've written.
{code}
package redis.clients.jedis.test;
import com.google.common.collect.Lists;
import org.apache.calcite.DataContext;
import org.apache.calcite.interpreter.Context;
import org.apache.calcite.interpreter.Scalar;
import org.apache.calcite.interpreter.StormContext;
import org.apache.calcite.interpreter.StormDataContext;
import org.codehaus.commons.compiler.CompileException;
import org.codehaus.janino.ScriptEvaluator;
import org.databene.contiperf.PerfTest;
import org.databene.contiperf.junit.ContiPerfRule;
import org.databene.contiperf.report.CSVSummaryReportModule;
import org.databene.contiperf.report.HtmlReportModule;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class NativeVsCodeEvaluatePerformantTest {
public static final int OPERATIONS_PER_TEST = 1000;
public static final int TEST_DURATION_MILLIS = 600 * 1000;
public static final int WARMUP_MILLIS = 60 * 1000;
private ScriptEvaluator evaluator;
private Scalar nativeClass;
private Object[] outputValues;
private DataContext dataContext;
@Rule
public ContiPerfRule rule = new ContiPerfRule(
new HtmlReportModule(),
new CSVSummaryReportModule());
private StormContext calciteContext;
public NativeVsCodeEvaluatePerformantTest() throws IOException {
super();
}
@Before
public void setUp() throws IOException, CompileException {
String expression = "{\n" + " final Object[] current =
context.values;\n"
+ " final java.util.Map inp2_ = (java.util.Map) current[2];\n"
+ " final java.util.List inp3_ = (java.util.List) current[3];\n"
+ " final boolean v4 = (Integer)
org.apache.calcite.runtime.SqlFunctions.arrayItemOptional(inp3_, 2) == null;\n"
+ " outputValues[0] = ((Integer)
org.apache.calcite.runtime.SqlFunctions.mapItemOptional(inp2_ == null ?
(java.util.Map) null : (java.util.Map)
org.apache.calcite.runtime.SqlFunctions.mapItemOptional(inp2_, \"a\"), \"b\")
== null || ((Integer)
org.apache.calcite.runtime.SqlFunctions.mapItemOptional(inp2_ == null ?
(java.util.Map) null : (java.util.Map)
org.apache.calcite.runtime.SqlFunctions.mapItemOptional(inp2_, \"a\"),
\"b\")).intValue() == 2) && (v4 || ((Integer)
org.apache.calcite.runtime.SqlFunctions.arrayItemOptional(inp3_, 2)).intValue()
== 200) ? ((Integer)
org.apache.calcite.runtime.SqlFunctions.mapItemOptional(inp2_ == null ?
(java.util.Map) null : (java.util.Map)
org.apache.calcite.runtime.SqlFunctions.mapItemOptional(inp2_, \"a\"), \"b\")
== null || v4 ? (Boolean) null : Boolean.TRUE) : Boolean.FALSE;\n"
+ "}\n" + "if (outputValues[0] == null || !((Boolean)
outputValues[0])) { return 0; }\n"
+ "\n" + "{\n" + " final Object[] current = context.values;\n"
+ " final java.util.Map inp1_ = (java.util.Map) current[1];\n"
+ " outputValues[0] =
org.apache.calcite.runtime.SqlFunctions.toInt(current[0]);\n"
+ " outputValues[1] = inp1_ == null ? (Integer) null : (Integer)
org.apache.calcite.runtime.SqlFunctions.mapItemOptional(inp1_, \"c\");\n"
+ " outputValues[2] = (java.util.Map) current[2];\n"
+ " outputValues[3] = (java.util.List) current[3];\n" + "}\n" +
"\n" + "return 1;\n";
evaluator = new ScriptEvaluator(expression, int.class,
new String[] {"context", "outputValues"},
new Class[] { Context.class, Object[].class });
nativeClass = new NativeClass();
outputValues = new Object[4];
dataContext = new StormDataContext();
calciteContext = new StormContext(dataContext);
calciteContext.values = getTestData();
}
@Test
@PerfTest(duration = TEST_DURATION_MILLIS, threads = 1, warmUp =
WARMUP_MILLIS)
public void testEvaluate() throws IOException, InvocationTargetException {
for (int i = 0 ; i < OPERATIONS_PER_TEST ; i++) {
evaluator.evaluate(new Object[] { calciteContext, outputValues });
}
}
@Test
@PerfTest(duration = TEST_DURATION_MILLIS, threads = 1, warmUp =
WARMUP_MILLIS)
public void testCallNativeClass() throws IOException {
for (int i = 0 ; i < OPERATIONS_PER_TEST ; i++) {
nativeClass.execute(calciteContext, outputValues);
}
}
@Test
@PerfTest(duration = TEST_DURATION_MILLIS, threads = 1, warmUp =
WARMUP_MILLIS)
public void testEvaluate2() throws IOException, InvocationTargetException {
for (int i = 0 ; i < OPERATIONS_PER_TEST ; i++) {
evaluator.evaluate(new Object[] { calciteContext, outputValues });
}
}
@Test
@PerfTest(duration = TEST_DURATION_MILLIS, threads = 1, warmUp =
WARMUP_MILLIS)
public void testCallNativeClass2() throws IOException {
for (int i = 0 ; i < OPERATIONS_PER_TEST ; i++) {
nativeClass.execute(calciteContext, outputValues);
}
}
@Test
@PerfTest(duration = TEST_DURATION_MILLIS, threads = 1, warmUp =
WARMUP_MILLIS)
public void testEvaluate3() throws IOException, InvocationTargetException {
for (int i = 0 ; i < OPERATIONS_PER_TEST ; i++) {
evaluator.evaluate(new Object[] { calciteContext, outputValues });
}
}
@Test
@PerfTest(duration = TEST_DURATION_MILLIS, threads = 1, warmUp =
WARMUP_MILLIS)
public void testCallNativeClass3() throws IOException {
for (int i = 0 ; i < OPERATIONS_PER_TEST ; i++) {
nativeClass.execute(calciteContext, outputValues);
}
}
class NativeClass implements Scalar {
public Object execute(Context context) {
{
Object[] outputValues = new Object[4];
final Object[] current = context.values;
final java.util.Map inp2_ = (java.util.Map) current[2];
final java.util.List inp3_ = (java.util.List) current[3];
final boolean v4 =
(Integer)
org.apache.calcite.runtime.SqlFunctions.arrayItemOptional(inp3_, 2) == null;
outputValues[0] = ((Integer)
org.apache.calcite.runtime.SqlFunctions.mapItemOptional(
inp2_ == null ?
(java.util.Map) null :
(java.util.Map) org.apache.calcite.runtime.SqlFunctions
.mapItemOptional(inp2_, "a"), "b") == null ||
((Integer)
org.apache.calcite.runtime.SqlFunctions.mapItemOptional(inp2_ == null ?
(java.util.Map) null :
(java.util.Map) org.apache.calcite.runtime.SqlFunctions
.mapItemOptional(inp2_, "a"), "b")).intValue() == 2) &&
(v4 ||
((Integer)
org.apache.calcite.runtime.SqlFunctions.arrayItemOptional(inp3_, 2)).intValue()
== 200) ?
((Integer)
org.apache.calcite.runtime.SqlFunctions.mapItemOptional(inp2_ == null ?
(java.util.Map) null :
(java.util.Map) org.apache.calcite.runtime.SqlFunctions
.mapItemOptional(inp2_, "a"), "b") == null || v4 ?
(Boolean) null : Boolean.TRUE) :
Boolean.FALSE;
}
if (outputValues[0] == null || !((Boolean) outputValues[0])) {
return 0; }
{
final Object[] current = context.values;
final java.util.Map inp1_ = (java.util.Map) current[1];
outputValues[0] =
org.apache.calcite.runtime.SqlFunctions.toInt(current[0]);
outputValues[1] = inp1_ == null ?
(Integer) null :
(Integer)
org.apache.calcite.runtime.SqlFunctions.mapItemOptional(inp1_, "c");
outputValues[2] = (java.util.Map) current[2];
outputValues[3] = (java.util.List) current[3];
return outputValues;
}
}
public void execute(Context context, Object[] results) {
{
Object[] outputValues = results;
final Object[] current = context.values;
final java.util.Map inp2_ = (java.util.Map) current[2];
final java.util.List inp3_ = (java.util.List) current[3];
final boolean v4 =
(Integer)
org.apache.calcite.runtime.SqlFunctions.arrayItemOptional(inp3_, 2) == null;
outputValues[0] = ((Integer)
org.apache.calcite.runtime.SqlFunctions.mapItemOptional(
inp2_ == null ?
(java.util.Map) null :
(java.util.Map) org.apache.calcite.runtime.SqlFunctions
.mapItemOptional(inp2_, "a"), "b") == null ||
((Integer)
org.apache.calcite.runtime.SqlFunctions.mapItemOptional(inp2_ == null ?
(java.util.Map) null :
(java.util.Map) org.apache.calcite.runtime.SqlFunctions
.mapItemOptional(inp2_, "a"), "b")).intValue() == 2) &&
(v4 ||
((Integer)
org.apache.calcite.runtime.SqlFunctions.arrayItemOptional(inp3_, 2)).intValue()
== 200) ?
((Integer)
org.apache.calcite.runtime.SqlFunctions.mapItemOptional(inp2_ == null ?
(java.util.Map) null :
(java.util.Map) org.apache.calcite.runtime.SqlFunctions
.mapItemOptional(inp2_, "a"), "b") == null || v4 ?
(Boolean) null : Boolean.TRUE) :
Boolean.FALSE;
}
if (outputValues[0] == null || !((Boolean) outputValues[0])) {
return; }
{
final Object[] current = context.values;
final java.util.Map inp1_ = (java.util.Map) current[1];
outputValues[0] =
org.apache.calcite.runtime.SqlFunctions.toInt(current[0]);
outputValues[1] = inp1_ == null ?
(Integer) null :
(Integer)
org.apache.calcite.runtime.SqlFunctions.mapItemOptional(inp1_, "c");
outputValues[2] = (java.util.Map) current[2];
outputValues[3] = (java.util.List) current[3];
}
}
}
private Object[] getTestData() {
List<Integer> ints = Arrays.asList(100, 200, 300);
int i = 1;
Map<String, Integer> map = new HashMap<String, Integer>();
map.put("b", i);
map.put("c", i*i);
Map<String, Map<String, Integer>> mm = new HashMap<String, Map<String,
Integer>>();
mm.put("a", map);
return Lists.newArrayList(i, map, mm, ints).toArray();
}
}
{code}
3 tests are for method call, and other 3 tests are for script evaluation. I
took median from 3 results.
|| metric || native class method call || evaluate script ||
| samples | 27438405 | 21017191 |
| max | 2 | 3 |
| average | 0.019548330159861696 | 0.025557173648942905 |
Native method call is approximately 30% faster than evaluating script.
Raw results are here:
redis.clients.jedis.test.NativeVsCodeEvaluatePerformantTest.testCallNativeClass
samples: 27438405
max: 2
average: 0.019548330159861696
median: 0
redis.clients.jedis.test.NativeVsCodeEvaluatePerformantTest.testCallNativeClass2
samples: 27677588
max: 1
average: 0.01939981186221863
median: 0
redis.clients.jedis.test.NativeVsCodeEvaluatePerformantTest.testCallNativeClass3
samples: 26451435
max: 18
average: 0.02029791578415311
median: 0
redis.clients.jedis.test.NativeVsCodeEvaluatePerformantTest.testEvaluate
samples: 21017191
max: 3
average: 0.025557173648942905
median: 0
redis.clients.jedis.test.NativeVsCodeEvaluatePerformantTest.testEvaluate2
samples: 21046684
max: 2
average: 0.025537039469020393
median: 0
redis.clients.jedis.test.NativeVsCodeEvaluatePerformantTest.testEvaluate3
samples: 19714388
max: 5
average: 0.02726546723134393
median: 0
> [Storm SQL] Trident mode: back to code generate and compile Trident topology
> ----------------------------------------------------------------------------
>
> Key: STORM-2148
> URL: https://issues.apache.org/jira/browse/STORM-2148
> Project: Apache Storm
> Issue Type: Improvement
> Components: storm-sql
> Reporter: Jungtaek Lim
> Assignee: Jungtaek Lim
>
> Now Storm SQL just converts Rex to code block and pass to Evaluation~ class
> so that each class can evaluate the code block in runtime.
> This change made the code greatly simplified, but I expect that it is not
> same as performant as compiling and execute natively.
> Linq4j in Calcite provides utility methods to make the code nicely. It's
> going to really really more verbose, but really better than having string
> concatenated code block since it doesn't have any guards.
> So let's convert it back to code generation, but more elegant.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
