[
https://issues.apache.org/jira/browse/SYSTEMML-2027?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Matthias Boehm updated SYSTEMML-2027:
-------------------------------------
Description:
Various large-scale test scenarios of the perftest suite show intermittent but
non-reproducible janino codegen compilation issues which result in rare task
failures. Due to missing synchronization, concurrent tasks might try to compile
and load the same class with the same class name multiple times leading to
unwanted side effects. The solution is simple, synchronize the class
compilation at the executors, which ensures that the first thread compiles a
non-existing class and all subsequent threads and tasks simply reuse this class
accordingly.
Below is a list of various exceptions leading to the intermittent task failures:
{code}
Caused by: org.codehaus.janino.JaninoRuntimeException:
codegen.TMP14.genexec(double[] a, int ai, SideInput[] b, double[] scalars,
double[] c, int len, int rix): Operand stack underrun at offset 91
at org.codehaus.janino.CodeContext.flowAnalysis(CodeContext.java:501)
at org.codehaus.janino.CodeContext.flowAnalysis(CodeContext.java:541)
at org.codehaus.janino.CodeContext.flowAnalysis(CodeContext.java:313)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:2839)
at
org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1262)
at
org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1234)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:538)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:420)
at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:374)
at
org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:369)
at
org.codehaus.janino.Java$AbstractPackageMemberClassDeclaration.accept(Java.java:1309)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369)
at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:345)
at
org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:396)
at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:205)
at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:196)
at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:91)
at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:84)
at
org.apache.sysml.runtime.codegen.CodegenUtils.compileClassJanino(CodegenUtils.java:174)
... 19 more
{code}
{code}
Caused by: org.codehaus.commons.compiler.CompileException: Line 14, Column 55:
No applicable constructor/method found for actual parameters "double[],
java.lang.Object, int, int"; candidates are: "public static double[]
org.apache.sysml.runtime.codegen.LibSpoofPrimitives.vectMultWrite(double[],
double, int[], int, int, int)", "public static double[]
org.apache.sysml.runtime.codegen.LibSpoofPrimitives.vectMultWrite(double[],
double[], int, int, int)", "public static double[]
org.apache.sysml.runtime.codegen.LibSpoofPrimitives.vectMultWrite(double,
double[], int, int)", "public static double[]
org.apache.sysml.runtime.codegen.LibSpoofPrimitives.vectMultWrite(double[],
double, int, int)", "public static double[]
org.apache.sysml.runtime.codegen.LibSpoofPrimitives.vectMultWrite(double[],
double[], int, int[], int, int, int)", "public static double[]
org.apache.sysml.runtime.codegen.LibSpoofPrimitives.vectMultWrite(double[],
double[], int[], int, int, int, int)", "public static double[]
org.apache.sysml.runtime.codegen.LibSpoofPrimitives.vectMultWrite(double,
double[], int[], int, int, int)"
at
org.codehaus.janino.UnitCompiler.compileError(UnitCompiler.java:11004)
at
org.codehaus.janino.UnitCompiler.findMostSpecificIInvocable(UnitCompiler.java:8307)
at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:8169)
at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:8071)
at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:4421)
at org.codehaus.janino.UnitCompiler.access$7500(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$12.visitMethodInvocation(UnitCompiler.java:3774)
at
org.codehaus.janino.UnitCompiler$12.visitMethodInvocation(UnitCompiler.java:3762)
at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:4328)
at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:3762)
at
org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:4933)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2330)
at org.codehaus.janino.UnitCompiler.access$2600(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$6.visitLocalVariableDeclarationStatement(UnitCompiler.java:1386)
at
org.codehaus.janino.UnitCompiler$6.visitLocalVariableDeclarationStatement(UnitCompiler.java:1370)
at
org.codehaus.janino.Java$LocalVariableDeclarationStatement.accept(Java.java:2974)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
at
org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1450)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:2811)
at
org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1262)
at
org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1234)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:538)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:420)
at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:374)
at
org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:369)
at
org.codehaus.janino.Java$AbstractPackageMemberClassDeclaration.accept(Java.java:1309)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369)
at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:345)
at
org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:396)
at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:205)
at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:196)
at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:91)
at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:84)
at
org.apache.sysml.runtime.codegen.CodegenUtils.compileClassJanino(CodegenUtils.java:174)
... 19 more
{code}
{code}
Caused by: org.codehaus.commons.compiler.CompileException: Line 16, Column 40:
Binary numeric promotion not possible on types "java.lang.Object" and "double"
at
org.codehaus.janino.UnitCompiler.compileError(UnitCompiler.java:11004)
at
org.codehaus.janino.UnitCompiler.binaryNumericPromotionType(UnitCompiler.java:10237)
at org.codehaus.janino.UnitCompiler.getType2(UnitCompiler.java:6227)
at org.codehaus.janino.UnitCompiler.access$13800(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$18.visitBinaryOperation(UnitCompiler.java:5675)
at
org.codehaus.janino.UnitCompiler$18.visitBinaryOperation(UnitCompiler.java:5660)
at org.codehaus.janino.Java$BinaryOperation.accept(Java.java:4155)
at org.codehaus.janino.Java$Rvalue.accept(Java.java:3532)
at org.codehaus.janino.UnitCompiler.getType(UnitCompiler.java:5660)
at
org.codehaus.janino.UnitCompiler.compileArithmeticOperation(UnitCompiler.java:6859)
at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:4367)
at org.codehaus.janino.UnitCompiler.access$6900(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$12.visitBinaryOperation(UnitCompiler.java:3768)
at
org.codehaus.janino.UnitCompiler$12.visitBinaryOperation(UnitCompiler.java:3762)
at org.codehaus.janino.Java$BinaryOperation.accept(Java.java:4155)
at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:3762)
at
org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:4933)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2330)
at org.codehaus.janino.UnitCompiler.access$2600(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$6.visitLocalVariableDeclarationStatement(UnitCompiler.java:1386)
at
org.codehaus.janino.UnitCompiler$6.visitLocalVariableDeclarationStatement(UnitCompiler.java:1370)
at
org.codehaus.janino.Java$LocalVariableDeclarationStatement.accept(Java.java:2974)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
at
org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1450)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:2811)
at
org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1262)
at
org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1234)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:538)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:420)
at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:374)
at
org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:369)
at
org.codehaus.janino.Java$AbstractPackageMemberClassDeclaration.accept(Java.java:1309)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369)
at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:345)
at
org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:396)
at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:205)
at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:196)
at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:91)
at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:84)
at
org.apache.sysml.runtime.codegen.CodegenUtils.compileClassJanino(CodegenUtils.java:174)
... 19 more
{code}
> Perftest distributed codegen ops w/ intermittent task failures
> --------------------------------------------------------------
>
> Key: SYSTEMML-2027
> URL: https://issues.apache.org/jira/browse/SYSTEMML-2027
> Project: SystemML
> Issue Type: Bug
> Reporter: Matthias Boehm
>
> Various large-scale test scenarios of the perftest suite show intermittent
> but non-reproducible janino codegen compilation issues which result in rare
> task failures. Due to missing synchronization, concurrent tasks might try to
> compile and load the same class with the same class name multiple times
> leading to unwanted side effects. The solution is simple, synchronize the
> class compilation at the executors, which ensures that the first thread
> compiles a non-existing class and all subsequent threads and tasks simply
> reuse this class accordingly.
> Below is a list of various exceptions leading to the intermittent task
> failures:
> {code}
> Caused by: org.codehaus.janino.JaninoRuntimeException:
> codegen.TMP14.genexec(double[] a, int ai, SideInput[] b, double[] scalars,
> double[] c, int len, int rix): Operand stack underrun at offset 91
> at org.codehaus.janino.CodeContext.flowAnalysis(CodeContext.java:501)
> at org.codehaus.janino.CodeContext.flowAnalysis(CodeContext.java:541)
> at org.codehaus.janino.CodeContext.flowAnalysis(CodeContext.java:313)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:2839)
> at
> org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1262)
> at
> org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1234)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:538)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:420)
> at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:206)
> at
> org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:374)
> at
> org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:369)
> at
> org.codehaus.janino.Java$AbstractPackageMemberClassDeclaration.accept(Java.java:1309)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369)
> at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:345)
> at
> org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:396)
> at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:205)
> at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:196)
> at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:91)
> at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:84)
> at
> org.apache.sysml.runtime.codegen.CodegenUtils.compileClassJanino(CodegenUtils.java:174)
> ... 19 more
> {code}
> {code}
> Caused by: org.codehaus.commons.compiler.CompileException: Line 14, Column
> 55: No applicable constructor/method found for actual parameters "double[],
> java.lang.Object, int, int"; candidates are: "public static double[]
> org.apache.sysml.runtime.codegen.LibSpoofPrimitives.vectMultWrite(double[],
> double, int[], int, int, int)", "public static double[]
> org.apache.sysml.runtime.codegen.LibSpoofPrimitives.vectMultWrite(double[],
> double[], int, int, int)", "public static double[]
> org.apache.sysml.runtime.codegen.LibSpoofPrimitives.vectMultWrite(double,
> double[], int, int)", "public static double[]
> org.apache.sysml.runtime.codegen.LibSpoofPrimitives.vectMultWrite(double[],
> double, int, int)", "public static double[]
> org.apache.sysml.runtime.codegen.LibSpoofPrimitives.vectMultWrite(double[],
> double[], int, int[], int, int, int)", "public static double[]
> org.apache.sysml.runtime.codegen.LibSpoofPrimitives.vectMultWrite(double[],
> double[], int[], int, int, int, int)", "public static double[]
> org.apache.sysml.runtime.codegen.LibSpoofPrimitives.vectMultWrite(double,
> double[], int[], int, int, int)"
> at
> org.codehaus.janino.UnitCompiler.compileError(UnitCompiler.java:11004)
> at
> org.codehaus.janino.UnitCompiler.findMostSpecificIInvocable(UnitCompiler.java:8307)
> at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:8169)
> at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:8071)
> at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:4421)
> at org.codehaus.janino.UnitCompiler.access$7500(UnitCompiler.java:206)
> at
> org.codehaus.janino.UnitCompiler$12.visitMethodInvocation(UnitCompiler.java:3774)
> at
> org.codehaus.janino.UnitCompiler$12.visitMethodInvocation(UnitCompiler.java:3762)
> at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:4328)
> at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:3762)
> at
> org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:4933)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2330)
> at org.codehaus.janino.UnitCompiler.access$2600(UnitCompiler.java:206)
> at
> org.codehaus.janino.UnitCompiler$6.visitLocalVariableDeclarationStatement(UnitCompiler.java:1386)
> at
> org.codehaus.janino.UnitCompiler$6.visitLocalVariableDeclarationStatement(UnitCompiler.java:1370)
> at
> org.codehaus.janino.Java$LocalVariableDeclarationStatement.accept(Java.java:2974)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
> at
> org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1450)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:2811)
> at
> org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1262)
> at
> org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1234)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:538)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:420)
> at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:206)
> at
> org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:374)
> at
> org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:369)
> at
> org.codehaus.janino.Java$AbstractPackageMemberClassDeclaration.accept(Java.java:1309)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369)
> at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:345)
> at
> org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:396)
> at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:205)
> at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:196)
> at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:91)
> at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:84)
> at
> org.apache.sysml.runtime.codegen.CodegenUtils.compileClassJanino(CodegenUtils.java:174)
> ... 19 more
> {code}
> {code}
> Caused by: org.codehaus.commons.compiler.CompileException: Line 16, Column
> 40: Binary numeric promotion not possible on types "java.lang.Object" and
> "double"
> at
> org.codehaus.janino.UnitCompiler.compileError(UnitCompiler.java:11004)
> at
> org.codehaus.janino.UnitCompiler.binaryNumericPromotionType(UnitCompiler.java:10237)
> at org.codehaus.janino.UnitCompiler.getType2(UnitCompiler.java:6227)
> at org.codehaus.janino.UnitCompiler.access$13800(UnitCompiler.java:206)
> at
> org.codehaus.janino.UnitCompiler$18.visitBinaryOperation(UnitCompiler.java:5675)
> at
> org.codehaus.janino.UnitCompiler$18.visitBinaryOperation(UnitCompiler.java:5660)
> at org.codehaus.janino.Java$BinaryOperation.accept(Java.java:4155)
> at org.codehaus.janino.Java$Rvalue.accept(Java.java:3532)
> at org.codehaus.janino.UnitCompiler.getType(UnitCompiler.java:5660)
> at
> org.codehaus.janino.UnitCompiler.compileArithmeticOperation(UnitCompiler.java:6859)
> at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:4367)
> at org.codehaus.janino.UnitCompiler.access$6900(UnitCompiler.java:206)
> at
> org.codehaus.janino.UnitCompiler$12.visitBinaryOperation(UnitCompiler.java:3768)
> at
> org.codehaus.janino.UnitCompiler$12.visitBinaryOperation(UnitCompiler.java:3762)
> at org.codehaus.janino.Java$BinaryOperation.accept(Java.java:4155)
> at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:3762)
> at
> org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:4933)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2330)
> at org.codehaus.janino.UnitCompiler.access$2600(UnitCompiler.java:206)
> at
> org.codehaus.janino.UnitCompiler$6.visitLocalVariableDeclarationStatement(UnitCompiler.java:1386)
> at
> org.codehaus.janino.UnitCompiler$6.visitLocalVariableDeclarationStatement(UnitCompiler.java:1370)
> at
> org.codehaus.janino.Java$LocalVariableDeclarationStatement.accept(Java.java:2974)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
> at
> org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1450)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:2811)
> at
> org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1262)
> at
> org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1234)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:538)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:420)
> at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:206)
> at
> org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:374)
> at
> org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:369)
> at
> org.codehaus.janino.Java$AbstractPackageMemberClassDeclaration.accept(Java.java:1309)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369)
> at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:345)
> at
> org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:396)
> at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:205)
> at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:196)
> at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:91)
> at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:84)
> at
> org.apache.sysml.runtime.codegen.CodegenUtils.compileClassJanino(CodegenUtils.java:174)
> ... 19 more
> {code}
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
