[
https://issues.apache.org/jira/browse/PIG-3629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13903892#comment-13903892
]
Daniel Dai commented on PIG-3629:
---------------------------------
Seems there is regression introduced by PIG-3629-reopen-1.patch. e2e tests
Scripting_1 - Scripting_12 fail:
org.apache.tez.dag.api.TezUncheckedException: java.io.IOException:
Deserialization error: could not instantiate
'org.apache.pig.scripting.jython.JythonFunction' with arguments
'[/Users/daijy/pig-tez/test/e2e/pig/testdist/libexec/python/scriptingudf.py,
square]'
at
org.apache.tez.mapreduce.committer.MROutputCommitter.getOutputCommitter(MROutputCommitter.java:128)
at
org.apache.tez.mapreduce.committer.MROutputCommitter.initialize(MROutputCommitter.java:66)
at
org.apache.tez.dag.app.dag.impl.VertexImpl$2.run(VertexImpl.java:1217)
at
org.apache.tez.dag.app.dag.impl.VertexImpl$2.run(VertexImpl.java:1202)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:394)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1495)
at
org.apache.tez.dag.app.dag.impl.VertexImpl.initializeVertex(VertexImpl.java:1202)
at
org.apache.tez.dag.app.dag.impl.VertexImpl.initializeVertexInInitializingState(VertexImpl.java:1479)
at
org.apache.tez.dag.app.dag.impl.VertexImpl.access$3600(VertexImpl.java:144)
at
org.apache.tez.dag.app.dag.impl.VertexImpl$RootInputInitializedTransition.transition(VertexImpl.java:1513)
at
org.apache.tez.dag.app.dag.impl.VertexImpl$RootInputInitializedTransition.transition(VertexImpl.java:1494)
at
org.apache.hadoop.yarn.state.StateMachineFactory$MultipleInternalArc.doTransition(StateMachineFactory.java:385)
at
org.apache.hadoop.yarn.state.StateMachineFactory.doTransition(StateMachineFactory.java:302)
at
org.apache.hadoop.yarn.state.StateMachineFactory.access$300(StateMachineFactory.java:46)
at
org.apache.hadoop.yarn.state.StateMachineFactory$InternalStateMachine.doTransition(StateMachineFactory.java:448)
at
org.apache.tez.dag.app.dag.impl.VertexImpl.handle(VertexImpl.java:955)
at
org.apache.tez.dag.app.dag.impl.VertexImpl.handle(VertexImpl.java:143)
at
org.apache.tez.dag.app.DAGAppMaster$VertexEventDispatcher.handle(DAGAppMaster.java:1354)
at
org.apache.tez.dag.app.DAGAppMaster$VertexEventDispatcher.handle(DAGAppMaster.java:1340)
at
org.apache.hadoop.yarn.event.AsyncDispatcher.dispatch(AsyncDispatcher.java:134)
at
org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:81)
at java.lang.Thread.run(Thread.java:695)
Caused by: java.io.IOException: Deserialization error: could not instantiate
'org.apache.pig.scripting.jython.JythonFunction' with arguments
'[/Users/daijy/pig-tez/test/e2e/pig/testdist/libexec/python/scriptingudf.py,
square]'
at
org.apache.pig.impl.util.ObjectSerializer.deserialize(ObjectSerializer.java:59)
at
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputFormat.getStores(PigOutputFormat.java:218)
at
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputFormat.setupUdfEnvAndStores(PigOutputFormat.java:245)
at
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputFormat.getOutputCommitter(PigOutputFormat.java:275)
at
org.apache.tez.mapreduce.committer.MROutputCommitter.getOutputCommitter(MROutputCommitter.java:126)
... 22 more
Caused by: java.lang.RuntimeException: could not instantiate
'org.apache.pig.scripting.jython.JythonFunction' with arguments
'[/Users/daijy/pig-tez/test/e2e/pig/testdist/libexec/python/scriptingudf.py,
square]'
at
org.apache.pig.impl.PigContext.instantiateFuncFromSpec(PigContext.java:727)
at
org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserFunc.instantiateFunc(POUserFunc.java:126)
at
org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserFunc.readObject(POUserFunc.java:572)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:979)
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1873)
at
java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329)
at java.io.ObjectInputStream.readObject(ObjectInputStream.java:349)
at java.util.ArrayList.readObject(ArrayList.java:593)
at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:979)
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1873)
at
java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329)
at java.io.ObjectInputStream.readObject(ObjectInputStream.java:349)
at java.util.HashMap.readObject(HashMap.java:1030)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:979)
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1873)
at
java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329)
at
java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970)
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1895)
at
java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329)
at
java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970)
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1895)
at
java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329)
at java.io.ObjectInputStream.readObject(ObjectInputStream.java:349)
at java.util.ArrayList.readObject(ArrayList.java:593)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:979)
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1873)
at
java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329)
at
java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970)
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1895)
at
java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329)
at java.io.ObjectInputStream.readObject(ObjectInputStream.java:349)
at java.util.ArrayList.readObject(ArrayList.java:593)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:979)
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1873)
at
java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329)
at
java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970)
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1895)
at
java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329)
at java.io.ObjectInputStream.readObject(ObjectInputStream.java:349)
at java.util.ArrayList.readObject(ArrayList.java:593)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:979)
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1873)
at
java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329)
at java.io.ObjectInputStream.readObject(ObjectInputStream.java:349)
at
org.apache.pig.impl.util.ObjectSerializer.deserialize(ObjectSerializer.java:57)
... 26 more
Caused by: java.lang.reflect.InvocationTargetException
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:39)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:27)
at java.lang.reflect.Constructor.newInstance(Constructor.java:513)
at
org.apache.pig.impl.PigContext.instantiateFuncFromSpec(PigContext.java:695)
... 103 more
Caused by: java.lang.IllegalStateException: Could not initialize:
/Users/daijy/pig-tez/test/e2e/pig/testdist/libexec/python/scriptingudf.py
at
org.apache.pig.scripting.jython.JythonFunction.<init>(JythonFunction.java:92)
... 108 more
Caused by: org.apache.pig.backend.executionengine.ExecException: ERROR 1121:
Python Error. Traceback (most recent call last):
File
"/Users/daijy/pig-tez/test/e2e/pig/testdist/libexec/python/scriptingudf.py",
line 21, in <module>
import stringutil
ImportError: No module named stringutil
at
org.apache.pig.scripting.jython.JythonScriptEngine$Interpreter.execfile(JythonScriptEngine.java:249)
at
org.apache.pig.scripting.jython.JythonScriptEngine$Interpreter.init(JythonScriptEngine.java:163)
at
org.apache.pig.scripting.jython.JythonScriptEngine.getFunction(JythonScriptEngine.java:388)
at
org.apache.pig.scripting.jython.JythonFunction.<init>(JythonFunction.java:55)
... 108 more
Caused by: Traceback (most recent call last):
File
"/Users/daijy/pig-tez/test/e2e/pig/testdist/libexec/python/scriptingudf.py",
line 21, in <module>
import stringutil
ImportError: No module named stringutil
at org.python.core.PyException.fillInStackTrace(PyException.java:70)
at java.lang.Throwable.<init>(Throwable.java:181)
at java.lang.Exception.<init>(Exception.java:29)
at java.lang.RuntimeException.<init>(RuntimeException.java:32)
at org.python.core.PyException.<init>(PyException.java:46)
at org.python.core.PyException.<init>(PyException.java:43)
at org.python.core.PyException.<init>(PyException.java:61)
at org.python.core.Py.ImportError(Py.java:304)
at org.python.core.imp.import_first(imp.java:755)
at org.python.core.imp.import_module_level(imp.java:837)
at org.python.core.imp.importName(imp.java:917)
at org.python.core.ImportFunction.__call__(__builtin__.java:1220)
at org.python.core.PyObject.__call__(PyObject.java:357)
at org.python.core.__builtin__.__import__(__builtin__.java:1173)
at org.python.core.imp.importOne(imp.java:936)
at
org.python.pycode._pyx3.f$0(/Users/daijy/pig-tez/test/e2e/pig/testdist/libexec/python/scriptingudf.py:97)
at
org.python.pycode._pyx3.call_function(/Users/daijy/pig-tez/test/e2e/pig/testdist/libexec/python/scriptingudf.py)
at org.python.core.PyTableCode.call(PyTableCode.java:165)
at org.python.core.PyCode.call(PyCode.java:18)
at org.python.core.Py.runCode(Py.java:1275)
at
org.python.util.PythonInterpreter.execfile(PythonInterpreter.java:235)
at
org.apache.pig.scripting.jython.JythonScriptEngine$Interpreter.execfile(JythonScriptEngine.java:217)
... 111 more
Will open a separate Jira for it.
> Implement STREAM in Tez
> -----------------------
>
> Key: PIG-3629
> URL: https://issues.apache.org/jira/browse/PIG-3629
> Project: Pig
> Issue Type: Sub-task
> Components: tez
> Affects Versions: tez-branch
> Reporter: Alex Bain
> Assignee: Alex Bain
> Fix For: tez-branch
>
> Attachments: PIG-3629-3.patch, PIG-3629-4.patch,
> PIG-3629-reopen-0.patch, PIG-3629-reopen-1.patch
>
>
> Implement the STREAM operator in Tez
--
This message was sent by Atlassian JIRA
(v6.1.5#6160)
