[
https://issues.apache.org/jira/browse/PIG-3181?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Fabian Alenius updated PIG-3181:
--------------------------------
Description:
Hi, I have a script that looks like this:
a = load '/input' as (f1, f2);
a = group a by f1;
a = foreach logs {
generate group, a.f2;
}
store logs into '/output/' using
org.apache.pig.piggybank.storage.MultiStorage('/output/', '0');
But for some reason it fails with:
FATAL org.apache.hadoop.mapred.Child: Error running child :
java.lang.OutOfMemoryError: Java heap space
at java.util.Arrays.copyOf(Arrays.java:2786)
at java.io.ByteArrayOutputStream.write(ByteArrayOutputStream.java:94)
at java.io.OutputStream.write(OutputStream.java:58)
at org.apache.pig.impl.util.StorageUtil.putField(StorageUtil.java:145)
at org.apache.pig.impl.util.StorageUtil.putField(StorageUtil.java:176)
at org.apache.pig.impl.util.StorageUtil.putField(StorageUtil.java:194)
at
org.apache.pig.piggybank.storage.MultiStorage$MultiStorageOutputFormat$1.write(MultiStorage.java:208)
at
org.apache.pig.piggybank.storage.MultiStorage$MultiStorageOutputFormat$1.write(MultiStorage.java:187)
at
org.apache.pig.piggybank.storage.MultiStorage.putNext(MultiStorage.java:138)
at
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputFormat$PigRecordWriter.write(PigOutputFormat.java:139)
at
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputFormat$PigRecordWriter.write(PigOutputFormat.java:98)
at
org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:537)
at
org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:88)
at
org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:99)
at
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapReduce$Reduce.runPipeline(PigGenericMapReduce.java:463)
at
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapReduce$Reduce.processOnePackageOutput(PigGenericMapReduce.java:428)
at
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapReduce$Reduce.reduce(PigGenericMapReduce.java:408)
at
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapReduce$Reduce.reduce(PigGenericMapReduce.java:262)
at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:164)
at
org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:595)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:433)
at org.apache.hadoop.mapred.Child$4.run(Child.java:268)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1332)
at org.apache.hadoop.mapred.Child.main(Child.java:262)
*stderr logs*
java.lang.RuntimeException: InternalCachedBag.spill() should not be called
at
org.apache.pig.data.InternalCachedBag.spill(InternalCachedBag.java:159)
at
org.apache.pig.impl.util.SpillableMemoryManager.handleNotification(SpillableMemoryManager.java:243)
at
sun.management.NotificationEmitterSupport.sendNotification(NotificationEmitterSupport.java:138)
at sun.management.MemoryImpl.createNotification(MemoryImpl.java:171)
at
sun.management.MemoryPoolImpl$PoolSensor.triggerAction(MemoryPoolImpl.java:272)
at sun.management.Sensor.trigger(Sensor.java:120)
Exception in thread "Low Memory Detector" java.lang.InternalError: Error in
invoking listener
at
sun.management.NotificationEmitterSupport.sendNotification(NotificationEmitterSupport.java:141)
at sun.management.MemoryImpl.createNotification(MemoryImpl.java:171)
at
sun.management.MemoryPoolImpl$PoolSensor.triggerAction(MemoryPoolImpl.java:272)
at sun.management.Sensor.trigger(Sensor.java:120)
was:
Hi, I have a script that looks like this:
a = load '/input' as (f1, f2);
a = group a by f1;
a = foreach logs {
generate group, a.f2;
}
store logs into '/output/' using
org.apache.pig.piggybank.storage.MultiStorage('/output/', '0');
But for some reason it fails with:
FATAL org.apache.hadoop.mapred.Child: Error running child :
java.lang.OutOfMemoryError: Java heap space
at java.util.Arrays.copyOf(Arrays.java:2786)
at java.io.ByteArrayOutputStream.write(ByteArrayOutputStream.java:94)
at java.io.OutputStream.write(OutputStream.java:58)
at org.apache.pig.impl.util.StorageUtil.putField(StorageUtil.java:145)
at org.apache.pig.impl.util.StorageUtil.putField(StorageUtil.java:176)
at org.apache.pig.impl.util.StorageUtil.putField(StorageUtil.java:194)
at
org.apache.pig.piggybank.storage.MultiStorage$MultiStorageOutputFormat$1.write(MultiStorage.java:208)
at
org.apache.pig.piggybank.storage.MultiStorage$MultiStorageOutputFormat$1.write(MultiStorage.java:187)
at
org.apache.pig.piggybank.storage.MultiStorage.putNext(MultiStorage.java:138)
at
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputFormat$PigRecordWriter.write(PigOutputFormat.java:139)
at
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputFormat$PigRecordWriter.write(PigOutputFormat.java:98)
at
org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:537)
at
org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:88)
at
org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:99)
at
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapReduce$Reduce.runPipeline(PigGenericMapReduce.java:463)
at
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapReduce$Reduce.processOnePackageOutput(PigGenericMapReduce.java:428)
at
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapReduce$Reduce.reduce(PigGenericMapReduce.java:408)
at
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapReduce$Reduce.reduce(PigGenericMapReduce.java:262)
at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:164)
at
org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:595)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:433)
at org.apache.hadoop.mapred.Child$4.run(Child.java:268)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1332)
at org.apache.hadoop.mapred.Child.main(Child.java:262)
stderr logs
java.lang.RuntimeException: InternalCachedBag.spill() should not be called
at
org.apache.pig.data.InternalCachedBag.spill(InternalCachedBag.java:159)
at
org.apache.pig.impl.util.SpillableMemoryManager.handleNotification(SpillableMemoryManager.java:243)
at
sun.management.NotificationEmitterSupport.sendNotification(NotificationEmitterSupport.java:138)
at sun.management.MemoryImpl.createNotification(MemoryImpl.java:171)
at
sun.management.MemoryPoolImpl$PoolSensor.triggerAction(MemoryPoolImpl.java:272)
at sun.management.Sensor.trigger(Sensor.java:120)
Exception in thread "Low Memory Detector" java.lang.InternalError: Error in
invoking listener
at
sun.management.NotificationEmitterSupport.sendNotification(NotificationEmitterSupport.java:141)
at sun.management.MemoryImpl.createNotification(MemoryImpl.java:171)
at
sun.management.MemoryPoolImpl$PoolSensor.triggerAction(MemoryPoolImpl.java:272)
at sun.management.Sensor.trigger(Sensor.java:120)
and
> MultiStorage - java.lang.OutOfMemoryError: Java heap space
> ----------------------------------------------------------
>
> Key: PIG-3181
> URL: https://issues.apache.org/jira/browse/PIG-3181
> Project: Pig
> Issue Type: Bug
> Components: piggybank
> Affects Versions: 0.10.0
> Reporter: Fabian Alenius
>
> Hi, I have a script that looks like this:
> a = load '/input' as (f1, f2);
>
> a = group a by f1;
>
>
> a = foreach logs {
>
>
> generate group, a.f2;
>
>
> }
> store logs into '/output/' using
> org.apache.pig.piggybank.storage.MultiStorage('/output/', '0');
> But for some reason it fails with:
> FATAL org.apache.hadoop.mapred.Child: Error running child :
> java.lang.OutOfMemoryError: Java heap space
> at java.util.Arrays.copyOf(Arrays.java:2786)
> at java.io.ByteArrayOutputStream.write(ByteArrayOutputStream.java:94)
> at java.io.OutputStream.write(OutputStream.java:58)
> at org.apache.pig.impl.util.StorageUtil.putField(StorageUtil.java:145)
> at org.apache.pig.impl.util.StorageUtil.putField(StorageUtil.java:176)
> at org.apache.pig.impl.util.StorageUtil.putField(StorageUtil.java:194)
> at
> org.apache.pig.piggybank.storage.MultiStorage$MultiStorageOutputFormat$1.write(MultiStorage.java:208)
> at
> org.apache.pig.piggybank.storage.MultiStorage$MultiStorageOutputFormat$1.write(MultiStorage.java:187)
> at
> org.apache.pig.piggybank.storage.MultiStorage.putNext(MultiStorage.java:138)
> at
> org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputFormat$PigRecordWriter.write(PigOutputFormat.java:139)
> at
> org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputFormat$PigRecordWriter.write(PigOutputFormat.java:98)
> at
> org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:537)
> at
> org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:88)
> at
> org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:99)
> at
> org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapReduce$Reduce.runPipeline(PigGenericMapReduce.java:463)
> at
> org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapReduce$Reduce.processOnePackageOutput(PigGenericMapReduce.java:428)
> at
> org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapReduce$Reduce.reduce(PigGenericMapReduce.java:408)
> at
> org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapReduce$Reduce.reduce(PigGenericMapReduce.java:262)
> at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:164)
> at
> org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:595)
> at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:433)
> at org.apache.hadoop.mapred.Child$4.run(Child.java:268)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:396)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1332)
> at org.apache.hadoop.mapred.Child.main(Child.java:262)
> *stderr logs*
> java.lang.RuntimeException: InternalCachedBag.spill() should not be called
> at
> org.apache.pig.data.InternalCachedBag.spill(InternalCachedBag.java:159)
> at
> org.apache.pig.impl.util.SpillableMemoryManager.handleNotification(SpillableMemoryManager.java:243)
> at
> sun.management.NotificationEmitterSupport.sendNotification(NotificationEmitterSupport.java:138)
> at sun.management.MemoryImpl.createNotification(MemoryImpl.java:171)
> at
> sun.management.MemoryPoolImpl$PoolSensor.triggerAction(MemoryPoolImpl.java:272)
> at sun.management.Sensor.trigger(Sensor.java:120)
> Exception in thread "Low Memory Detector" java.lang.InternalError: Error in
> invoking listener
> at
> sun.management.NotificationEmitterSupport.sendNotification(NotificationEmitterSupport.java:141)
> at sun.management.MemoryImpl.createNotification(MemoryImpl.java:171)
> at
> sun.management.MemoryPoolImpl$PoolSensor.triggerAction(MemoryPoolImpl.java:272)
> at sun.management.Sensor.trigger(Sensor.java:120)
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
