[
https://issues.apache.org/jira/browse/IGNITE-3312?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Ivan Veselovsky updated IGNITE-3312:
------------------------------------
Description:
HadoopMapReduceEmbeddedSelfTest.testMultiReducerWholeMapReduceExecution fails
with ~20% probability .
Failure cause is either the 1st or 2nd marked line in the following code
(org.apache.ignite.internal.processors.igfs.IgfsMetaManager#create) :
{code}
// Check: can we overwrite it?
if (!overwrite)
throw new
IgfsPathAlreadyExistsException("Failed to create a file: " + path); // ***** #1
// Check if file already opened for write.
if (oldInfo.lockId() != null)
throw new IgfsException("File is already opened
for write: " + path); // ***** #2
{code}
Diagnostic shows that the same file really attempted to be created several
times on one thread.
Situation #2 reproducible with the following stack when IGFS is used as a
secondary fs:
{code}
Hadoop-task-efea2ae1-09cb-4c49-9465-dcbbedee1835_1-REDUCE-2-0-#679%hadoop.HadoopMapReduceEmbeddedSelfTest2%@20675,
prio=5, in group 'ignite', status: 'RUNNING'
at
org.apache.ignite.internal.processors.igfs.IgfsMetaManager.create(IgfsMetaManager.java:2857)
at
org.apache.ignite.internal.processors.igfs.IgfsImpl$15.call(IgfsImpl.java:1051)
at
org.apache.ignite.internal.processors.igfs.IgfsImpl$15.call(IgfsImpl.java:1019)
at
org.apache.ignite.internal.processors.igfs.IgfsImpl.safeOp(IgfsImpl.java:1823)
at
org.apache.ignite.internal.processors.igfs.IgfsImpl.create0(IgfsImpl.java:1019)
at
org.apache.ignite.internal.processors.igfs.IgfsImpl.create(IgfsImpl.java:990)
at
org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsInProc$15.apply(HadoopIgfsInProc.java:359)
at
org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsInProc$15.apply(HadoopIgfsInProc.java:357)
at
org.apache.ignite.igfs.IgfsUserContext.doAs(IgfsUserContext.java:49)
at
org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsInProc.create(HadoopIgfsInProc.java:357)
at
org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsWrapper$15.apply(HadoopIgfsWrapper.java:258)
at
org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsWrapper$15.apply(HadoopIgfsWrapper.java:255)
at
org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsWrapper.withReconnectHandling(HadoopIgfsWrapper.java:310)
at
org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsWrapper.create(HadoopIgfsWrapper.java:255)
at
org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem.create(IgniteHadoopFileSystem.java:632)
at
org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem.create(IgniteHadoopIgfsSecondaryFileSystem.java:406)
at
org.apache.ignite.internal.processors.igfs.IgfsSecondaryFileSystemCreateContext.create(IgfsSecondaryFileSystemCreateContext.java:87)
at
org.apache.ignite.internal.processors.igfs.IgfsMetaManager.create(IgfsMetaManager.java:2922)
at
org.apache.ignite.internal.processors.igfs.IgfsImpl$15.call(IgfsImpl.java:1051)
at
org.apache.ignite.internal.processors.igfs.IgfsImpl$15.call(IgfsImpl.java:1019)
at
org.apache.ignite.internal.processors.igfs.IgfsImpl.safeOp(IgfsImpl.java:1823)
at
org.apache.ignite.internal.processors.igfs.IgfsImpl.create0(IgfsImpl.java:1019)
at
org.apache.ignite.internal.processors.igfs.IgfsImpl.create(IgfsImpl.java:990)
at
org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsInProc$15.apply(HadoopIgfsInProc.java:359)
at
org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsInProc$15.apply(HadoopIgfsInProc.java:357)
at
org.apache.ignite.igfs.IgfsUserContext.doAs(IgfsUserContext.java:54)
at
org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsInProc.create(HadoopIgfsInProc.java:357)
at
org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsWrapper$15.apply(HadoopIgfsWrapper.java:258)
at
org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsWrapper$15.apply(HadoopIgfsWrapper.java:255)
at
org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsWrapper.withReconnectHandling(HadoopIgfsWrapper.java:310)
at
org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsWrapper.create(HadoopIgfsWrapper.java:255)
at
org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem.create(IgniteHadoopFileSystem.java:632)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:906)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:799)
at
org.apache.hadoop.mapred.TextOutputFormat.getRecordWriter(TextOutputFormat.java:123)
at
org.apache.ignite.internal.processors.hadoop.v1.HadoopV1OutputCollector.<init>(HadoopV1OutputCollector.java:68)
at
org.apache.ignite.internal.processors.hadoop.v1.HadoopV1Task$1.<init>(HadoopV1Task.java:73)
at
org.apache.ignite.internal.processors.hadoop.v1.HadoopV1Task.collector(HadoopV1Task.java:72)
at
org.apache.ignite.internal.processors.hadoop.v1.HadoopV1ReduceTask.run(HadoopV1ReduceTask.java:65)
at
org.apache.ignite.internal.processors.hadoop.v2.HadoopV2TaskContext.run(HadoopV2TaskContext.java:249)
at
org.apache.ignite.internal.processors.hadoop.taskexecutor.HadoopRunnableTask.runTask(HadoopRunnableTask.java:201)
at
org.apache.ignite.internal.processors.hadoop.taskexecutor.HadoopRunnableTask.call0(HadoopRunnableTask.java:144)
at
org.apache.ignite.internal.processors.hadoop.taskexecutor.HadoopRunnableTask$1.call(HadoopRunnableTask.java:116)
at
org.apache.ignite.internal.processors.hadoop.taskexecutor.HadoopRunnableTask$1.call(HadoopRunnableTask.java:114)
at
org.apache.ignite.internal.processors.hadoop.v2.HadoopV2TaskContext.runAsJobOwner(HadoopV2TaskContext.java:544)
at
org.apache.ignite.internal.processors.hadoop.taskexecutor.HadoopRunnableTask.call(HadoopRunnableTask.java:114)
at
org.apache.ignite.internal.processors.hadoop.taskexecutor.HadoopRunnableTask.call(HadoopRunnableTask.java:46)
at
org.apache.ignite.internal.processors.hadoop.taskexecutor.HadoopExecutorService$2.body(HadoopExecutorService.java:186)
at
org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110)
at java.lang.Thread.run(Thread.java:745)
{code}
was:
HadoopMapReduceEmbeddedSelfTest.testMultiReducerWholeMapReduceExecution fails
with ~20% probability .
Failure cause is either the 1st or 2nd marked line in the following code
(org.apache.ignite.internal.processors.igfs.IgfsMetaManager#create) :
{code}
// Check: can we overwrite it?
if (!overwrite)
throw new
IgfsPathAlreadyExistsException("Failed to create a file: " + path); // ***** #1
// Check if file already opened for write.
if (oldInfo.lockId() != null)
throw new IgfsException("File is already opened
for write: " + path); // ***** #2
{code}
Diagnostic shows that the same file really attempted to be created several
times on one thread.
> [Test]
> HadoopMapReduceEmbeddedSelfTest.testMultiReducerWholeMapReduceExecution
> flakily fails.
> ---------------------------------------------------------------------------------------------
>
> Key: IGNITE-3312
> URL: https://issues.apache.org/jira/browse/IGNITE-3312
> Project: Ignite
> Issue Type: Bug
> Components: IGFS
> Reporter: Ivan Veselovsky
> Assignee: Ivan Veselovsky
> Labels: test
>
> HadoopMapReduceEmbeddedSelfTest.testMultiReducerWholeMapReduceExecution fails
> with ~20% probability .
> Failure cause is either the 1st or 2nd marked line in the following code
> (org.apache.ignite.internal.processors.igfs.IgfsMetaManager#create) :
> {code}
> // Check: can we overwrite it?
> if (!overwrite)
> throw new
> IgfsPathAlreadyExistsException("Failed to create a file: " + path); // *****
> #1
> // Check if file already opened for write.
> if (oldInfo.lockId() != null)
> throw new IgfsException("File is already
> opened for write: " + path); // ***** #2
> {code}
> Diagnostic shows that the same file really attempted to be created several
> times on one thread.
> Situation #2 reproducible with the following stack when IGFS is used as a
> secondary fs:
> {code}
> Hadoop-task-efea2ae1-09cb-4c49-9465-dcbbedee1835_1-REDUCE-2-0-#679%hadoop.HadoopMapReduceEmbeddedSelfTest2%@20675,
> prio=5, in group 'ignite', status: 'RUNNING'
> at
> org.apache.ignite.internal.processors.igfs.IgfsMetaManager.create(IgfsMetaManager.java:2857)
> at
> org.apache.ignite.internal.processors.igfs.IgfsImpl$15.call(IgfsImpl.java:1051)
> at
> org.apache.ignite.internal.processors.igfs.IgfsImpl$15.call(IgfsImpl.java:1019)
> at
> org.apache.ignite.internal.processors.igfs.IgfsImpl.safeOp(IgfsImpl.java:1823)
> at
> org.apache.ignite.internal.processors.igfs.IgfsImpl.create0(IgfsImpl.java:1019)
> at
> org.apache.ignite.internal.processors.igfs.IgfsImpl.create(IgfsImpl.java:990)
> at
> org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsInProc$15.apply(HadoopIgfsInProc.java:359)
> at
> org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsInProc$15.apply(HadoopIgfsInProc.java:357)
> at
> org.apache.ignite.igfs.IgfsUserContext.doAs(IgfsUserContext.java:49)
> at
> org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsInProc.create(HadoopIgfsInProc.java:357)
> at
> org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsWrapper$15.apply(HadoopIgfsWrapper.java:258)
> at
> org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsWrapper$15.apply(HadoopIgfsWrapper.java:255)
> at
> org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsWrapper.withReconnectHandling(HadoopIgfsWrapper.java:310)
> at
> org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsWrapper.create(HadoopIgfsWrapper.java:255)
> at
> org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem.create(IgniteHadoopFileSystem.java:632)
> at
> org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem.create(IgniteHadoopIgfsSecondaryFileSystem.java:406)
> at
> org.apache.ignite.internal.processors.igfs.IgfsSecondaryFileSystemCreateContext.create(IgfsSecondaryFileSystemCreateContext.java:87)
> at
> org.apache.ignite.internal.processors.igfs.IgfsMetaManager.create(IgfsMetaManager.java:2922)
> at
> org.apache.ignite.internal.processors.igfs.IgfsImpl$15.call(IgfsImpl.java:1051)
> at
> org.apache.ignite.internal.processors.igfs.IgfsImpl$15.call(IgfsImpl.java:1019)
> at
> org.apache.ignite.internal.processors.igfs.IgfsImpl.safeOp(IgfsImpl.java:1823)
> at
> org.apache.ignite.internal.processors.igfs.IgfsImpl.create0(IgfsImpl.java:1019)
> at
> org.apache.ignite.internal.processors.igfs.IgfsImpl.create(IgfsImpl.java:990)
> at
> org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsInProc$15.apply(HadoopIgfsInProc.java:359)
> at
> org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsInProc$15.apply(HadoopIgfsInProc.java:357)
> at
> org.apache.ignite.igfs.IgfsUserContext.doAs(IgfsUserContext.java:54)
> at
> org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsInProc.create(HadoopIgfsInProc.java:357)
> at
> org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsWrapper$15.apply(HadoopIgfsWrapper.java:258)
> at
> org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsWrapper$15.apply(HadoopIgfsWrapper.java:255)
> at
> org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsWrapper.withReconnectHandling(HadoopIgfsWrapper.java:310)
> at
> org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsWrapper.create(HadoopIgfsWrapper.java:255)
> at
> org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem.create(IgniteHadoopFileSystem.java:632)
> at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:906)
> at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:799)
> at
> org.apache.hadoop.mapred.TextOutputFormat.getRecordWriter(TextOutputFormat.java:123)
> at
> org.apache.ignite.internal.processors.hadoop.v1.HadoopV1OutputCollector.<init>(HadoopV1OutputCollector.java:68)
> at
> org.apache.ignite.internal.processors.hadoop.v1.HadoopV1Task$1.<init>(HadoopV1Task.java:73)
> at
> org.apache.ignite.internal.processors.hadoop.v1.HadoopV1Task.collector(HadoopV1Task.java:72)
> at
> org.apache.ignite.internal.processors.hadoop.v1.HadoopV1ReduceTask.run(HadoopV1ReduceTask.java:65)
> at
> org.apache.ignite.internal.processors.hadoop.v2.HadoopV2TaskContext.run(HadoopV2TaskContext.java:249)
> at
> org.apache.ignite.internal.processors.hadoop.taskexecutor.HadoopRunnableTask.runTask(HadoopRunnableTask.java:201)
> at
> org.apache.ignite.internal.processors.hadoop.taskexecutor.HadoopRunnableTask.call0(HadoopRunnableTask.java:144)
> at
> org.apache.ignite.internal.processors.hadoop.taskexecutor.HadoopRunnableTask$1.call(HadoopRunnableTask.java:116)
> at
> org.apache.ignite.internal.processors.hadoop.taskexecutor.HadoopRunnableTask$1.call(HadoopRunnableTask.java:114)
> at
> org.apache.ignite.internal.processors.hadoop.v2.HadoopV2TaskContext.runAsJobOwner(HadoopV2TaskContext.java:544)
> at
> org.apache.ignite.internal.processors.hadoop.taskexecutor.HadoopRunnableTask.call(HadoopRunnableTask.java:114)
> at
> org.apache.ignite.internal.processors.hadoop.taskexecutor.HadoopRunnableTask.call(HadoopRunnableTask.java:46)
> at
> org.apache.ignite.internal.processors.hadoop.taskexecutor.HadoopExecutorService$2.body(HadoopExecutorService.java:186)
> at
> org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110)
> at java.lang.Thread.run(Thread.java:745)
> {code}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
