[
https://issues.apache.org/jira/browse/CASSANDRA-18320?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17699752#comment-17699752
]
David Capwell edited comment on CASSANDRA-18320 at 3/13/23 5:35 PM:
--------------------------------------------------------------------
[~maedhroz] is rebasing accord branch and has an interesting stack trace
https://app.circleci.com/pipelines/github/maedhroz/cassandra/696/workflows/180e9555-6796-4733-8214-ba94fb9484fd/jobs/6936/tests#failed-test-0
{code}
java.lang.IllegalArgumentException: Incompatible file system
at org.apache.cassandra.io.util.File.<init>(File.java:135)
at
org.apache.cassandra.service.snapshot.SnapshotLoader.loadSnapshots(SnapshotLoader.java:173)
at
org.apache.cassandra.service.snapshot.SnapshotLoader.loadSnapshots(SnapshotLoader.java:189)
at
org.apache.cassandra.service.snapshot.SnapshotManager.loadSnapshots(SnapshotManager.java:119)
at
org.apache.cassandra.service.snapshot.SnapshotManager.start(SnapshotManager.java:88)
at
org.apache.cassandra.service.StorageService.startSnapshotManager(StorageService.java:1160)
at
org.apache.cassandra.distributed.impl.Instance.lambda$startup$12(Instance.java:712)
at org.apache.cassandra.concurrent.FutureTask$1.call(FutureTask.java:96)
at
org.apache.cassandra.concurrent.SyncFutureTask.run(SyncFutureTask.java:68)
at
org.apache.cassandra.simulator.systems.InterceptingExecutor$InterceptingPooledExecutor$WaitingThread.lambda$new$1(InterceptingExecutor.java:317)
at
io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.base/java.lang.Thread.run(Thread.java:829)
Suppressed: java.io.UncheckedIOException
at
org.apache.cassandra.io.util.PathUtils.propagateUnchecked(PathUtils.java:831)
at
org.apache.cassandra.io.util.PathUtils.propagateUnchecked(PathUtils.java:816)
at
org.apache.cassandra.io.util.PathUtils.delete(PathUtils.java:257)
at
org.apache.cassandra.io.util.PathUtils.deleteRecursive(PathUtils.java:381)
at
java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
at java.base/java.util.ArrayList.forEach(ArrayList.java:1541)
at
java.base/java.util.stream.SortedOps$RefSortingSink.end(SortedOps.java:395)
at
java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:485)
at
java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474)
at
java.base/java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
at
java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
at
java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
at
java.base/java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:497)
at
org.apache.cassandra.io.util.PathUtils.forEach(PathUtils.java:155)
at
org.apache.cassandra.io.util.PathUtils.deleteRecursive(PathUtils.java:378)
at
java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
at java.base/java.util.ArrayList.forEach(ArrayList.java:1541)
at
java.base/java.util.stream.SortedOps$RefSortingSink.end(SortedOps.java:395)
at
java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:485)
at
java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474)
at
java.base/java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
at
java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
at
java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
at
java.base/java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:497)
at
org.apache.cassandra.io.util.PathUtils.forEach(PathUtils.java:155)
at
org.apache.cassandra.io.util.PathUtils.deleteRecursive(PathUtils.java:378)
at
org.apache.cassandra.distributed.impl.AbstractCluster.close(AbstractCluster.java:1047)
at
org.apache.cassandra.simulator.ClusterSimulation.close(ClusterSimulation.java:819)
at
org.apache.cassandra.simulator.SimulationRunner$Run.run(SimulationRunner.java:370)
at
org.apache.cassandra.simulator.paxos.AccordSimulationRunner$Run.run(AccordSimulationRunner.java:39)
at
org.apache.cassandra.simulator.paxos.AccordSimulationRunner$Run.run(AccordSimulationRunner.java:30)
at
org.apache.cassandra.simulator.SimulationRunner$BasicCommand.run(SimulationRunner.java:355)
at
org.apache.cassandra.simulator.paxos.AccordSimulationRunner$Run.run(AccordSimulationRunner.java:30)
at
org.apache.cassandra.simulator.paxos.AccordSimulationRunner.main(AccordSimulationRunner.java:76)
at
org.apache.cassandra.simulator.test.ShortAccordSimulationTest.simulationTest(ShortAccordSimulationTest.java:32)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
Caused by: java.nio.file.DirectoryNotEmptyException:
/cassandra/node1/data0
at
com.google.common.jimfs.FileSystemView.checkEmpty(FileSystemView.java:535)
at
com.google.common.jimfs.FileSystemView.checkDeletable(FileSystemView.java:517)
at
com.google.common.jimfs.FileSystemView.delete(FileSystemView.java:479)
at
com.google.common.jimfs.FileSystemView.deleteFile(FileSystemView.java:465)
at
com.google.common.jimfs.JimfsFileSystemProvider.delete(JimfsFileSystemProvider.java:261)
at java.base/java.nio.file.Files.delete(Files.java:1142)
at
org.apache.cassandra.io.util.PathUtils.delete(PathUtils.java:252)
{code}
We still see this on
{code}
at
org.apache.cassandra.distributed.impl.AbstractCluster.close(AbstractCluster.java:1047)
at
org.apache.cassandra.simulator.ClusterSimulation.close(ClusterSimulation.java:819)
{code}
but, the top-level exception is
{code}
java.lang.IllegalArgumentException: Incompatible file system
at org.apache.cassandra.io.util.File.<init>(File.java:135)
at
org.apache.cassandra.service.snapshot.SnapshotLoader.loadSnapshots(SnapshotLoader.java:173)
at
org.apache.cassandra.service.snapshot.SnapshotLoader.loadSnapshots(SnapshotLoader.java:189)
at
org.apache.cassandra.service.snapshot.SnapshotManager.loadSnapshots(SnapshotManager.java:119)
at
org.apache.cassandra.service.snapshot.SnapshotManager.start(SnapshotManager.java:88)
at
org.apache.cassandra.service.StorageService.startSnapshotManager(StorageService.java:1160)
at
org.apache.cassandra.distributed.impl.Instance.lambda$startup$12(Instance.java:712)
at org.apache.cassandra.concurrent.FutureTask$1.call(FutureTask.java:96)
at
org.apache.cassandra.concurrent.SyncFutureTask.run(SyncFutureTask.java:68)
at
org.apache.cassandra.simulator.systems.InterceptingExecutor$InterceptingPooledExecutor$WaitingThread.lambda$new$1(InterceptingExecutor.java:317)
at
io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.base/java.lang.Thread.run(Thread.java:829)
{code}
since close error is suppressed, looks like startup fails and close sees a
write, so possible this stack trace may help show what is running while the
future is complete?
was (Author: dcapwell):
[~maedhroz] is rebasing accord branch and has an interesting stack trace
https://app.circleci.com/pipelines/github/maedhroz/cassandra/696/workflows/180e9555-6796-4733-8214-ba94fb9484fd/jobs/6936/tests#failed-test-0
{code}
java.lang.IllegalArgumentException: Incompatible file system
at org.apache.cassandra.io.util.File.<init>(File.java:135)
at
org.apache.cassandra.service.snapshot.SnapshotLoader.loadSnapshots(SnapshotLoader.java:173)
at
org.apache.cassandra.service.snapshot.SnapshotLoader.loadSnapshots(SnapshotLoader.java:189)
at
org.apache.cassandra.service.snapshot.SnapshotManager.loadSnapshots(SnapshotManager.java:119)
at
org.apache.cassandra.service.snapshot.SnapshotManager.start(SnapshotManager.java:88)
at
org.apache.cassandra.service.StorageService.startSnapshotManager(StorageService.java:1160)
at
org.apache.cassandra.distributed.impl.Instance.lambda$startup$12(Instance.java:712)
at org.apache.cassandra.concurrent.FutureTask$1.call(FutureTask.java:96)
at
org.apache.cassandra.concurrent.SyncFutureTask.run(SyncFutureTask.java:68)
at
org.apache.cassandra.simulator.systems.InterceptingExecutor$InterceptingPooledExecutor$WaitingThread.lambda$new$1(InterceptingExecutor.java:317)
at
io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.base/java.lang.Thread.run(Thread.java:829)
Suppressed: java.io.UncheckedIOException
at
org.apache.cassandra.io.util.PathUtils.propagateUnchecked(PathUtils.java:831)
at
org.apache.cassandra.io.util.PathUtils.propagateUnchecked(PathUtils.java:816)
at
org.apache.cassandra.io.util.PathUtils.delete(PathUtils.java:257)
at
org.apache.cassandra.io.util.PathUtils.deleteRecursive(PathUtils.java:381)
at
java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
at java.base/java.util.ArrayList.forEach(ArrayList.java:1541)
at
java.base/java.util.stream.SortedOps$RefSortingSink.end(SortedOps.java:395)
at
java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:485)
at
java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474)
at
java.base/java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
at
java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
at
java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
at
java.base/java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:497)
at
org.apache.cassandra.io.util.PathUtils.forEach(PathUtils.java:155)
at
org.apache.cassandra.io.util.PathUtils.deleteRecursive(PathUtils.java:378)
at
java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
at java.base/java.util.ArrayList.forEach(ArrayList.java:1541)
at
java.base/java.util.stream.SortedOps$RefSortingSink.end(SortedOps.java:395)
at
java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:485)
at
java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474)
at
java.base/java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
at
java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
at
java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
at
java.base/java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:497)
at
org.apache.cassandra.io.util.PathUtils.forEach(PathUtils.java:155)
at
org.apache.cassandra.io.util.PathUtils.deleteRecursive(PathUtils.java:378)
at
org.apache.cassandra.distributed.impl.AbstractCluster.close(AbstractCluster.java:1047)
at
org.apache.cassandra.simulator.ClusterSimulation.close(ClusterSimulation.java:819)
at
org.apache.cassandra.simulator.SimulationRunner$Run.run(SimulationRunner.java:370)
at
org.apache.cassandra.simulator.paxos.AccordSimulationRunner$Run.run(AccordSimulationRunner.java:39)
at
org.apache.cassandra.simulator.paxos.AccordSimulationRunner$Run.run(AccordSimulationRunner.java:30)
at
org.apache.cassandra.simulator.SimulationRunner$BasicCommand.run(SimulationRunner.java:355)
at
org.apache.cassandra.simulator.paxos.AccordSimulationRunner$Run.run(AccordSimulationRunner.java:30)
at
org.apache.cassandra.simulator.paxos.AccordSimulationRunner.main(AccordSimulationRunner.java:76)
at
org.apache.cassandra.simulator.test.ShortAccordSimulationTest.simulationTest(ShortAccordSimulationTest.java:32)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
Caused by: java.nio.file.DirectoryNotEmptyException:
/cassandra/node1/data0
at
com.google.common.jimfs.FileSystemView.checkEmpty(FileSystemView.java:535)
at
com.google.common.jimfs.FileSystemView.checkDeletable(FileSystemView.java:517)
at
com.google.common.jimfs.FileSystemView.delete(FileSystemView.java:479)
at
com.google.common.jimfs.FileSystemView.deleteFile(FileSystemView.java:465)
at
com.google.common.jimfs.JimfsFileSystemProvider.delete(JimfsFileSystemProvider.java:261)
at java.base/java.nio.file.Files.delete(Files.java:1142)
at
org.apache.cassandra.io.util.PathUtils.delete(PathUtils.java:252)
{code}
We still see this on
{code}
at
org.apache.cassandra.distributed.impl.AbstractCluster.close(AbstractCluster.java:1047)
at
org.apache.cassandra.simulator.ClusterSimulation.close(ClusterSimulation.java:819)
{code}
but, the top-level exception is
{code}
java.lang.IllegalArgumentException: Incompatible file system
at org.apache.cassandra.io.util.File.<init>(File.java:135)
at
org.apache.cassandra.service.snapshot.SnapshotLoader.loadSnapshots(SnapshotLoader.java:173)
at
org.apache.cassandra.service.snapshot.SnapshotLoader.loadSnapshots(SnapshotLoader.java:189)
at
org.apache.cassandra.service.snapshot.SnapshotManager.loadSnapshots(SnapshotManager.java:119)
at
org.apache.cassandra.service.snapshot.SnapshotManager.start(SnapshotManager.java:88)
at
org.apache.cassandra.service.StorageService.startSnapshotManager(StorageService.java:1160)
at
org.apache.cassandra.distributed.impl.Instance.lambda$startup$12(Instance.java:712)
at org.apache.cassandra.concurrent.FutureTask$1.call(FutureTask.java:96)
at
org.apache.cassandra.concurrent.SyncFutureTask.run(SyncFutureTask.java:68)
at
org.apache.cassandra.simulator.systems.InterceptingExecutor$InterceptingPooledExecutor$WaitingThread.lambda$new$1(InterceptingExecutor.java:317)
at
io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.base/java.lang.Thread.run(Thread.java:829)
{code}
so something is triggering startup while we are shutting down!
> simulationTest is flaky
> ------------------------
>
> Key: CASSANDRA-18320
> URL: https://issues.apache.org/jira/browse/CASSANDRA-18320
> Project: Cassandra
> Issue Type: Bug
> Components: CI
> Reporter: Ekaterina Dimitrova
> Priority: Normal
> Fix For: 4.1.x, 5.x
>
>
> {code}
> java.io.UncheckedIOException
> at
> org.apache.cassandra.io.util.PathUtils.propagateUnchecked(PathUtils.java:831)
> at
> org.apache.cassandra.io.util.PathUtils.propagateUnchecked(PathUtils.java:816)
> at org.apache.cassandra.io.util.PathUtils.delete(PathUtils.java:257)
> at
> org.apache.cassandra.io.util.PathUtils.deleteRecursive(PathUtils.java:381)
> at
> java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
> at java.util.ArrayList.forEach(ArrayList.java:1259)
> at java.util.stream.SortedOps$RefSortingSink.end(SortedOps.java:395)
> at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:483)
> at
> java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
> at
> java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
> at
> java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
> at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
> at
> java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:485)
> at org.apache.cassandra.io.util.PathUtils.forEach(PathUtils.java:155)
> at
> org.apache.cassandra.io.util.PathUtils.deleteRecursive(PathUtils.java:378)
> at
> java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
> at java.util.ArrayList.forEach(ArrayList.java:1259)
> at java.util.stream.SortedOps$RefSortingSink.end(SortedOps.java:395)
> at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:483)
> at
> java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
> at
> java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
> at
> java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
> at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
> at
> java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:485)
> at org.apache.cassandra.io.util.PathUtils.forEach(PathUtils.java:155)
> at
> org.apache.cassandra.io.util.PathUtils.deleteRecursive(PathUtils.java:378)
> at
> org.apache.cassandra.distributed.impl.AbstractCluster.close(AbstractCluster.java:1047)
> at
> org.apache.cassandra.simulator.ClusterSimulation.close(ClusterSimulation.java:816)
> at
> org.apache.cassandra.simulator.SimulationRunner$Run.run(SimulationRunner.java:370)
> at
> org.apache.cassandra.simulator.SimulationRunner$BasicCommand.run(SimulationRunner.java:345)
> at
> org.apache.cassandra.simulator.paxos.PaxosSimulationRunner.main(PaxosSimulationRunner.java:148)
> at
> org.apache.cassandra.simulator.test.ShortPaxosSimulationTest.simulationTest(ShortPaxosSimulationTest.java:33)
> Caused by: java.nio.file.DirectoryNotEmptyException:
> /cassandra/node1/commitlog
> at
> com.google.common.jimfs.FileSystemView.checkEmpty(FileSystemView.java:535)
> at
> com.google.common.jimfs.FileSystemView.checkDeletable(FileSystemView.java:517)
> at
> com.google.common.jimfs.FileSystemView.delete(FileSystemView.java:479)
> at
> com.google.common.jimfs.FileSystemView.deleteFile(FileSystemView.java:465)
> at
> com.google.common.jimfs.JimfsFileSystemProvider.delete(JimfsFileSystemProvider.java:261)
> at java.nio.file.Files.delete(Files.java:1126)
> at org.apache.cassandra.io.util.PathUtils.delete(PathUtils.java:252)
> {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
