Hi again,
apt-get install libjna-java installed nothing, I was already up to date.
I made the symbolic link jna.jar to target jna-3.4.1.jar (downloaded @
the given link) instead of jna-3.2.4.jar.
I could restart with the 'JNA mlockall successful' message.
I am still unable to snapshot my data.
I got the following output :
Exception in thread "main" java.io.IOError: java.io.IOException:
Unable to create hard link from
/raid0/cassandra/data/cassa_teads/stats_product-hc-233-Data.db to
/raid0/cassandra/data/cassa_teads/snapshots/1340099026781/stats_product-hc-233-Data.db
(errno 1)
at
org.apache.cassandra.db.ColumnFamilyStore.snapshotWithoutFlush(ColumnFamilyStore.java:1433)
at
org.apache.cassandra.db.ColumnFamilyStore.snapshot(ColumnFamilyStore.java:1462)
at org.apache.cassandra.db.Table.snapshot(Table.java:210)
at
org.apache.cassandra.service.StorageService.takeSnapshot(StorageService.java:1710)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:93)
at
com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:27)
at
com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(MBeanIntrospector.java:208)
at com.sun.jmx.mbeanserver.PerInterface.invoke(PerInterface.java:120)
at com.sun.jmx.mbeanserver.MBeanSupport.invoke(MBeanSupport.java:262)
at
com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(DefaultMBeanServerInterceptor.java:836)
at
com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(JmxMBeanServer.java:761)
at
javax.management.remote.rmi.RMIConnectionImpl.doOperation(RMIConnectionImpl.java:1427)
at
javax.management.remote.rmi.RMIConnectionImpl.access$200(RMIConnectionImpl.java:72)
at
javax.management.remote.rmi.RMIConnectionImpl$PrivilegedOperation.run(RMIConnectionImpl.java:1265)
at
javax.management.remote.rmi.RMIConnectionImpl.doPrivilegedOperation(RMIConnectionImpl.java:1360)
at
javax.management.remote.rmi.RMIConnectionImpl.invoke(RMIConnectionImpl.java:788)
at sun.reflect.GeneratedMethodAccessor42.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:303)
at sun.rmi.transport.Transport$1.run(Transport.java:159)
at java.security.AccessController.doPrivileged(Native Method)
at sun.rmi.transport.Transport.serviceCall(Transport.java:155)
at
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:535)
at
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:790)
at
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:649)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
Caused by: java.io.IOException: Unable to create hard link from
/raid0/cassandra/data/cassa_teads/stats_product-hc-233-Data.db to
/raid0/cassandra/data/cassa_teads/snapshots/1340099026781/stats_product-hc-233-Data.db
(errno 1)
at org.apache.cassandra.utils.CLibrary.createHardLink(CLibrary.java:158)
at
org.apache.cassandra.io.sstable.SSTableReader.createLinks(SSTableReader.java:857)
at
org.apache.cassandra.db.ColumnFamilyStore.snapshotWithoutFlush(ColumnFamilyStore.java:1412)
... 32 more
Logs tell me this :
ERROR 09:43:46,840 Unable to create hard link
com.sun.jna.LastErrorException: [1]ÃX
at org.apache.cassandra.utils.CLibrary.link(Native Method)
at org.apache.cassandra.utils.CLibrary.createHardLink(CLibrary.java:145)
at
org.apache.cassandra.io.sstable.SSTableReader.createLinks(SSTableReader.java:857)
at
org.apache.cassandra.db.ColumnFamilyStore.snapshotWithoutFlush(ColumnFamilyStore.java:1412)
at
org.apache.cassandra.db.ColumnFamilyStore.snapshot(ColumnFamilyStore.java:1462)
at org.apache.cassandra.db.Table.snapshot(Table.java:210)
at
org.apache.cassandra.service.StorageService.takeSnapshot(StorageService.java:1710)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:93)
at
com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:27)
at
com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(MBeanIntrospector.java:208)
at com.sun.jmx.mbeanserver.PerInterface.invoke(PerInterface.java:120)
at com.sun.jmx.mbeanserver.MBeanSupport.invoke(MBeanSupport.java:262)
at
com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(DefaultMBeanServerInterceptor.java:836)
at
com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(JmxMBeanServer.java:761)
at
javax.management.remote.rmi.RMIConnectionImpl.doOperation(RMIConnectionImpl.java:1427)
at
javax.management.remote.rmi.RMIConnectionImpl.access$200(RMIConnectionImpl.java:72)
at
javax.management.remote.rmi.RMIConnectionImpl$PrivilegedOperation.run(RMIConnectionImpl.java:1265)
at
javax.management.remote.rmi.RMIConnectionImpl.doPrivilegedOperation(RMIConnectionImpl.java:1360)
at
javax.management.remote.rmi.RMIConnectionImpl.invoke(RMIConnectionImpl.java:788)
at sun.reflect.GeneratedMethodAccessor42.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:303)
at sun.rmi.transport.Transport$1.run(Transport.java:159)
at java.security.AccessController.doPrivileged(Native Method)
at sun.rmi.transport.Transport.serviceCall(Transport.java:155)
at
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:535)
at
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:790)
at
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:649)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
The weird thing is that I have the folder "snapshot" created and an
sstable inside :
$ ll -h /raid0/cassandra/data/cassa_teads/snapshots/1340099026781/
total 56M
drwxr-xr-x 2 cassandra cassandra 4.0K 2012-06-19 09:43 ./
drwxr-xr-x 3 cassandra cassandra 26 2012-06-19 09:43 ../
-rw-r--r-- 2 cassandra cassandra 56M 2012-05-24 17:58
stats_product-hc-2372-Data.db
-rw-r--r-- 2 cassandra cassandra 71 2012-05-24 17:58
stats_product-hc-2372-Digest.sha1
-rw-r--r-- 2 cassandra cassandra 3.3K 2012-05-24 17:58
stats_product-hc-2372-Filter.db
-rw-r--r-- 2 cassandra cassandra 11K 2012-05-24 17:58
stats_product-hc-2372-Index.db
-rw-r--r-- 2 cassandra cassandra 4.3K 2012-05-24 17:58
stats_product-hc-2372-Statistics.db
Not being able to create snapshot is really annoying, any idea ?
Alain
2012/6/13 Joaquin Casares <joaq...@datastax.com>:
> Hello Alain,
>
> Yes, the AMI is geared to working "out of the box" for most dev purposes.
>
> We recently spotted an issue with JNA 3.2.7 on Ubuntu 10.10 not being picked
> up.
>
> You can try running `apt-get install libjna-java` but in order for the
> change to be activated, you must restart your Cassandra service.
>
> Could you try removing the older version in the /usr/share/java/ folder and
> restarting your Cassandra service? You should see a 'JNA mlockall
> successful' message in your system.log. If not, then remove the remaining
> jna.jar and replace it with the newest jna.jar as found
> here: https://github.com/twall/jna.
>
> Upon another restart of your node, you should see the 'JNA mlockall
> successful' message as well as the ability to run the snapshots.
>
> I was unable to replicate this on a new instance so all new launches should
> include the patched code.
>
> Do let me know if anyone else sees this issue.
>
> Thanks,
>
> Joaquin Casares
> DataStax
> Software Engineer/Support
>
>
>
> On Wed, Jun 13, 2012 at 10:28 AM, Alain RODRIGUEZ <arodr...@gmail.com>
> wrote:
>>
>> if I do : locate jna
>>
>> /opt/java/64/jdk1.6.0_31/db/docs/html/ref/rrefsqljnaturaljoin.html
>> /root/.m2/repository/net/java/dev/jna
>> /root/.m2/repository/net/java/dev/jna/jna
>> /root/.m2/repository/net/java/dev/jna/jna/3.2.7
>> /root/.m2/repository/net/java/dev/jna/jna/3.2.7/jna-3.2.7-sources.jar
>> /root/.m2/repository/net/java/dev/jna/jna/3.2.7/jna-3.2.7-sources.jar.sha1
>> /root/.m2/repository/net/java/dev/jna/jna/3.2.7/jna-3.2.7.jar
>> /root/.m2/repository/net/java/dev/jna/jna/3.2.7/jna-3.2.7.jar.sha1
>> /root/.m2/repository/net/java/dev/jna/jna/3.2.7/jna-3.2.7.pom
>> /root/.m2/repository/net/java/dev/jna/jna/3.2.7/jna-3.2.7.pom.sha1
>> /usr/share/doc/libjna-java
>> /usr/share/doc/libjna-java/README.Debian
>> /usr/share/doc/libjna-java/changelog.Debian.gz
>> /usr/share/doc/libjna-java/copyright
>> /usr/share/java/jna-3.2.4.jar
>> /usr/share/java/jna.jar
>> /usr/share/maven-repo/net/java/dev/jna
>> /usr/share/maven-repo/net/java/dev/jna/jna
>> /usr/share/maven-repo/net/java/dev/jna/jna/3.2.4
>> /usr/share/maven-repo/net/java/dev/jna/jna/debian
>> /usr/share/maven-repo/net/java/dev/jna/jna/3.2.4/jna-3.2.4.jar
>> /usr/share/maven-repo/net/java/dev/jna/jna/3.2.4/jna-3.2.4.pom
>> /usr/share/maven-repo/net/java/dev/jna/jna/debian/jna-debian.jar
>> /usr/share/maven-repo/net/java/dev/jna/jna/debian/jna-debian.pom
>> /var/cache/apt/archives/libjna-java_3.2.4-2_amd64.deb
>> /var/lib/dpkg/info/libjna-java.list
>> /var/lib/dpkg/info/libjna-java.md5sums
>>
>> So what version am I using (jna 3.2.7 or 3.2.4 ?)
>>
>> Should I do an "apt-get install libjna-java" and does this need a restart
>> ?
>>
>> Alain
>>
>> 2012/6/13 Alain RODRIGUEZ <arodr...@gmail.com>:
>> > Hi Aaron,
>> >
>> > We are using Ubuntu (AMI Datastax 1.0.9 as I said).
>> >
>> > Release: 10.10
>> > Codename: maverick
>> >
>> > ERROR [RMI TCP Connection(37732)-10.248.10.94] 2012-06-13 15:00:17,157
>> > CLibrary.java (line 153) Unable to create hard link
>> > com.sun.jna.LastErrorException: errno was 1
>> > at org.apache.cassandra.utils.CLibrary.link(Native Method)
>> > at
>> > org.apache.cassandra.utils.CLibrary.createHardLink(CLibrary.java:145)
>> > at
>> > org.apache.cassandra.io.sstable.SSTableReader.createLinks(SSTableReader.java:857)
>> > at
>> > org.apache.cassandra.db.ColumnFamilyStore.snapshotWithoutFlush(ColumnFamilyStore.java:1412)
>> > at
>> > org.apache.cassandra.db.ColumnFamilyStore.snapshot(ColumnFamilyStore.java:1462)
>> > at org.apache.cassandra.db.Table.snapshot(Table.java:210)
>> > at
>> > org.apache.cassandra.service.StorageService.takeSnapshot(StorageService.java:1710)
>> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> > at
>> > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
>> > at
>> > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
>> > at java.lang.reflect.Method.invoke(Method.java:597)
>> > at
>> > com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:93)
>> > at
>> > com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:27)
>> > at
>> > com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(MBeanIntrospector.java:208)
>> > at
>> > com.sun.jmx.mbeanserver.PerInterface.invoke(PerInterface.java:120)
>> > at
>> > com.sun.jmx.mbeanserver.MBeanSupport.invoke(MBeanSupport.java:262)
>> > at
>> > com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(DefaultMBeanServerInterceptor.java:836)
>> > at
>> > com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(JmxMBeanServer.java:761)
>> > at
>> > javax.management.remote.rmi.RMIConnectionImpl.doOperation(RMIConnectionImpl.java:1427)
>> > at
>> > javax.management.remote.rmi.RMIConnectionImpl.access$200(RMIConnectionImpl.java:72)
>> > at
>> > javax.management.remote.rmi.RMIConnectionImpl$PrivilegedOperation.run(RMIConnectionImpl.java:1265)
>> > at
>> > javax.management.remote.rmi.RMIConnectionImpl.doPrivilegedOperation(RMIConnectionImpl.java:1360)
>> > at
>> > javax.management.remote.rmi.RMIConnectionImpl.invoke(RMIConnectionImpl.java:788)
>> > at sun.reflect.GeneratedMethodAccessor46.invoke(Unknown Source)
>> > at
>> > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
>> > at java.lang.reflect.Method.invoke(Method.java:597)
>> > at
>> > sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:303)
>> > at sun.rmi.transport.Transport$1.run(Transport.java:159)
>> > at java.security.AccessController.doPrivileged(Native Method)
>> > at sun.rmi.transport.Transport.serviceCall(Transport.java:155)
>> > at
>> > sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:535)
>> > at
>> > sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:790)
>> > at
>> > sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:649)
>> > at
>> > java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
>> > at
>> > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
>> > at java.lang.Thread.run(Thread.java:662)
>> > WARN [RMI TCP Connection(37732)-10.248.10.94] 2012-06-13 15:00:17,158
>> > CLibrary.java (line 97) Obsolete version of JNA present; unable to
>> > read errno. Upgrade to JNA 3.2.7 or later
>> >
>> > I see that a JNA update seems necessary. Shouldn't this AMI be fully
>> > worrking "out of the box" ?
>> >
>> > I am going to search how to upgrade JNA.
>> >
>> > Alain
>> >
>> > 2012/5/31 aaron morton <aa...@thelastpickle.com>:
>> >> CASSANDRA-4230 is a bug in 1.1
>> >>
>> >> I am not aware of issues using snapshot on 1.0.9. But errno 0 is a bit
>> >> odd.
>> >>
>> >> On the server side there should be a log message at ERROR level that
>> >> contains the string "Unable to create hard link" and the error message.
>> >> What
>> >> does that say ?
>> >>
>> >> Can you also include the OS version.
>> >>
>> >> Cheers
>> >>
>> >> -----------------
>> >> Aaron Morton
>> >> Freelance Developer
>> >> @aaronmorton
>> >> http://www.thelastpickle.com
>> >>
>> >> On 28/05/2012, at 9:27 PM, Alain RODRIGUEZ wrote:
>> >>
>> >> I have the same error with the last Datastax AMI (1.0.9). Is that the
>> >> same
>> >> bug ?
>> >>
>> >> Requested snapshot for: cassa_teads
>> >> Exception in thread "main" java.io.IOError: java.io.IOException:
>> >> Unable to create hard link from
>> >> /raid0/cassandra/data/cassa_teads/stats_product-hc-233-Index.db to
>> >>
>> >> /raid0/cassandra/data/cassa_teads/snapshots/20120528/stats_product-hc-233-Index.db
>> >> (errno 0)
>> >> at
>> >>
>> >> org.apache.cassandra.db.ColumnFamilyStore.snapshotWithoutFlush(ColumnFamilyStore.java:1433)
>> >> at
>> >>
>> >> org.apache.cassandra.db.ColumnFamilyStore.snapshot(ColumnFamilyStore.java:1462)
>> >> at org.apache.cassandra.db.Table.snapshot(Table.java:210)
>> >> at
>> >>
>> >> org.apache.cassandra.service.StorageService.takeSnapshot(StorageService.java:1710)
>> >> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> >> at
>> >>
>> >> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
>> >> at
>> >>
>> >> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
>> >> at java.lang.reflect.Method.invoke(Method.java:597)
>> >> at
>> >>
>> >> com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:93)
>> >> at
>> >>
>> >> com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:27)
>> >> at
>> >>
>> >> com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(MBeanIntrospector.java:208)
>> >> at
>> >> com.sun.jmx.mbeanserver.PerInterface.invoke(PerInterface.java:120)
>> >> at
>> >> com.sun.jmx.mbeanserver.MBeanSupport.invoke(MBeanSupport.java:262)
>> >> at
>> >>
>> >> com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(DefaultMBeanServerInterceptor.java:836)
>> >> at
>> >> com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(JmxMBeanServer.java:761)
>> >> at
>> >>
>> >> javax.management.remote.rmi.RMIConnectionImpl.doOperation(RMIConnectionImpl.java:1427)
>> >> at
>> >>
>> >> javax.management.remote.rmi.RMIConnectionImpl.access$200(RMIConnectionImpl.java:72)
>> >> at
>> >>
>> >> javax.management.remote.rmi.RMIConnectionImpl$PrivilegedOperation.run(RMIConnectionImpl.java:1265)
>> >> at
>> >>
>> >> javax.management.remote.rmi.RMIConnectionImpl.doPrivilegedOperation(RMIConnectionImpl.java:1360)
>> >> at
>> >>
>> >> javax.management.remote.rmi.RMIConnectionImpl.invoke(RMIConnectionImpl.java:788)
>> >> at sun.reflect.GeneratedMethodAccessor50.invoke(Unknown Source)
>> >> at
>> >>
>> >> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
>> >> at java.lang.reflect.Method.invoke(Method.java:597)
>> >> at
>> >> sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:303)
>> >> at sun.rmi.transport.Transport$1.run(Transport.java:159)
>> >> at java.security.AccessController.doPrivileged(Native Method)
>> >> at sun.rmi.transport.Transport.serviceCall(Transport.java:155)
>> >> at
>> >>
>> >> sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:535)
>> >> at
>> >>
>> >> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:790)
>> >> at
>> >>
>> >> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:649)
>> >> at
>> >>
>> >> java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
>> >> at
>> >>
>> >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
>> >> at java.lang.Thread.run(Thread.java:662)
>> >> Caused by: java.io.IOException: Unable to create hard link from
>> >> /raid0/cassandra/data/cassa_teads/stats_product-hc-233-Index.db to
>> >>
>> >> /raid0/cassandra/data/cassa_teads/snapshots/20120528/stats_product-hc-233-Index.db
>> >> (errno 0)
>> >> at
>> >> org.apache.cassandra.utils.CLibrary.createHardLink(CLibrary.java:158)
>> >> at
>> >>
>> >> org.apache.cassandra.io.sstable.SSTableReader.createLinks(SSTableReader.java:857)
>> >> at
>> >>
>> >> org.apache.cassandra.db.ColumnFamilyStore.snapshotWithoutFlush(ColumnFamilyStore.java:1412)
>> >> ... 32 more
>> >>
>> >> Can we do a snapshot manually (like flushing and after copying all the
>> >> file into the snapshot folder) ?
>> >>
>> >> Alain
>> >>
>> >> 2012/5/19 Jonathan Ellis <jbel...@gmail.com>:
>> >>
>> >> When these bugs are fixed:
>> >>
>> >>
>> >> https://issues.apache.org/jira/secure/IssueNavigator.jspa?reset=true&jqlQuery=project+%3D+CASSANDRA+AND+fixVersion+%3D+%221.1.1%22+AND+resolution+%3D+Unresolved+ORDER+BY+due+ASC%2C+priority+DESC%2C+created+ASC&mode=hide
>> >>
>> >>
>> >> On Wed, May 16, 2012 at 6:35 PM, Bryan Fernandez
>> >> <bfernande...@gmail.com>
>> >> wrote:
>> >>
>> >> Does anyone know when 1.1.1 will be released?
>> >>
>> >>
>> >> Thanks.
>> >>
>> >>
>> >> On Tue, May 15, 2012 at 5:40 PM, Brandon Williams <dri...@gmail.com>
>> >> wrote:
>> >>
>> >>
>> >> Probably https://issues.apache.org/jira/browse/CASSANDRA-4230
>> >>
>> >>
>> >> On Tue, May 15, 2012 at 4:08 PM, Bryan Fernandez
>> >> <bfernande...@gmail.com>
>> >>
>> >> wrote:
>> >>
>> >> Greetings,
>> >>
>> >>
>> >> We recently upgraded from 1.0.8 to 1.1.0. Everything has been running
>> >>
>> >> fine
>> >>
>> >> with the exception of snapshots. When attempting to snapshot any of the
>> >>
>> >> nodes in our six node cluster we are seeing the following error.
>> >>
>> >>
>> >> [root@cassandra-n6 blotter]# /opt/apache-cassandra-1.1.0/bin/nodetool
>> >> -h
>> >>
>> >> 10.20.50.58 snapshot
>> >>
>> >> Requested snapshot for: all keyspaces
>> >>
>> >> Exception in thread "main" java.io.IOError: java.io.IOException: Unable
>> >>
>> >> to
>> >>
>> >> create hard link from
>> >>
>> >> /var/lib/cassandra/data/blotter/twitter_users/twitter_users.json to
>> >>
>> >>
>> >>
>> >> /var/lib/cassandra/data/blotter/twitter_users/snapshots/1337115022389/twitter_users.json
>> >>
>> >> (errno 17)
>> >>
>> >> at
>> >>
>> >>
>> >>
>> >> org.apache.cassandra.db.ColumnFamilyStore.snapshotWithoutFlush(ColumnFamilyStore.java:1454)
>> >>
>> >> at
>> >>
>> >>
>> >>
>> >> org.apache.cassandra.db.ColumnFamilyStore.snapshot(ColumnFamilyStore.java:1483)
>> >>
>> >> at org.apache.cassandra.db.Table.snapshot(Table.java:205)
>> >>
>> >> at
>> >>
>> >>
>> >>
>> >> org.apache.cassandra.service.StorageService.takeSnapshot(StorageService.java:1793)
>> >>
>> >> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> >>
>> >> at
>> >>
>> >>
>> >>
>> >> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
>> >>
>> >> at
>> >>
>> >>
>> >>
>> >> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
>> >>
>> >> at java.lang.reflect.Method.invoke(Method.java:597)
>> >>
>> >> at
>> >>
>> >>
>> >>
>> >> com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:93)
>> >>
>> >> at
>> >>
>> >>
>> >>
>> >> com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:27)
>> >>
>> >> at
>> >>
>> >>
>> >>
>> >> com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(MBeanIntrospector.java:208)
>> >>
>> >> at com.sun.jmx.mbeanserver.PerInterface.invoke(PerInterface.java:120)
>> >>
>> >> at com.sun.jmx.mbeanserver.MBeanSupport.invoke(MBeanSupport.java:262)
>> >>
>> >> at
>> >>
>> >>
>> >>
>> >> com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(DefaultMBeanServerInterceptor.java:836)
>> >>
>> >> at
>> >>
>> >> com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(JmxMBeanServer.java:761)
>> >>
>> >> at
>> >>
>> >>
>> >>
>> >> javax.management.remote.rmi.RMIConnectionImpl.doOperation(RMIConnectionImpl.java:1427)
>> >>
>> >> at
>> >>
>> >>
>> >>
>> >> javax.management.remote.rmi.RMIConnectionImpl.access$200(RMIConnectionImpl.java:72)
>> >>
>> >> at
>> >>
>> >>
>> >>
>> >> javax.management.remote.rmi.RMIConnectionImpl$PrivilegedOperation.run(RMIConnectionImpl.java:1265)
>> >>
>> >> at
>> >>
>> >>
>> >>
>> >> javax.management.remote.rmi.RMIConnectionImpl.doPrivilegedOperation(RMIConnectionImpl.java:1360)
>> >>
>> >> at
>> >>
>> >>
>> >>
>> >> javax.management.remote.rmi.RMIConnectionImpl.invoke(RMIConnectionImpl.java:788)
>> >>
>> >> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> >>
>> >> at
>> >>
>> >>
>> >>
>> >> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
>> >>
>> >> at
>> >>
>> >>
>> >>
>> >> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
>> >>
>> >> at java.lang.reflect.Method.invoke(Method.java:597)
>> >>
>> >> at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:303)
>> >>
>> >> at sun.rmi.transport.Transport$1.run(Transport.java:159)
>> >>
>> >> at java.security.AccessController.doPrivileged(Native Method)
>> >>
>> >> at sun.rmi.transport.Transport.serviceCall(Transport.java:155)
>> >>
>> >> at
>> >>
>> >>
>> >> sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:535)
>> >>
>> >> at
>> >>
>> >>
>> >>
>> >> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:790)
>> >>
>> >> at
>> >>
>> >>
>> >>
>> >> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:649)
>> >>
>> >> at
>> >>
>> >>
>> >>
>> >> java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
>> >>
>> >> at
>> >>
>> >>
>> >>
>> >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
>> >>
>> >> at java.lang.Thread.run(Thread.java:662)
>> >>
>> >> Caused by: java.io.IOException: Unable to create hard link from
>> >>
>> >> /var/lib/cassandra/data/blotter/twitter_users/twitter_users.json to
>> >>
>> >>
>> >>
>> >> /var/lib/cassandra/data/blotter/twitter_users/snapshots/1337115022389/twitter_users.json
>> >>
>> >> (errno 17)
>> >>
>> >> at
>> >> org.apache.cassandra.utils.CLibrary.createHardLink(CLibrary.java:163)
>> >>
>> >> at
>> >>
>> >>
>> >>
>> >> org.apache.cassandra.db.Directories.snapshotLeveledManifest(Directories.java:343)
>> >>
>> >> at
>> >>
>> >>
>> >>
>> >> org.apache.cassandra.db.ColumnFamilyStore.snapshotWithoutFlush(ColumnFamilyStore.java:1450)
>> >>
>> >> ... 33 more
>> >>
>> >>
>> >>
>> >> However, an LS shows that both of these JSON files exist on the
>> >>
>> >> filesystem
>> >>
>> >> (although slightly different sizes).
>> >>
>> >>
>> >> [root@cassandra-n6 blotter]# ls -al
>> >>
>> >> /var/lib/cassandra/data/blotter/twitter_users/twitter_users.json
>> >>
>> >> -rw-r--r-- 1 root root 38786 May 15 20:51
>> >>
>> >> /var/lib/cassandra/data/blotter/twitter_users/twitter_users.json
>> >>
>> >>
>> >> [root@cassandra-n6 blotter]# ls -al
>> >>
>> >>
>> >>
>> >> /var/lib/cassandra/data/blotter/twitter_users/snapshots/1337115022389/twitter_users.json
>> >>
>> >> -rw-r--r-- 1 root root 38778 May 15 20:50
>> >>
>> >>
>> >>
>> >> /var/lib/cassandra/data/blotter/twitter_users/snapshots/1337115022389/twitter_users.json
>> >>
>> >>
>> >>
>> >> We are using Leveled Compaction on the twitter_users CF with I assume
>> >> is
>> >>
>> >> creating the JSON files.
>> >>
>> >>
>> >> [root@cassandra-n6 blotter]# ls -al
>> >>
>> >> /var/lib/cassandra/data/blotter/twitter_users/*.json
>> >>
>> >> -rw-r--r-- 1 root root 38779 May 15 20:51
>> >>
>> >> /var/lib/cassandra/data/blotter/twitter_users/twitter_users.json
>> >>
>> >> -rw-r--r-- 1 root root 38779 May 15 20:51
>> >>
>> >> /var/lib/cassandra/data/blotter/twitter_users/twitter_users-old.json
>> >>
>> >> -rw-r--r-- 1 root root 1040 May 15 20:51
>> >>
>> >>
>> >>
>> >> /var/lib/cassandra/data/blotter/twitter_users/twitter_users.twitter_user_attributes_screenname_idx.json
>> >>
>> >> -rw-r--r-- 1 root root 1046 May 15 20:50
>> >>
>> >>
>> >>
>> >> /var/lib/cassandra/data/blotter/twitter_users/twitter_users.twitter_user_attributes_screenname_idx-old.json
>> >>
>> >>
>> >>
>> >> The other column families which are not using Leveled Compaction seem
>> >> to
>> >>
>> >> have their snapshots created successfully.
>> >>
>> >>
>> >> Any ideas other than turning off Leveled Compaction?
>> >>
>> >>
>> >>
>> >> Thanks,
>> >>
>> >>
>> >> Brian
>> >>
>> >>
>> >>
>> >>
>> >>
>> >>
>> >>
>> >>
>> >>
>> >>
>> >> --
>> >>
>> >> Jonathan Ellis
>> >>
>> >> Project Chair, Apache Cassandra
>> >>
>> >> co-founder of DataStax, the source for professional Cassandra support
>> >>
>> >> http://www.datastax.com
>> >>
>> >>
>
>
