Thanks for the quick reply. Am I correct in assuming that the bad inode is "/dev/mapper/limsd_archp1?"
[root@ausracdbd01 tmp]# cat debug_cmds.txt ls -l //orphan_dir:0000 ls -l //orphan_dir:0001 ls -l //orphan_dir:0002 ls -l //orphan_dir:0003 [root@ausracdbd01 tmp]# cat orphans.sh #!/bin/sh while read DEVICE do echo Working on $DEVICE debugfs.ocfs2 -f ./debug_cmds.txt $DEVICE done < ocfs2_vols.txt [root@ausracdbd01 tmp]# sh orphans.sh Working on /dev/mapper/ph1tp1 debugfs.ocfs2 1.4.4 debugfs: ls -l //orphan_dir:0000 12 drwxr-xr-x 2 0 0 4096 1-Sep-2008 09:40 . 6 drwxr-xr-x 6 0 0 4096 25-Jun-2008 11:51 .. debugfs: ls -l //orphan_dir:0001 13 drwxr-xr-x 2 0 0 4096 16-Jul-2008 08:18 . 6 drwxr-xr-x 6 0 0 4096 25-Jun-2008 11:51 .. debugfs: ls -l //orphan_dir:0002 14 drwxr-xr-x 2 0 0 4096 16-Jul-2008 12:48 . 6 drwxr-xr-x 6 0 0 4096 25-Jun-2008 11:51 .. debugfs: ls -l //orphan_dir:0003 15 drwxr-xr-x 2 0 0 4096 12-Nov-2008 16:04 . 6 drwxr-xr-x 6 0 0 4096 25-Jun-2008 11:51 .. Working on /dev/mapper/ph1t_archp1 debugfs.ocfs2 1.4.4 debugfs: ls -l //orphan_dir:0000 12 drwxr-xr-x 2 0 0 4096 14-Sep-2011 11:30 . 6 drwxr-xr-x 6 0 0 4096 25-Jun-2008 11:51 .. debugfs: ls -l //orphan_dir:0001 13 drwxr-xr-x 2 0 0 8192 23-Aug-2011 10:00 . 6 drwxr-xr-x 6 0 0 4096 25-Jun-2008 11:51 .. debugfs: ls -l //orphan_dir:0002 14 drwxr-xr-x 2 0 0 4096 21-Oct-2008 11:30 . 6 drwxr-xr-x 6 0 0 4096 25-Jun-2008 11:51 .. debugfs: ls -l //orphan_dir:0003 15 drwxr-xr-x 2 0 0 8192 7-Nov-2008 08:06 . 6 drwxr-xr-x 6 0 0 4096 25-Jun-2008 11:51 .. Working on /dev/mapper/limstp1 debugfs.ocfs2 1.4.4 debugfs: ls -l //orphan_dir:0000 12 drwxr-xr-x 2 0 0 4096 18-Aug-2011 12:58 . 6 drwxr-xr-x 6 0 0 4096 25-Jun-2008 11:53 .. debugfs: ls -l //orphan_dir:0001 13 drwxr-xr-x 2 0 0 4096 25-Jun-2008 11:53 . 6 drwxr-xr-x 6 0 0 4096 25-Jun-2008 11:53 .. debugfs: ls -l //orphan_dir:0002 14 drwxr-xr-x 2 0 0 4096 25-Jun-2008 11:53 . 6 drwxr-xr-x 6 0 0 4096 25-Jun-2008 11:53 .. debugfs: ls -l //orphan_dir:0003 15 drwxr-xr-x 2 0 0 4096 25-Jun-2008 11:53 . 6 drwxr-xr-x 6 0 0 4096 25-Jun-2008 11:53 .. Working on /dev/mapper/limst_archp1 debugfs.ocfs2 1.4.4 debugfs: ls -l //orphan_dir:0000 12 drwxr-xr-x 2 0 0 4096 14-Sep-2011 11:30 . 6 drwxr-xr-x 6 0 0 4096 25-Jun-2008 11:54 .. debugfs: ls -l //orphan_dir:0001 13 drwxr-xr-x 2 0 0 8192 18-Aug-2011 12:53 . 6 drwxr-xr-x 6 0 0 4096 25-Jun-2008 11:54 .. debugfs: ls -l //orphan_dir:0002 14 drwxr-xr-x 2 0 0 4096 21-Oct-2008 11:30 . 6 drwxr-xr-x 6 0 0 4096 25-Jun-2008 11:54 .. debugfs: ls -l //orphan_dir:0003 15 drwxr-xr-x 2 0 0 4096 21-Oct-2008 11:30 . 6 drwxr-xr-x 6 0 0 4096 25-Jun-2008 11:54 .. Working on /dev/mapper/limsdp1 debugfs.ocfs2 1.4.4 debugfs: ls -l //orphan_dir:0000 12 drwxr-xr-x 2 0 0 4096 3-Jun-2008 16:59 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:58 .. debugfs: ls -l //orphan_dir:0001 13 drwxr-xr-x 2 0 0 4096 21-Jun-2008 17:39 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:58 .. debugfs: ls -l //orphan_dir:0002 14 drwxr-xr-x 2 0 0 4096 22-May-2008 11:58 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:58 .. debugfs: ls -l //orphan_dir:0003 15 drwxr-xr-x 2 0 0 4096 22-May-2008 11:58 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:58 .. Working on /dev/mapper/limsd_archp1 debugfs.ocfs2 1.4.4 debugfs: ls -l //orphan_dir:0000 12 drwxr-xr-x 2 0 0 4096 14-Sep-2011 11:30 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:59 .. debugfs: ls -l //orphan_dir:0001 13 drwxr-xr-x 2 0 0 4096 6-Oct-2009 11:30 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:59 .. 66040 -rw-rw---- 0 503 505 0 12-Sep-2008 22:40 00000000000101f8 debugfs: ls -l //orphan_dir:0002 14 drwxr-xr-x 2 0 0 4096 22-May-2008 11:59 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:59 .. debugfs: ls -l //orphan_dir:0003 15 drwxr-xr-x 2 0 0 4096 22-May-2008 11:59 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:59 .. Working on /dev/mapper/casep1 debugfs.ocfs2 1.4.4 debugfs: ls -l //orphan_dir:0000 12 drwxr-xr-x 2 0 0 4096 9-Jun-2008 13:54 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:56 .. debugfs: ls -l //orphan_dir:0001 13 drwxr-xr-x 2 0 0 4096 22-May-2008 11:56 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:56 .. debugfs: ls -l //orphan_dir:0002 14 drwxr-xr-x 2 0 0 4096 22-May-2008 11:56 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:56 .. debugfs: ls -l //orphan_dir:0003 15 drwxr-xr-x 2 0 0 4096 22-May-2008 11:56 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:56 .. Working on /dev/mapper/case_archp1 debugfs.ocfs2 1.4.4 debugfs: ls -l //orphan_dir:0000 12 drwxr-xr-x 2 0 0 4096 14-Sep-2011 11:30 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:57 .. debugfs: ls -l //orphan_dir:0001 13 drwxr-xr-x 2 0 0 4096 6-Oct-2009 11:30 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:57 .. debugfs: ls -l //orphan_dir:0002 14 drwxr-xr-x 2 0 0 4096 22-May-2008 11:57 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:57 .. debugfs: ls -l //orphan_dir:0003 15 drwxr-xr-x 2 0 0 4096 22-May-2008 11:57 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:57 .. Working on /dev/mapper/cretp1 debugfs.ocfs2 1.4.4 debugfs: ls -l //orphan_dir:0000 12 drwxr-xr-x 2 0 0 4096 3-Jun-2008 17:47 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:57 .. debugfs: ls -l //orphan_dir:0001 13 drwxr-xr-x 2 0 0 4096 30-Aug-2009 14:55 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:57 .. debugfs: ls -l //orphan_dir:0002 14 drwxr-xr-x 2 0 0 4096 22-May-2008 11:57 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:57 .. debugfs: ls -l //orphan_dir:0003 15 drwxr-xr-x 2 0 0 4096 22-May-2008 11:57 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:57 .. Working on /dev/mapper/cret_archp1 debugfs.ocfs2 1.4.4 debugfs: ls -l //orphan_dir:0000 12 drwxr-xr-x 2 0 0 4096 14-Sep-2011 11:30 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:57 .. debugfs: ls -l //orphan_dir:0001 13 drwxr-xr-x 2 0 0 4096 16-Aug-2011 16:43 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:57 .. debugfs: ls -l //orphan_dir:0002 14 drwxr-xr-x 2 0 0 4096 22-May-2008 11:57 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:57 .. debugfs: ls -l //orphan_dir:0003 15 drwxr-xr-x 2 0 0 4096 22-May-2008 11:57 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:57 .. Working on /dev/mapper/credp1 debugfs.ocfs2 1.4.4 debugfs: ls -l //orphan_dir:0000 12 drwxr-xr-x 2 0 0 4096 12-Dec-2008 13:41 . 6 drwxr-xr-x 6 0 0 4096 21-Nov-2008 10:54 .. debugfs: ls -l //orphan_dir:0001 13 drwxr-xr-x 2 0 0 4096 21-Nov-2008 10:54 . 6 drwxr-xr-x 6 0 0 4096 21-Nov-2008 10:54 .. debugfs: ls -l //orphan_dir:0002 14 drwxr-xr-x 2 0 0 4096 21-Nov-2008 10:54 . 6 drwxr-xr-x 6 0 0 4096 21-Nov-2008 10:54 .. debugfs: ls -l //orphan_dir:0003 15 drwxr-xr-x 2 0 0 4096 21-Nov-2008 10:54 . 6 drwxr-xr-x 6 0 0 4096 21-Nov-2008 10:54 .. Working on /dev/mapper/cred_archp1 debugfs.ocfs2 1.4.4 debugfs: ls -l //orphan_dir:0000 12 drwxr-xr-x 2 0 0 4096 14-Sep-2011 11:30 . 6 drwxr-xr-x 6 0 0 4096 21-Nov-2008 10:55 .. debugfs: ls -l //orphan_dir:0001 13 drwxr-xr-x 2 0 0 4096 5-Jul-2011 22:01 . 6 drwxr-xr-x 6 0 0 4096 21-Nov-2008 10:55 .. debugfs: ls -l //orphan_dir:0002 14 drwxr-xr-x 2 0 0 4096 12-Dec-2008 11:30 . 6 drwxr-xr-x 6 0 0 4096 21-Nov-2008 10:55 .. debugfs: ls -l //orphan_dir:0003 15 drwxr-xr-x 2 0 0 4096 1-Dec-2008 11:30 . 6 drwxr-xr-x 6 0 0 4096 21-Nov-2008 10:55 .. Working on /dev/mapper/mpath35p1 debugfs.ocfs2 1.4.4 debugfs: ls -l //orphan_dir:0000 12 drwxr-xr-x 2 0 0 4096 18-Aug-2011 12:58 . 6 drwxr-xr-x 6 0 0 4096 25-Jun-2008 11:53 .. debugfs: ls -l //orphan_dir:0001 13 drwxr-xr-x 2 0 0 4096 25-Jun-2008 11:53 . 6 drwxr-xr-x 6 0 0 4096 25-Jun-2008 11:53 .. debugfs: ls -l //orphan_dir:0002 14 drwxr-xr-x 2 0 0 4096 25-Jun-2008 11:53 . 6 drwxr-xr-x 6 0 0 4096 25-Jun-2008 11:53 .. debugfs: ls -l //orphan_dir:0003 15 drwxr-xr-x 2 0 0 4096 25-Jun-2008 11:53 . 6 drwxr-xr-x 6 0 0 4096 25-Jun-2008 11:53 .. Working on /dev/mapper/ph1dp1 debugfs.ocfs2 1.4.4 debugfs: ls -l //orphan_dir:0000 12 drwxr-xr-x 2 0 0 4096 5-Jun-2008 14:50 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 12:01 .. debugfs: ls -l //orphan_dir:0001 13 drwxr-xr-x 2 0 0 4096 21-Jun-2008 16:42 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 12:01 .. debugfs: ls -l //orphan_dir:0002 14 drwxr-xr-x 2 0 0 4096 22-May-2008 12:01 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 12:01 .. debugfs: ls -l //orphan_dir:0003 15 drwxr-xr-x 2 0 0 4096 22-May-2008 12:01 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 12:01 .. Working on /dev/mapper/ph1d_archp1 debugfs.ocfs2 1.4.4 debugfs: ls -l //orphan_dir:0000 12 drwxr-xr-x 2 0 0 12288 14-Sep-2011 11:30 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 12:01 .. debugfs: ls -l //orphan_dir:0001 13 drwxr-xr-x 2 0 0 8192 23-Mar-2010 18:03 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 12:01 .. debugfs: ls -l //orphan_dir:0002 14 drwxr-xr-x 2 0 0 4096 22-May-2008 12:01 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 12:01 .. debugfs: ls -l //orphan_dir:0003 15 drwxr-xr-x 2 0 0 4096 22-May-2008 12:01 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 12:01 .. Working on /dev/mapper/mpath21p1 debugfs.ocfs2 1.4.4 debugfs: ls -l //orphan_dir:0000 12 drwxr-xr-x 2 0 0 4096 5-Jun-2008 14:50 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 12:01 .. debugfs: ls -l //orphan_dir:0001 13 drwxr-xr-x 2 0 0 4096 21-Jun-2008 16:42 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 12:01 .. debugfs: ls -l //orphan_dir:0002 14 drwxr-xr-x 2 0 0 4096 22-May-2008 12:01 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 12:01 .. debugfs: ls -l //orphan_dir:0003 15 drwxr-xr-x 2 0 0 4096 22-May-2008 12:01 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 12:01 .. Working on /dev/mapper/mpath20p1 debugfs.ocfs2 1.4.4 debugfs: ls -l //orphan_dir:0000 12 drwxr-xr-x 2 0 0 4096 3-Jun-2008 16:59 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:58 .. debugfs: ls -l //orphan_dir:0001 13 drwxr-xr-x 2 0 0 4096 21-Jun-2008 17:39 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:58 .. debugfs: ls -l //orphan_dir:0002 14 drwxr-xr-x 2 0 0 4096 22-May-2008 11:58 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:58 .. debugfs: ls -l //orphan_dir:0003 15 drwxr-xr-x 2 0 0 4096 22-May-2008 11:58 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:58 .. Working on /dev/mapper/mpath18p1 debugfs.ocfs2 1.4.4 debugfs: ls -l //orphan_dir:0000 12 drwxr-xr-x 2 0 0 4096 9-Jun-2008 13:54 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:56 .. debugfs: ls -l //orphan_dir:0001 13 drwxr-xr-x 2 0 0 4096 22-May-2008 11:56 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:56 .. debugfs: ls -l //orphan_dir:0002 14 drwxr-xr-x 2 0 0 4096 22-May-2008 11:56 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:56 .. debugfs: ls -l //orphan_dir:0003 15 drwxr-xr-x 2 0 0 4096 22-May-2008 11:56 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:56 .. Working on /dev/mapper/mpath19p1 debugfs.ocfs2 1.4.4 debugfs: ls -l //orphan_dir:0000 12 drwxr-xr-x 2 0 0 4096 3-Jun-2008 17:47 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:57 .. debugfs: ls -l //orphan_dir:0001 13 drwxr-xr-x 2 0 0 4096 30-Aug-2009 14:55 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:57 .. debugfs: ls -l //orphan_dir:0002 14 drwxr-xr-x 2 0 0 4096 22-May-2008 11:57 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:57 .. debugfs: ls -l //orphan_dir:0003 15 drwxr-xr-x 2 0 0 4096 22-May-2008 11:57 . 6 drwxr-xr-x 6 0 0 4096 22-May-2008 11:57 .. Working on /dev/mapper/mpath33p1 debugfs.ocfs2 1.4.4 debugfs: ls -l //orphan_dir:0000 12 drwxr-xr-x 2 0 0 4096 12-Dec-2008 13:41 . 6 drwxr-xr-x 6 0 0 4096 21-Nov-2008 10:54 .. debugfs: ls -l //orphan_dir:0001 13 drwxr-xr-x 2 0 0 4096 21-Nov-2008 10:54 . 6 drwxr-xr-x 6 0 0 4096 21-Nov-2008 10:54 .. debugfs: ls -l //orphan_dir:0002 14 drwxr-xr-x 2 0 0 4096 21-Nov-2008 10:54 . 6 drwxr-xr-x 6 0 0 4096 21-Nov-2008 10:54 .. debugfs: ls -l //orphan_dir:0003 15 drwxr-xr-x 2 0 0 4096 21-Nov-2008 10:54 . 6 drwxr-xr-x 6 0 0 4096 21-Nov-2008 10:54 .. [root@ausracdbd01 tmp]# ________________________________ From: Sunil Mushran [mailto:sunil.mush...@oracle.com] Sent: Thursday, September 15, 2011 10:04 AM To: Daniel Keisling Cc: ocfs2-users@oss.oracle.com Subject: Re: [Ocfs2-users] Syslog reports (ocfs2_wq, 15527, 2):ocfs2_orphan_del:1841 ERROR: status = -2 The issue that caused it has been fixed. The fix is here. http://oss.oracle.com/git/?p=ocfs2-1.4.git;a=commit;h=b6f3de3fd54026df74 8bfd1449bbe31b9803f8f7 The actual problem could have happened much earlier. 1.4.4 is showing the messages as it is more aggressive (than 1.4.1) in cleaning up the orphans. By default, the fs scans for orphans once every 10 mins on a node in the cluster. fsck should fix it. I would have to think you must not have fscked that volume. You can use debugfs.ocfs2 to look at the orphan dirs. List the system dir using "ls-l //". Then list orphan dirs using "ls -l //orphan_dir:0000", etc. Look at the timestamp on the inodes. The one that is oldest could be the problematic one. That way you know which volume to fsck. BTW, it is safe to run debugfs while the fs is mounted. At worst it will provide you stale info. On 09/15/2011 07:40 AM, Daniel Keisling wrote: Hello, I recently upgraded from OCFS2 v1.4.1 running on RHEL 5.1 with kernel-2.6.18-92.1.13.el5 to OCFS2 v1.4.4 running on RHEL 5.6 with kernel-2.6.18-194.32.1.el5. I now see this is syslog every couple of minutes: Sep 15 09:31:51 ausracdbd01 kernel: (ocfs2_wq,15527,2):ocfs2_orphan_del:1841 ERROR: status = -2 Sep 15 09:31:51 ausracdbd01 kernel: (ocfs2_wq,15527,2):ocfs2_remove_inode:628 ERROR: status = -2 Sep 15 09:31:51 ausracdbd01 kernel: (ocfs2_wq,15527,2):ocfs2_wipe_inode:754 ERROR: status = -2 Sep 15 09:31:51 ausracdbd01 kernel: (ocfs2_wq,15527,2):ocfs2_delete_inode:999 ERROR: status = -2 This is happening on all cluster nodes, on 3 separate OCFS2 clusters. I have performed a 'fsck.ocfs2 -f -y <device>' on _most_ of the filesystems, but not all. fsck has always came back clean. Does anyone know the source and fix for this error? Thanks, Daniel Daniel Keisling Sr. Systems Administrator II Information Technology PPD 7551 Metro Center Drive, Suite 300 Austin, TX 78744 Phone +1 512 747 5256 Cell +1 512 653 1895 Fax +1 512 685 7256 e-mail daniel.keisl...@ppdi.com <blocked::mailto:daniel.keisl...@ppdi.com> Web site www.ppdi.com <blocked::http://www.ppdi.com/> This email transmission and any documents, files or previous email messages attached to it may contain information that is confidential or legally privileged. If you are not the intended recipient or a person responsible for delivering this transmission to the intended recipient, you are hereby notified that you must not read this transmission and that any disclosure, copying, printing, distribution or use of this transmission is strictly prohibited. If you have received this transmission in error, please immediately notify the sender by telephone or return email and delete the original transmission and its attachments without reading or saving in any manner. _______________________________________________ Ocfs2-users mailing list Ocfs2-users@oss.oracle.com http://oss.oracle.com/mailman/listinfo/ocfs2-users This email transmission and any documents, files or previous email messages attached to it may contain information that is confidential or legally privileged. If you are not the intended recipient or a person responsible for delivering this transmission to the intended recipient, you are hereby notified that you must not read this transmission and that any disclosure, copying, printing, distribution or use of this transmission is strictly prohibited. If you have received this transmission in error, please immediately notify the sender by telephone or return email and delete the original transmission and its attachments without reading or saving in any manner. _______________________________________________ Ocfs2-users mailing list Ocfs2-users@oss.oracle.com http://oss.oracle.com/mailman/listinfo/ocfs2-users