Hi Pranith,
I have attached two source files: one for the reader (reader.cpp) and one for
the writer (writer.cpp). I run those simultaneously on two different nodes.
The test relies on the existence of a small file (<1000 bytes) called "target"
under /mnt/gv0. Alternatively you can modify the source files to point to
another location.
I have four nodes: one primary server, one replicate secondary server, one for
the reader (client node) and one for the writer (client node).
The gluster file system is physically located on a single disk local to the
primary server.
Let me know if you need more details.
Thanks,
Louis
Le 06/08/14, Pranith Kumar Karampuri <[email protected]> a écrit :
>
>
>
>
> Louis,
> Do you have a small test case or something we can try to re-create the issue?
>
> Pranith
>
> On 08/06/2014 08:33 PM, [email protected] wrote:
>
>
> > Hi,
> >
> > Version: gluster 3.4.2
> > Use case: on one node a target file is updated while another node is
> > reading it.
> > Issue: only about half of the read succeeds.
> > The strategy elected is to write to a temporary file then rename the
> > temporary file to the target file.
> > During file renaming the gluster log from the reading node complains about
> > stale file handle amongst other things:
> >
> > [2014-08-06 13:54:24.653951] W [fuse-resolve.c:147:fuse_resolve_gfid_cbk]
> > 0-fuse: f98aa46f-0e37-49bc-b578-8e70bec57a7f: failed to resolve (Invalid
> > argument)
> > [2014-08-06 13:54:24.653983] E [fuse-bridge.c:1996:fuse_open_resume]
> > 0-glusterfs-fuse: 428919: OPEN f98aa46f-0e37-49bc-b578-8e70bec57a7f
> > resolution failed
> > [2014-08-06 13:54:24.754741] W
> > [client-rpc-fops.c:2697:client3_3_readv_cbk] 0-gv0-client-0: remote
> > operation failed: No such file or directory
> > [2014-08-06 13:54:24.755263] W
> > [client-rpc-fops.c:1179:client3_3_fgetxattr_cbk] 0-gv0-client-0: remote
> > operation failed: No such file or directory
> > [2014-08-06 13:54:24.755302] E
> > [dht-helper.c:725:dht_migration_complete_check_task] 0-gv0-dht: (null):
> > failed to get the 'linkto' xattr No such file or directory
> > [2014-08-06 13:54:24.755347] W [fuse-bridge.c:2089:fuse_readv_cbk]
> > 0-glusterfs-fuse: 428921: READ => -1 (No such file or directory)
> > [2014-08-06 13:54:24.755687] W
> > [client-rpc-fops.c:2697:client3_3_readv_cbk] 0-gv0-client-0: remote
> > operation failed: No such file or directory
> > [2014-08-06 13:54:24.756193] W
> > [client-rpc-fops.c:1179:client3_3_fgetxattr_cbk] 0-gv0-client-0: remote
> > operation failed: No such file or directory
> > [2014-08-06 13:54:24.756232] E
> > [dht-helper.c:725:dht_migration_complete_check_task] 0-gv0-dht: (null):
> > failed to get the 'linkto' xattr No such file or directory
> > [2014-08-06 13:54:24.756267] W [fuse-bridge.c:2089:fuse_readv_cbk]
> > 0-glusterfs-fuse: 428922: READ => -1 (No such file or directory)
> > [2014-08-06 13:54:24.756794] W
> > [client-rpc-fops.c:2697:client3_3_readv_cbk] 0-gv0-client-0: remote
> > operation failed: No such file or directory
> > [2014-08-06 13:54:24.757292] W
> > [client-rpc-fops.c:1179:client3_3_fgetxattr_cbk] 0-gv0-client-0: remote
> > operation failed: No such file or directory
> > [2014-08-06 13:54:24.757324] E
> > [dht-helper.c:725:dht_migration_complete_check_task] 0-gv0-dht: (null):
> > failed to get the 'linkto' xattr No such file or directory
> > [2014-08-06 13:54:24.757356] W [fuse-bridge.c:2089:fuse_readv_cbk]
> > 0-glusterfs-fuse: 428924: READ => -1 (No such file or directory)
> > [2014-08-06 13:54:24.958465] W
> > [client-rpc-fops.c:2697:client3_3_readv_cbk] 0-gv0-client-0: remote
> > operation failed: No such file or directory
> > [2014-08-06 13:54:24.959020] W
> > [client-rpc-fops.c:1179:client3_3_fgetxattr_cbk] 0-gv0-client-0: remote
> > operation failed: No such file or directory
> > [2014-08-06 13:54:24.959059] E
> > [dht-helper.c:725:dht_migration_complete_check_task] 0-gv0-dht: (null):
> > failed to get the 'linkto' xattr No such file or directory
> > [2014-08-06 13:54:24.959100] W [fuse-bridge.c:2089:fuse_readv_cbk]
> > 0-glusterfs-fuse: 428929: READ => -1 (No such file or directory)
> > [2014-08-06 13:54:24.959420] W
> > [client-rpc-fops.c:2697:client3_3_readv_cbk] 0-gv0-client-0: remote
> > operation failed: No such file or directory
> > [2014-08-06 13:54:24.959885] W
> > [client-rpc-fops.c:1179:client3_3_fgetxattr_cbk] 0-gv0-client-0: remote
> > operation failed: No such file or directory
> > [2014-08-06 13:54:24.959912] E
> > [dht-helper.c:725:dht_migration_complete_check_task] 0-gv0-dht: (null):
> > failed to get the 'linkto' xattr No such file or directory
> > [2014-08-06 13:54:24.959949] W [fuse-bridge.c:2089:fuse_readv_cbk]
> > 0-glusterfs-fuse: 428930: READ => -1 (No such file or directory)
> > [2014-08-06 13:54:24.960420] W
> > [client-rpc-fops.c:2697:client3_3_readv_cbk] 0-gv0-client-0: remote
> > operation failed: No such file or directory
> > [2014-08-06 13:54:24.960940] W
> > [client-rpc-fops.c:1179:client3_3_fgetxattr_cbk] 0-gv0-client-0: remote
> > operation failed: No such file or directory
> > [2014-08-06 13:54:24.960966] E
> > [dht-helper.c:725:dht_migration_complete_check_task] 0-gv0-dht: (null):
> > failed to get the 'linkto' xattr No such file or directory
> > [2014-08-06 13:54:24.960993] W [fuse-bridge.c:2089:fuse_readv_cbk]
> > 0-glusterfs-fuse: 428932: READ => -1 (No such file or directory)
> > [2014-08-06 13:54:25.161921] W [client-rpc-fops.c:526:client3_3_stat_cbk]
> > 0-gv0-client-0: remote operation failed: No such file or directory
> > [2014-08-06 13:54:25.263287] W [client-rpc-fops.c:526:client3_3_stat_cbk]
> > 0-gv0-client-0: remote operation failed: No such file or directory
> > [2014-08-06 13:54:25.364792] W [client-rpc-fops.c:526:client3_3_stat_cbk]
> > 0-gv0-client-0: remote operation failed: No such file or directory
> > [2014-08-06 13:54:25.466796] W [client-rpc-fops.c:526:client3_3_stat_cbk]
> > 0-gv0-client-0: remote operation failed: No such file or directory
> > [2014-08-06 13:54:25.568329] W
> > [client-rpc-fops.c:2624:client3_3_lookup_cbk] 0-gv0-client-0: remote
> > operation failed: Stale file handle. Path: /target_file
> > (f98aa46f-0e37-49bc-b578-8e70bec57a7f)
> > [2014-08-06 13:54:25.568381] W
> > [client-rpc-fops.c:2624:client3_3_lookup_cbk] 0-gv0-client-1: remote
> > operation failed: Stale file handle. Path: /target_file
> > (f98aa46f-0e37-49bc-b578-8e70bec57a7f)
> >
> > I tried using RSYNC_FRIENDLY_NAME so that the file name hashing is the
> > same for the temporary and target file (all my temporary file follows the
> > rsync pattern ".<target filename>.tmp") but the issue remains.
> >
> > Is there a way to make the "renaming" atomic? If not what is the best
> > strategy to use when a node tries to read a file while another tries to
> > update it?
> >
> > Thanks,
> >
> > Louis
> >
> >
> >
> > _______________________________________________
> > Gluster-users mailing list
> > [email protected]
> > http://supercolony.gluster.org/mailman/listinfo/gluster-users
> >
>
>
>
>
>
#include <string.h>
#include <iostream>
#include <stdio.h>
#include <errno.h>
#include <unistd.h>
#include <time.h>
using namespace std;
const string gFileName ("/mnt/gv0/target");
int main()
{
int lErrno = 0;
unsigned long lSourceFileSize = 0;
char * lpSourceBuffer = NULL;
FILE* lpSourceFile = fopen(gFileName.c_str (), "rb");
if (lpSourceFile == NULL)
{
lErrno = errno;
cout << "Cannot open file " << gFileName << ": " << strerror (lErrno) << endl;
}
else
{
if (fseeko64 (lpSourceFile, 0L, SEEK_END) == 0)
{
lSourceFileSize = ftello64 (lpSourceFile);
rewind (lpSourceFile);
if (fseeko64 (lpSourceFile, 0, SEEK_SET) == 0)
{
lpSourceBuffer = new char [lSourceFileSize];
unsigned long lBytesRead = fread (lpSourceBuffer, 1, lSourceFileSize, lpSourceFile);
if (lBytesRead != lSourceFileSize)
{
lErrno = errno;
cout << "Cannot read all bytes from file " << gFileName << " " << lBytesRead << "/" << lSourceFileSize << ": feof " << feof (lpSourceFile) << ", ferror " << ferror (lpSourceFile) << ": " << strerror(lErrno) << endl;
}
}
else
{
lErrno = errno;
cout << "Cannot seek to beginning of file " << gFileName << ": " << strerror (lErrno) << endl;
}
}
else
{
lErrno = errno;
cout << "Cannot seek to end of file " << gFileName << ": " << strerror (lErrno) << endl;
}
fclose (lpSourceFile);
}
if (lpSourceBuffer == NULL)
{
return -1;
}
while (1)
{
std:string lFilePath = gFileName;
std::string lTmpFilePath = "/mnt/gv0/.target.tmp";
FILE* lpTmpFile = fopen (lTmpFilePath.c_str (), "wb");
if (lpTmpFile)
{
if (fwrite (lpSourceBuffer, 1, lSourceFileSize, lpTmpFile) == lSourceFileSize)
{
if (fclose (lpTmpFile) != 0)
{
int lError = errno;
cout << " Fail to flush temporary file " << lTmpFilePath << ": " << strerror (lError);
}
else if (rename (lTmpFilePath.c_str (), lFilePath.c_str ()) == -1)
{
int lError = errno;
cout << " Fail to write file " << lFilePath << ": " << strerror (lError);
}
}
else
{
fclose (lpTmpFile);
int lError = errno;
cout << " Fail to write temporary file " << lTmpFilePath << ": " << strerror (lError);
}
}
else
{
int lError = errno;
cout << " Fail to open temporary file " << lTmpFilePath << ": " << strerror (lError);
}
}
cout << "Hello World!" << endl;
return 0;
}
#include <string.h>
#include <iomanip>
#include <iostream>
#include <stdio.h>
#include <errno.h>
#include <time.h>
#include <sys/time.h>
using namespace std;
const string gFileName ("/mnt/gv0/target");
int main()
{
timespec lReadDelay;
timespec lRemainingDelay;
lReadDelay.tv_sec = 0;
lReadDelay.tv_nsec = 100000000;
while (1)
{
int lErrno = 0;
unsigned long lFileSize = 0;
FILE* lpFile = fopen(gFileName.c_str (), "rb");
if (lpFile == NULL)
{
lErrno = errno;
cout << "Cannot open file " << gFileName << ": " << strerror (lErrno) << endl;
}
else
{
if (fseeko64 (lpFile, 0L, SEEK_END) == 0)
{
lFileSize = ftello64 (lpFile);
rewind (lpFile);
unsigned long lBytesRead = 0;
if (fseeko64 (lpFile, 0, SEEK_SET) == 0)
{
char * lpBuffer = new char [lFileSize];
lBytesRead = fread (lpBuffer, 1, lFileSize, lpFile);
if (lBytesRead != lFileSize)
{
cout << "Cannot read all bytes from file " << gFileName << " " << lBytesRead << "/" << lFileSize << ": feof " << feof (lpFile) << ", ferror " << ferror (lpFile) << ": " << strerror(lErrno) << " read offset " << ftello64 (lpFile) << endl;
}
}
else
{
lErrno = errno;
cout << "Cannot seek to beginning of file " << gFileName << ": " << strerror (lErrno) << endl;
}
}
else
{
lErrno = errno;
cout << "Cannot seek to end of file " << gFileName << ": " << strerror (lErrno) << endl;
}
fclose (lpFile);
}
nanosleep (&lReadDelay, &lRemainingDelay);
}
return 0;
}
_______________________________________________
Gluster-users mailing list
[email protected]
http://supercolony.gluster.org/mailman/listinfo/gluster-users
