Hello. I've setup the UFO stack as described in the documentation but I'm running into a problem. I have a script that inserts a large number of objects with size < 4MB, and after it has inserted a few thousand files the API suddenly stops responding. PUTs fail after a 60s wait with either a 404 or 503 result.
This is what I see in the logs:
Feb 24 16:35:08 vmdev-gluster03 account-server 127.0.0.1 - -
[24/Feb/2012:16:35:08 +0000] "HEAD
/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/0/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620"
204 - "txd1be8077fb3e4c1e870dcee493dd8681" "-" "-" 0 .2608 ""
Feb 24 16:36:08 vmdev-gluster03 proxy-server ERROR with Container server
127.0.0.1:6011/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620 re: Trying to get
container info for /AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/blocks10: Timeout
(60s) (t xn: txd1be8077fb3e4c1e870dcee493dd8681)
Feb 24 16:36:08 vmdev-gluster03 proxy-server 10.135.7.165 10.135.7.165
24/Feb/2012/16/36/08 PUT
/v1/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/blocks10/treta HTTP/1.0 404 -
curl/7.21.0%20%28x86_64-pc-linux-gnu%29%20libcurl/7.21.0%20O
penSSL/0.9.8o%20zlib/1.2.3.4%20libidn/1.15%20libssh2/1.2.6
TESTER%3Asapodisco%2CAUTH_tk5934e6d77edf42ebb388ee5bc4cfe788 - - -
txd1be8077fb3e4c1e870dcee493dd8681 - 60.2707
Feb 24 16:36:08 vmdev-gluster03 proxy-server STDOUT: 10.135.7.165 - -
[24/Feb/2012 16:36:08] code 400, message Bad request syntax ('Feb 22 06:32:57
vmdev-gluster02 mpt-statusd: detected non-optimal RAID status') (txn:
txd1be8077 fb3e4c1e870dcee493dd8681)
Feb 24 16:36:08 vmdev-gluster03 proxy-server STDOUT: 10.135.7.165 - -
[24/Feb/2012 16:36:08] "Feb 22 06:32:57 vmdev-gluster02 mpt-statusd: detected
non-optimal RAID status" 400 - (txn: txd1be8077fb3e4c1e870dcee493dd8681)
Feb 24 16:36:16 vmdev-gluster03 container-server 127.0.0.1 - -
[24/Feb/2012:16:36:16 +0000] "HEAD
/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/0/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/blocks10"
204 - "txd1be8077fb3e4c1e870dcee493dd8681 " "-" "-" 67.5196
So between my PUTs some process that runs locally executes a HEAD request
which, since the container has a few thousand files, takes too long to respond
and the request fails. Which process send this HEAD request?
When the container-server receives this HEAD request it starts scanning the
whole dir, as expected:
stat("/mnt/gluster-object/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/blocks10/5/V/m/5VmTFnaQ6G5WL0klYlRjMizCBzZSme4_-VNrRnP35lhef3yzIxyvzjaG_NnerWeFrtlgHf80g4VnmP0DS2G32g",
{st_mode=S_IFREG|0600, st_size=37768, ...}) = 0
getxattr("/mnt/gluster-object/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/blocks10/5/V/m/5VmTFnaQ6G5WL0klYlRjMizCBzZSme4_-VNrRnP35lhef3yzIxyvzjaG_NnerWeFrtlgHf80g4VnmP0DS2G32g",
"user.swift.metadata", 0x0, 0) = 207
getxattr("/mnt/gluster-object/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/blocks10/5/V/m/5VmTFnaQ6G5WL0klYlRjMizCBzZSme4_-VNrRnP35lhef3yzIxyvzjaG_NnerWeFrtlgHf80g4VnmP0DS2G32g",
"user.swift.metadata",
"\x80\x02}q\x01(U\x10X-Content-Lengthq\x02U\x0537768U\x0eX-Content-Typeq\x03U\x18application/octet-streamq\x04U\x0bX-Timestampq\x05U\x101330097164.20775q\x06U\x0dX-Object-Typeq\x07U\x04fileq\x08U\x06X-Typeq\x09U\x06Objectq\x0aU\x06X-ETagq\x0bU
837ed1a98765ac2d471f499f5c44f618q\x0cu.", 207) = 207
getxattr("/mnt/gluster-object/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/blocks10/5/V/m/5VmTFnaQ6G5WL0klYlRjMizCBzZSme4_-VNrRnP35lhef3yzIxyvzjaG_NnerWeFrtlgHf80g4VnmP0DS2G32g",
"user.swift.metadata1", 0x0, 0) = -1 ENODATA (No data available)
stat("/mnt/gluster-object/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/blocks10/5/V/m/5VmTFnaQ6G5WL0klYlRjMizCBzZSme4_-VNrRnP35lhef3yzIxyvzjaG_NnerWeFrtlgHf80g4VnmP0DS2G32g",
{st_mode=S_IFREG|0600, st_size=37768, ...}) = 0
lstat("/mnt/gluster-object/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/blocks10/5/o",
{st_mode=S_IFDIR|0777, st_size=44, ...}) = 0
open("/mnt/gluster-object/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/blocks10/5/o",
O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 11
getdents(11, /* 4 entries */, 32768) = 96
getdents(11, /* 0 entries */, 32768) = 0
close(11) = 0
stat("/mnt/gluster-object/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/blocks10/5/o/A",
{st_mode=S_IFDIR|0777, st_size=105, ...}) = 0
stat("/mnt/gluster-object/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/blocks10/5/o/t",
{st_mode=S_IFDIR|0777, st_size=198, ...}) = 0
getxattr("/mnt/gluster-object/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/blocks10/5/o/A",
"user.swift.metadata", 0x0, 0) = 194
getxattr("/mnt/gluster-object/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/blocks10/5/o/A",
"user.swift.metadata",
"\x80\x02}q\x01(U\x10X-Content-Lengthq\x02K\x00U\x0eX-Content-Typeq\x03U\x15application/directoryq\x04U\x0bX-Timestampq\x05U\x101330097187.00000U\x0dX-Object-Typeq\x06U\x03dirq\x07U\x06X-Typeq\x08U\x06Objectq\x09U\x06X-ETagq\x0aU
d41d8cd98f00b204e9800998ecf8427eu.", 194) = 194
getxattr("/mnt/gluster-object/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/blocks10/5/o/A",
"user.swift.metadata1", 0x0, 0) = -1 ENODATA (No data available)
getxattr("/mnt/gluster-object/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/blocks10/5/o/t",
"user.swift.metadata", 0x0, 0) = 194
getxattr("/mnt/gluster-object/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/blocks10/5/o/t",
"user.swift.metadata",
"\x80\x02}q\x01(U\x10X-Content-Lengthq\x02K\x00U\x0eX-Content-Typeq\x03U\x15application/directoryq\x04U\x0bX-Timestampq\x05U\x101330097357.00000U\x0dX-Object-Typeq\x06U\x03dirq\x07U\x06X-Typeq\x08U\x06Objectq\x09U\x06X-ETagq\x0aU
d41d8cd98f00b204e9800998ecf8427eu.", 194) = 194
getxattr("/mnt/gluster-object/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/blocks10/5/o/t",
"user.swift.metadata1", 0x0, 0) = -1 ENODATA (No data available)
lstat("/mnt/gluster-object/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/blocks10/5/o/A",
{st_mode=S_IFDIR|0777, st_size=105, ...}) = 0
open("/mnt/gluster-object/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/blocks10/5/o/A",
O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 11
getdents(11, /* 2 entries */, 32768) = 48
getdents(11, /* 1 entries */, 32768) = 112
getdents(11, /* 0 entries */, 32768) = 0
close(11) = 0
stat("/mnt/gluster-object/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/blocks10/5/o/A/5oAs3UlU-Ah3ml--FMrmZfsx_G-QXDXeSh1ZJ6vnjLnJy6iJrLZO8gLYTnSuWh5hEsNlaIbqB-J6iHaEwQJQ8A",
{st_mode=S_IFREG|0600, st_size=8189, ...}) = 0
getxattr("/mnt/gluster-object/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/blocks10/5/o/A/5oAs3UlU-Ah3ml--FMrmZfsx_G-QXDXeSh1ZJ6vnjLnJy6iJrLZO8gLYTnSuWh5hEsNlaIbqB-J6iHaEwQJQ8A",
"user.swift.metadata", 0x0, 0) = 203
getxattr("/mnt/gluster-object/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/blocks10/5/o/A/5oAs3UlU-Ah3ml--FMrmZfsx_G-QXDXeSh1ZJ6vnjLnJy6iJrLZO8gLYTnSuWh5hEsNlaIbqB-J6iHaEwQJQ8A",
"user.swift.metadata",
"\x80\x02}q\x01(U\x10X-Content-Lengthq\x02U\x010U\x0eX-Content-Typeq\x03U\x18application/octet-streamq\x04U\x0bX-Timestampq\x05U\x101330097187.91304q\x06U\x0dX-Object-Typeq\x07U\x04fileq\x08U\x06X-Typeq\x09U\x06Objectq\x0aU\x06X-ETagq\x0bU
e185fccc1ed0dbd69b304726ff54d294q\x0cu.", 203) = 203
getxattr("/mnt/gluster-object/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/blocks10/5/o/A/5oAs3UlU-Ah3ml--FMrmZfsx_G-QXDXeSh1ZJ6vnjLnJy6iJrLZO8gLYTnSuWh5hEsNlaIbqB-J6iHaEwQJQ8A",
"user.swift.metadata1", 0x0, 0) = -1 ENODATA (No data available)
stat("/mnt/gluster-object/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/blocks10/5/o/A/5oAs3UlU-Ah3ml--FMrmZfsx_G-QXDXeSh1ZJ6vnjLnJy6iJrLZO8gLYTnSuWh5hEsNlaIbqB-J6iHaEwQJQ8A",
{st_mode=S_IFREG|0600, st_size=8189, ...}) = 0
lstat("/mnt/gluster-object/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/blocks10/5/o/t",
{st_mode=S_IFDIR|0777, st_size=198, ...}) = 0
open("/mnt/gluster-object/AUTH_8140ab2e72bd3fd9d3e89e00d2d2c620/blocks10/5/o/t",
O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 11
getdents(11, /* 3 entries */, 32768) = 160
getdents(11, /* 0 entries */, 32768) = 0
...
Is this the normal behavior for the UFO API?
Best regards,
André Cruz
smime.p7s
Description: S/MIME cryptographic signature
_______________________________________________ Gluster-users mailing list [email protected] http://gluster.org/cgi-bin/mailman/listinfo/gluster-users
