Hi Greg,
from the client the list is huge:
Thats the situation while the dd's are stuck.
[root@cn201 ~]# ceph daemon /var/run/ceph/ceph-client.admin.asok
objecter_requests
{
"ops": [
{
"tid": 12,
"pg": "6.7230bd94",
"osd": 1,
"object_id": "10000000017.00000004",
"object_locator": "@6",
"target_object_id": "10000000017.00000004",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:28.788800",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:41:18.001327",
"osd_ops": [
"write 0~4194304 [5@0]"
]
},
{
"tid": 28,
"pg": "6.7230bd94",
"osd": 1,
"object_id": "10000000017.00000004",
"object_locator": "@6",
"target_object_id": "10000000017.00000004",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:28.788869",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:12.997492",
"osd_ops": [
"write 0~4194304 [6@0]"
]
},
{
"tid": 44,
"pg": "6.7230bd94",
"osd": 1,
"object_id": "10000000017.00000004",
"object_locator": "@6",
"target_object_id": "10000000017.00000004",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:28.788923",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:28.278782",
"osd_ops": [
"write 0~4194304 [7@0]"
]
},
{
"tid": 57,
"pg": "6.5209ef64",
"osd": 1,
"object_id": "10000000017.00000011",
"object_locator": "@6",
"target_object_id": "10000000017.00000011",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:28.789085",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:29.357976",
"osd_ops": [
"write 0~4194304 [7@0]"
]
},
{
"tid": 64,
"pg": "6.7230bd94",
"osd": 1,
"object_id": "10000000017.00000004",
"object_locator": "@6",
"target_object_id": "10000000017.00000004",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:28.789251",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:38.295260",
"osd_ops": [
"write 0~4194304 [8@0]"
]
},
{
"tid": 77,
"pg": "6.5209ef64",
"osd": 1,
"object_id": "10000000017.00000011",
"object_locator": "@6",
"target_object_id": "10000000017.00000011",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:28.789419",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:39.466741",
"osd_ops": [
"write 0~4194304 [8@0]"
]
},
{
"tid": 6,
"pg": "6.4dbcd0b2",
"osd": 2,
"object_id": "10000000017.00000001",
"object_locator": "@6",
"target_object_id": "10000000017.00000001",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:17.543793",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:41:07.938683",
"osd_ops": [
"write 0~2359296 [4@0]"
]
},
{
"tid": 9,
"pg": "6.4dbcd0b2",
"osd": 2,
"object_id": "10000000017.00000001",
"object_locator": "@6",
"target_object_id": "10000000017.00000001",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:17.543868",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:41:17.949075",
"osd_ops": [
"write 0~4194304 [5@0]"
]
},
{
"tid": 10,
"pg": "6.42c29750",
"osd": 2,
"object_id": "10000000017.00000002",
"object_locator": "@6",
"target_object_id": "10000000017.00000002",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:17.543917",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:41:17.967418",
"osd_ops": [
"write 0~4194304 [5@0]"
]
},
{
"tid": 13,
"pg": "6.e88cfd97",
"osd": 2,
"object_id": "10000000017.00000005",
"object_locator": "@6",
"target_object_id": "10000000017.00000005",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:17.543964",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:41:18.019627",
"osd_ops": [
"write 0~4194304 [5@0]"
]
},
{
"tid": 25,
"pg": "6.4dbcd0b2",
"osd": 2,
"object_id": "10000000017.00000001",
"object_locator": "@6",
"target_object_id": "10000000017.00000001",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:17.544011",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:12.943431",
"osd_ops": [
"write 0~4194304 [6@0]"
]
},
{
"tid": 26,
"pg": "6.42c29750",
"osd": 2,
"object_id": "10000000017.00000002",
"object_locator": "@6",
"target_object_id": "10000000017.00000002",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:17.544059",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:12.960874",
"osd_ops": [
"write 0~4194304 [6@0]"
]
},
{
"tid": 29,
"pg": "6.e88cfd97",
"osd": 2,
"object_id": "10000000017.00000005",
"object_locator": "@6",
"target_object_id": "10000000017.00000005",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:17.544105",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:13.015905",
"osd_ops": [
"write 0~4194304 [6@0]"
]
},
{
"tid": 41,
"pg": "6.4dbcd0b2",
"osd": 2,
"object_id": "10000000017.00000001",
"object_locator": "@6",
"target_object_id": "10000000017.00000001",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:17.544152",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:28.055155",
"osd_ops": [
"write 0~4194304 [7@0]"
]
},
{
"tid": 42,
"pg": "6.42c29750",
"osd": 2,
"object_id": "10000000017.00000002",
"object_locator": "@6",
"target_object_id": "10000000017.00000002",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:17.544330",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:28.128659",
"osd_ops": [
"write 0~4194304 [7@0]"
]
},
{
"tid": 45,
"pg": "6.e88cfd97",
"osd": 2,
"object_id": "10000000017.00000005",
"object_locator": "@6",
"target_object_id": "10000000017.00000005",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:17.544509",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:28.351606",
"osd_ops": [
"write 0~4194304 [7@0]"
]
},
{
"tid": 61,
"pg": "6.4dbcd0b2",
"osd": 2,
"object_id": "10000000017.00000001",
"object_locator": "@6",
"target_object_id": "10000000017.00000001",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:17.544689",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:38.065953",
"osd_ops": [
"write 0~4194304 [8@0]"
]
},
{
"tid": 62,
"pg": "6.42c29750",
"osd": 2,
"object_id": "10000000017.00000002",
"object_locator": "@6",
"target_object_id": "10000000017.00000002",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:17.544960",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:38.142360",
"osd_ops": [
"write 0~4194304 [8@0]"
]
},
{
"tid": 65,
"pg": "6.e88cfd97",
"osd": 2,
"object_id": "10000000017.00000005",
"object_locator": "@6",
"target_object_id": "10000000017.00000005",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:17.545154",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:38.369731",
"osd_ops": [
"write 0~4194304 [8@0]"
]
},
{
"tid": 83,
"pg": "6.a769f350",
"osd": 2,
"object_id": "10000000017.00000017",
"object_locator": "@6",
"target_object_id": "10000000017.00000017",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:17.545341",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:39.927528",
"osd_ops": [
"write 0~4194304 [8@0]"
]
},
{
"tid": 86,
"pg": "6.44681e45",
"osd": 2,
"object_id": "100000007f3.00000000",
"object_locator": "@6",
"target_object_id": "100000007f3.00000000",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:17.545535",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:44:48.630163",
"osd_ops": [
"write 0~65536"
]
},
{
"tid": 21,
"pg": "6.e9463d78",
"osd": 4,
"object_id": "10000000017.0000000d",
"object_locator": "@6",
"target_object_id": "10000000017.0000000d",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:28.585601",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:41:18.164460",
"osd_ops": [
"write 0~4194304 [5@0]"
]
},
{
"tid": 37,
"pg": "6.e9463d78",
"osd": 4,
"object_id": "10000000017.0000000d",
"object_locator": "@6",
"target_object_id": "10000000017.0000000d",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:28.585665",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:13.172004",
"osd_ops": [
"write 0~4194304 [6@0]"
]
},
{
"tid": 53,
"pg": "6.e9463d78",
"osd": 4,
"object_id": "10000000017.0000000d",
"object_locator": "@6",
"target_object_id": "10000000017.0000000d",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:28.585717",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:28.989817",
"osd_ops": [
"write 0~4194304 [7@0]"
]
},
{
"tid": 56,
"pg": "6.614c51c7",
"osd": 4,
"object_id": "10000000017.00000010",
"object_locator": "@6",
"target_object_id": "10000000017.00000010",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:28.585914",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:29.285570",
"osd_ops": [
"write 0~4194304 [7@0]"
]
},
{
"tid": 73,
"pg": "6.e9463d78",
"osd": 4,
"object_id": "10000000017.0000000d",
"object_locator": "@6",
"target_object_id": "10000000017.0000000d",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:28.586093",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:39.119695",
"osd_ops": [
"write 0~4194304 [8@0]"
]
},
{
"tid": 76,
"pg": "6.614c51c7",
"osd": 4,
"object_id": "10000000017.00000010",
"object_locator": "@6",
"target_object_id": "10000000017.00000010",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:28.586272",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:39.389947",
"osd_ops": [
"write 0~4194304 [8@0]"
]
},
{
"tid": 19,
"pg": "6.b05088a0",
"osd": 5,
"object_id": "10000000017.0000000b",
"object_locator": "@6",
"target_object_id": "10000000017.0000000b",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.737883",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:41:18.127904",
"osd_ops": [
"write 0~4194304 [5@0]"
]
},
{
"tid": 35,
"pg": "6.b05088a0",
"osd": 5,
"object_id": "10000000017.0000000b",
"object_locator": "@6",
"target_object_id": "10000000017.0000000b",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.737941",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:13.132335",
"osd_ops": [
"write 0~4194304 [6@0]"
]
},
{
"tid": 51,
"pg": "6.b05088a0",
"osd": 5,
"object_id": "10000000017.0000000b",
"object_locator": "@6",
"target_object_id": "10000000017.0000000b",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.737989",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:28.818212",
"osd_ops": [
"write 0~4194304 [7@0]"
]
},
{
"tid": 58,
"pg": "6.c72d70cb",
"osd": 5,
"object_id": "10000000017.00000012",
"object_locator": "@6",
"target_object_id": "10000000017.00000012",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.738174",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:29.431176",
"osd_ops": [
"write 0~4194304 [7@0]"
]
},
{
"tid": 71,
"pg": "6.b05088a0",
"osd": 5,
"object_id": "10000000017.0000000b",
"object_locator": "@6",
"target_object_id": "10000000017.0000000b",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.738395",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:38.902311",
"osd_ops": [
"write 0~4194304 [8@0]"
]
},
{
"tid": 78,
"pg": "6.c72d70cb",
"osd": 5,
"object_id": "10000000017.00000012",
"object_locator": "@6",
"target_object_id": "10000000017.00000012",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.738576",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:39.541007",
"osd_ops": [
"write 0~4194304 [8@0]"
]
},
{
"tid": 22,
"pg": "6.270fac80",
"osd": 6,
"object_id": "10000000017.0000000f",
"object_locator": "@6",
"target_object_id": "10000000017.0000000f",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.736441",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:41:18.196365",
"osd_ops": [
"write 0~2621440 [5@0]"
]
},
{
"tid": 38,
"pg": "6.270fac80",
"osd": 6,
"object_id": "10000000017.0000000f",
"object_locator": "@6",
"target_object_id": "10000000017.0000000f",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.736504",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:13.204495",
"osd_ops": [
"write 0~2621440 [6@0]"
]
},
{
"tid": 55,
"pg": "6.270fac80",
"osd": 6,
"object_id": "10000000017.0000000f",
"object_locator": "@6",
"target_object_id": "10000000017.0000000f",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.736547",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:29.192276",
"osd_ops": [
"write 0~4194304 [7@0]"
]
},
{
"tid": 75,
"pg": "6.270fac80",
"osd": 6,
"object_id": "10000000017.0000000f",
"object_locator": "@6",
"target_object_id": "10000000017.0000000f",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.736741",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:39.313210",
"osd_ops": [
"write 0~4194304 [8@0]"
]
},
{
"tid": 81,
"pg": "6.908c8c60",
"osd": 6,
"object_id": "10000000017.00000015",
"object_locator": "@6",
"target_object_id": "10000000017.00000015",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.737073",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:39.763809",
"osd_ops": [
"write 0~4194304 [8@0]"
]
},
{
"tid": 82,
"pg": "6.5809f836",
"osd": 7,
"object_id": "10000000017.00000016",
"object_locator": "@6",
"target_object_id": "10000000017.00000016",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:12:50.446548",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:39.850359",
"osd_ops": [
"write 0~4194304 [8@0]"
]
},
{
"tid": 16,
"pg": "6.a394e26b",
"osd": 9,
"object_id": "10000000017.00000008",
"object_locator": "@6",
"target_object_id": "10000000017.00000008",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.411755",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:41:18.073335",
"osd_ops": [
"write 0~4194304 [5@0]"
]
},
{
"tid": 32,
"pg": "6.a394e26b",
"osd": 9,
"object_id": "10000000017.00000008",
"object_locator": "@6",
"target_object_id": "10000000017.00000008",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.411839",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:13.076196",
"osd_ops": [
"write 0~4194304 [6@0]"
]
},
{
"tid": 48,
"pg": "6.a394e26b",
"osd": 9,
"object_id": "10000000017.00000008",
"object_locator": "@6",
"target_object_id": "10000000017.00000008",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.411888",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:28.571613",
"osd_ops": [
"write 0~4194304 [7@0]"
]
},
{
"tid": 59,
"pg": "6.814a2323",
"osd": 9,
"object_id": "10000000017.00000013",
"object_locator": "@6",
"target_object_id": "10000000017.00000013",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.412046",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:29.470338",
"osd_ops": [
"write 0~2228224 [7@0]"
]
},
{
"tid": 68,
"pg": "6.a394e26b",
"osd": 9,
"object_id": "10000000017.00000008",
"object_locator": "@6",
"target_object_id": "10000000017.00000008",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.412142",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:38.591588",
"osd_ops": [
"write 0~4194304 [8@0]"
]
},
{
"tid": 79,
"pg": "6.814a2323",
"osd": 9,
"object_id": "10000000017.00000013",
"object_locator": "@6",
"target_object_id": "10000000017.00000013",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.412300",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:39.614062",
"osd_ops": [
"write 0~4194304 [8@0]"
]
},
{
"tid": 11,
"pg": "6.ec27d4f9",
"osd": 10,
"object_id": "10000000017.00000003",
"object_locator": "@6",
"target_object_id": "10000000017.00000003",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.572966",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:41:17.984529",
"osd_ops": [
"write 0~4194304 [5@0]"
]
},
{
"tid": 27,
"pg": "6.ec27d4f9",
"osd": 10,
"object_id": "10000000017.00000003",
"object_locator": "@6",
"target_object_id": "10000000017.00000003",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.573016",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:12.978150",
"osd_ops": [
"write 0~4194304 [6@0]"
]
},
{
"tid": 43,
"pg": "6.ec27d4f9",
"osd": 10,
"object_id": "10000000017.00000003",
"object_locator": "@6",
"target_object_id": "10000000017.00000003",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.573066",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:28.204523",
"osd_ops": [
"write 0~4194304 [7@0]"
]
},
{
"tid": 63,
"pg": "6.ec27d4f9",
"osd": 10,
"object_id": "10000000017.00000003",
"object_locator": "@6",
"target_object_id": "10000000017.00000003",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.573249",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:38.221211",
"osd_ops": [
"write 0~4194304 [8@0]"
]
},
{
"tid": 4,
"pg": "6.fe5731e",
"osd": 12,
"object_id": "10000000017.00000000",
"object_locator": "@6",
"target_object_id": "10000000017.00000000",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:10:08.150573",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:40:03.169580",
"osd_ops": [
"write 0~65536 [2@0]"
]
},
{
"tid": 5,
"pg": "6.fe5731e",
"osd": 12,
"object_id": "10000000017.00000000",
"object_locator": "@6",
"target_object_id": "10000000017.00000000",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:10:08.150633",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:40:57.899480",
"osd_ops": [
"write 0~655360 [3@0]"
]
},
{
"tid": 7,
"pg": "6.fe5731e",
"osd": 12,
"object_id": "10000000017.00000000",
"object_locator": "@6",
"target_object_id": "10000000017.00000000",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:10:08.150663",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:41:07.923942",
"osd_ops": [
"write 0~4194304 [4@0]"
]
},
{
"tid": 8,
"pg": "6.fe5731e",
"osd": 12,
"object_id": "10000000017.00000000",
"object_locator": "@6",
"target_object_id": "10000000017.00000000",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:10:08.150749",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:41:17.925282",
"osd_ops": [
"write 0~4194304 [5@0]"
]
},
{
"tid": 18,
"pg": "6.cdf685be",
"osd": 12,
"object_id": "10000000017.0000000a",
"object_locator": "@6",
"target_object_id": "10000000017.0000000a",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:10:08.150796",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:41:18.109067",
"osd_ops": [
"write 0~4194304 [5@0]"
]
},
{
"tid": 24,
"pg": "6.fe5731e",
"osd": 12,
"object_id": "10000000017.00000000",
"object_locator": "@6",
"target_object_id": "10000000017.00000000",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:10:08.150845",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:12.919016",
"osd_ops": [
"write 0~4194304 [6@0]"
]
},
{
"tid": 34,
"pg": "6.cdf685be",
"osd": 12,
"object_id": "10000000017.0000000a",
"object_locator": "@6",
"target_object_id": "10000000017.0000000a",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:10:08.150892",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:13.112401",
"osd_ops": [
"write 0~4194304 [6@0]"
]
},
{
"tid": 40,
"pg": "6.fe5731e",
"osd": 12,
"object_id": "10000000017.00000000",
"object_locator": "@6",
"target_object_id": "10000000017.00000000",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:10:08.150941",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:27.979012",
"osd_ops": [
"write 0~4194304 [7@0]"
]
},
{
"tid": 50,
"pg": "6.cdf685be",
"osd": 12,
"object_id": "10000000017.0000000a",
"object_locator": "@6",
"target_object_id": "10000000017.0000000a",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:10:08.151127",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:28.733192",
"osd_ops": [
"write 0~4194304 [7@0]"
]
},
{
"tid": 60,
"pg": "6.fe5731e",
"osd": 12,
"object_id": "10000000017.00000000",
"object_locator": "@6",
"target_object_id": "10000000017.00000000",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:10:08.151312",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:37.986788",
"osd_ops": [
"write 0~4194304 [8@0]"
]
},
{
"tid": 70,
"pg": "6.cdf685be",
"osd": 12,
"object_id": "10000000017.0000000a",
"object_locator": "@6",
"target_object_id": "10000000017.0000000a",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:10:08.151505",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:38.794709",
"osd_ops": [
"write 0~4194304 [8@0]"
]
},
{
"tid": 80,
"pg": "6.819c48d4",
"osd": 12,
"object_id": "10000000017.00000014",
"object_locator": "@6",
"target_object_id": "10000000017.00000014",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:10:08.151774",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:39.688711",
"osd_ops": [
"write 0~4194304 [8@0]"
]
},
{
"tid": 84,
"pg": "6.a2bb8bc5",
"osd": 12,
"object_id": "10000000017.00000018",
"object_locator": "@6",
"target_object_id": "10000000017.00000018",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:10:08.151970",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:40.004348",
"osd_ops": [
"write 0~4194304 [8@0]"
]
},
{
"tid": 15,
"pg": "6.cde1721f",
"osd": 13,
"object_id": "10000000017.00000007",
"object_locator": "@6",
"target_object_id": "10000000017.00000007",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.186859",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:41:18.055000",
"osd_ops": [
"write 0~4194304 [5@0]"
]
},
{
"tid": 31,
"pg": "6.cde1721f",
"osd": 13,
"object_id": "10000000017.00000007",
"object_locator": "@6",
"target_object_id": "10000000017.00000007",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.186924",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:13.056208",
"osd_ops": [
"write 0~4194304 [6@0]"
]
},
{
"tid": 47,
"pg": "6.cde1721f",
"osd": 13,
"object_id": "10000000017.00000007",
"object_locator": "@6",
"target_object_id": "10000000017.00000007",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.186983",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:28.497923",
"osd_ops": [
"write 0~4194304 [7@0]"
]
},
{
"tid": 67,
"pg": "6.cde1721f",
"osd": 13,
"object_id": "10000000017.00000007",
"object_locator": "@6",
"target_object_id": "10000000017.00000007",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.187137",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:38.515997",
"osd_ops": [
"write 0~4194304 [8@0]"
]
},
{
"tid": 20,
"pg": "6.bcd572c0",
"osd": 14,
"object_id": "10000000017.0000000c",
"object_locator": "@6",
"target_object_id": "10000000017.0000000c",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.572285",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:41:18.146794",
"osd_ops": [
"write 0~4194304 [5@0]"
]
},
{
"tid": 36,
"pg": "6.bcd572c0",
"osd": 14,
"object_id": "10000000017.0000000c",
"object_locator": "@6",
"target_object_id": "10000000017.0000000c",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.572362",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:13.152185",
"osd_ops": [
"write 0~4194304 [6@0]"
]
},
{
"tid": 52,
"pg": "6.bcd572c0",
"osd": 14,
"object_id": "10000000017.0000000c",
"object_locator": "@6",
"target_object_id": "10000000017.0000000c",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.572417",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:28.903736",
"osd_ops": [
"write 0~4194304 [7@0]"
]
},
{
"tid": 72,
"pg": "6.bcd572c0",
"osd": 14,
"object_id": "10000000017.0000000c",
"object_locator": "@6",
"target_object_id": "10000000017.0000000c",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:27.572668",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:39.013349",
"osd_ops": [
"write 0~4194304 [8@0]"
]
},
{
"tid": 23,
"pg": "6.1fd38a65",
"osd": 15,
"object_id": "10000000017.0000000e",
"object_locator": "@6",
"target_object_id": "10000000017.0000000e",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:22.372161",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:41:18.183007",
"osd_ops": [
"write 0~4194304 [5@0]"
]
},
{
"tid": 39,
"pg": "6.1fd38a65",
"osd": 15,
"object_id": "10000000017.0000000e",
"object_locator": "@6",
"target_object_id": "10000000017.0000000e",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:22.372232",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:13.191923",
"osd_ops": [
"write 0~4194304 [6@0]"
]
},
{
"tid": 54,
"pg": "6.1fd38a65",
"osd": 15,
"object_id": "10000000017.0000000e",
"object_locator": "@6",
"target_object_id": "10000000017.0000000e",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:22.372289",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:29.080108",
"osd_ops": [
"write 0~4194304 [7@0]"
]
},
{
"tid": 74,
"pg": "6.1fd38a65",
"osd": 15,
"object_id": "10000000017.0000000e",
"object_locator": "@6",
"target_object_id": "10000000017.0000000e",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:22.372471",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:39.223261",
"osd_ops": [
"write 0~4194304 [8@0]"
]
},
{
"tid": 17,
"pg": "6.cda94712",
"osd": 16,
"object_id": "10000000017.00000009",
"object_locator": "@6",
"target_object_id": "10000000017.00000009",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:22.020814",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:41:18.091258",
"osd_ops": [
"write 0~4194304 [5@0]"
]
},
{
"tid": 33,
"pg": "6.cda94712",
"osd": 16,
"object_id": "10000000017.00000009",
"object_locator": "@6",
"target_object_id": "10000000017.00000009",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:22.020883",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:13.094016",
"osd_ops": [
"write 0~4194304 [6@0]"
]
},
{
"tid": 49,
"pg": "6.cda94712",
"osd": 16,
"object_id": "10000000017.00000009",
"object_locator": "@6",
"target_object_id": "10000000017.00000009",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:22.020933",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:28.648157",
"osd_ops": [
"write 0~4194304 [7@0]"
]
},
{
"tid": 69,
"pg": "6.cda94712",
"osd": 16,
"object_id": "10000000017.00000009",
"object_locator": "@6",
"target_object_id": "10000000017.00000009",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:22.021110",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:38.687546",
"osd_ops": [
"write 0~4194304 [8@0]"
]
},
{
"tid": 14,
"pg": "6.345417ca",
"osd": 17,
"object_id": "10000000017.00000006",
"object_locator": "@6",
"target_object_id": "10000000017.00000006",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:22.372850",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:41:18.037706",
"osd_ops": [
"write 0~4194304 [5@0]"
]
},
{
"tid": 30,
"pg": "6.345417ca",
"osd": 17,
"object_id": "10000000017.00000006",
"object_locator": "@6",
"target_object_id": "10000000017.00000006",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:22.372904",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:13.035922",
"osd_ops": [
"write 0~4194304 [6@0]"
]
},
{
"tid": 46,
"pg": "6.345417ca",
"osd": 17,
"object_id": "10000000017.00000006",
"object_locator": "@6",
"target_object_id": "10000000017.00000006",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:22.372952",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:28.424248",
"osd_ops": [
"write 0~4194304 [7@0]"
]
},
{
"tid": 66,
"pg": "6.345417ca",
"osd": 17,
"object_id": "10000000017.00000006",
"object_locator": "@6",
"target_object_id": "10000000017.00000006",
"target_object_locator": "@6",
"paused": 0,
"used_replica": 0,
"precalc_pgid": 0,
"last_sent": "2016-01-22 23:11:22.373134",
"attempts": 95,
"snapid": "head",
"snap_context": "1=[]",
"mtime": "2016-01-21 23:42:38.442751",
"osd_ops": [
"write 0~4194304 [8@0]"
]
}
],
"linger_ops": [],
"pool_ops": [],
"pool_stat_ops": [],
"statfs_ops": [],
"command_ops": []
}
=============================================================
[root@cn201 ~]# ceph daemon /var/run/ceph/ceph-client.admin.asok perf dump
{
"client": {
"reply": {
"avgcount": 18408,
"sum": 39.943500347
},
"lat": {
"avgcount": 18408,
"sum": 39.943500347
},
"owrlat": {
"avgcount": 0,
"sum": 0.000000000
},
"ordlat": {
"avgcount": 0,
"sum": 0.000000000
},
"wrlat": {
"avgcount": 43296,
"sum": 22.456565568
}
},
"objectcacher-libcephfs": {
"cache_ops_hit": 2,
"cache_ops_miss": 3,
"cache_bytes_hit": 160694,
"cache_bytes_miss": 75857,
"data_read": 155031,
"data_written": 325483803,
"data_flushed": 325483803,
"data_overwritten_while_flushing": 0,
"write_ops_blocked": 0,
"write_bytes_blocked": 0,
"write_time_blocked": 0.000000000
},
"objecter": {
"op_active": 82,
"op_laggy": 82,
"op_send": 26194,
"op_send_bytes": 0,
"op_resend": 7721,
"op_ack": 5,
"op_commit": 18386,
"op": 18473,
"op_r": 3,
"op_w": 18470,
"op_rmw": 0,
"op_pg": 0,
"osdop_stat": 1,
"osdop_create": 1,
"osdop_read": 3,
"osdop_write": 18468,
"osdop_writefull": 0,
"osdop_append": 0,
"osdop_zero": 0,
"osdop_truncate": 0,
"osdop_delete": 0,
"osdop_mapext": 0,
"osdop_sparse_read": 0,
"osdop_clonerange": 0,
"osdop_getxattr": 0,
"osdop_setxattr": 0,
"osdop_cmpxattr": 0,
"osdop_rmxattr": 0,
"osdop_resetxattrs": 0,
"osdop_tmap_up": 0,
"osdop_tmap_put": 0,
"osdop_tmap_get": 0,
"osdop_call": 0,
"osdop_watch": 0,
"osdop_notify": 0,
"osdop_src_cmpxattr": 0,
"osdop_pgls": 0,
"osdop_pgls_filter": 0,
"osdop_other": 0,
"linger_active": 0,
"linger_send": 0,
"linger_resend": 0,
"linger_ping": 0,
"poolop_active": 0,
"poolop_send": 0,
"poolop_resend": 0,
"poolstat_active": 0,
"poolstat_send": 0,
"poolstat_resend": 0,
"statfs_active": 0,
"statfs_send": 6,
"statfs_resend": 0,
"command_active": 0,
"command_send": 0,
"command_resend": 0,
"map_epoch": 12597,
"map_full": 0,
"map_inc": 5,
"osd_sessions": 120,
"osd_session_open": 1425,
"osd_session_close": 1410,
"osd_laggy": 14
},
"throttle-msgr_dispatch_throttler-client": {
"val": 0,
"max": 104857600,
"get": 111139,
"get_sum": 21278267,
"get_or_fail_fail": 0,
"get_or_fail_success": 0,
"take": 0,
"take_sum": 0,
"put": 111139,
"put_sum": 21278267,
"wait": {
"avgcount": 0,
"sum": 0.000000000
}
},
"throttle-objecter_bytes": {
"val": 325189632,
"max": 104857600,
"get": 0,
"get_sum": 0,
"get_or_fail_fail": 0,
"get_or_fail_success": 0,
"take": 18473,
"take_sum": 325559660,
"put": 18389,
"put_sum": 370028,
"wait": {
"avgcount": 0,
"sum": 0.000000000
}
},
"throttle-objecter_ops": {
"val": 82,
"max": 1024,
"get": 0,
"get_sum": 0,
"get_or_fail_fail": 0,
"get_or_fail_success": 0,
"take": 18473,
"take_sum": 18473,
"put": 18391,
"put_sum": 18391,
"wait": {
"avgcount": 0,
"sum": 0.000000000
}
}
}
--
Mit freundlichen Gruessen / Best regards
Oliver Dzombic
IP-Interactive
mailto:[email protected]
Anschrift:
IP Interactive UG ( haftungsbeschraenkt )
Zum Sonnenberg 1-3
63571 Gelnhausen
HRB 93402 beim Amtsgericht Hanau
Geschäftsführung: Oliver Dzombic
Steuer Nr.: 35 236 3622 1
UST ID: DE274086107
_______________________________________________
ceph-users mailing list
[email protected]
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
