Hello all,
We have a replicated cluster in Nautilus. Recently, we resharded a bucket
without stopping the gateways, as a consequence, the bucket on the
secondary zone now reports 0KB usage, even though you can still see the
objects in the data pool.
I was able to reproduce the issue in a lab, so it's easy to show. i
uploaded around 3000 empty text files :
*Bucket Stats :*
Cluster 1:
radosgw-admin bucket stats --bucket=testbucket
{
"bucket": "testbucket",
"num_shards": 24,
"tenant": "",
"zonegroup": "e184d7c4-0ea3-46ee-9aab-52ff7508f533",
"placement_rule": "default-placement",
"explicit_placement": {
"data_pool": "",
"data_extra_pool": "",
"index_pool": ""
},
"id": "604ead4e-53ec-43d4-897e-991bb7df4649.17288.1",
"marker": "604ead4e-53ec-43d4-897e-991bb7df4649.6133.1",
"index_type": "Normal",
"owner": "testuser",
"ver":
"0#6,1#8,2#6,3#8,4#6,5#8,6#8,7#6,8#6,9#8,10#6,11#6,12#8,13#6,14#6,15#8,16#6,17#6,18#6,19#6,20#10,21#12,22#8,23#6",
"master_ver":
"0#0,1#0,2#0,3#0,4#0,5#0,6#0,7#0,8#0,9#0,10#0,11#0,12#0,13#0,14#0,15#0,16#0,17#0,18#0,19#0,20#0,21#0,22#0,23#0",
"mtime": "2021-07-15 16:56:38.760765Z",
"max_marker":
"0#00000000005.18.1,1#00000000007.54.5,2#00000000005.7.1,3#00000000007.54.5,4#00000000005.7.1,5#00000000007.54.5,6#00000000007.107.5,7#00000000005.15.1,8#00000000005.7.1,9#00000000007.109.5,10#00000000005.21.1,11#00000000005.7.1,12#00000000007.156.5,13#00000000005.7.1,14#00000000005.21.1,15#00000000007.160.5,16#00000000005.14.1,17#00000000005.23.1,18#00000000005.15.1,19#00000000005.22.1,20#00000000009.57.5,21#00000000011.58.5,22#00000000007.163.5,23#00000000005.15.1",
* "usage": {*
* "rgw.main": {*
* "size": 0,*
* "size_actual": 0,*
* "size_utilized": 0,*
* "size_kb": 0,*
* "size_kb_actual": 0,*
* "size_kb_utilized": 0,*
* "num_objects": 3948*
* }*
},
"bucket_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
}
}
Cluster2
radosgw-admin bucket stats --bucket=testbucket
{
"bucket": "testbucket",
"num_shards": 24,
"tenant": "",
"zonegroup": "e184d7c4-0ea3-46ee-9aab-52ff7508f533",
"placement_rule": "default-placement",
"explicit_placement": {
"data_pool": "",
"data_extra_pool": "",
"index_pool": ""
},
"id": "604ead4e-53ec-43d4-897e-991bb7df4649.17288.1",
"marker": "604ead4e-53ec-43d4-897e-991bb7df4649.6133.1",
"index_type": "Normal",
"owner": "testuser",
"ver":
"0#1,1#5,2#1,3#5,4#1,5#5,6#5,7#1,8#1,9#5,10#1,11#1,12#5,13#1,14#1,15#5,16#1,17#1,18#1,19#1,20#9,21#13,22#5,23#1",
"master_ver":
"0#0,1#0,2#0,3#0,4#0,5#0,6#0,7#0,8#0,9#0,10#0,11#0,12#0,13#0,14#0,15#0,16#0,17#0,18#0,19#0,20#0,21#0,22#0,23#0",
"mtime": "2021-07-15 16:56:38.760765Z",
"max_marker":
"0#,1#00000000004.17.5,2#,3#00000000004.18.5,4#,5#00000000004.18.5,6#00000000004.31.5,7#,8#,9#00000000004.32.5,10#,11#,12#00000000004.126.5,13#,14#,15#00000000004.46.5,16#,17#,18#,19#,20#00000000008.21.5,21#00000000012.26.5,22#00000000004.46.5,23#",
* "usage": {*
* }*
},
"bucket_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
}
}
*RADOS DF *
Cluster1:
POOL_NAME USED OBJECTS CLONES COPIES
MISSING_ON_PRIMARY UNFOUND DEGRADED RD_OPS RD WR_OPS WR USED
COMPR UNDER COMPR
.rgw.root 3.2 MiB 18 0 54
0 0 0 792 824 KiB 50 35 KiB 0 B 0 B
*master-zone.rgw.buckets.data 0 B 3935 0 11805*
0 0 0 11723 7.4 MiB 47387 0 B 0 B 0
B
master-zone.rgw.buckets.index 355 KiB 25 0 75
0 0 0 26544 38 MiB 27971 19 MiB 0 B 0 B
master-zone.rgw.control 0 B 8 0 24
0 0 0 0 0 B 0 0 B 0 B 0 B
master-zone.rgw.log 25 MiB 672 0 2016
0 0 0 1880845 1.8 GiB 1088822 3.6 MiB 0 B 0 B
master-zone.rgw.meta 1.7 MiB 10 0 30
0 0 0 7877 5.5 MiB 75 30 KiB 0 B 0 B
master-zone.rgw.otp 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0 B
ncsa-ne1.rgw.buckets.extra 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0 B
ncsa-ne1.rgw.buckets.index 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0 B
ncsa-ne1.rgw.buckets.non-ec 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0 B
ncsa-ne1.rgw.control 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0 B
ncsa-ne1.rgw.data.root 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0 B
ncsa-ne1.rgw.gc 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0 B
ncsa-ne1.rgw.intent-log 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0 B
ncsa-ne1.rgw.log 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0 B
ncsa-ne1.rgw.meta 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0 B
ncsa-ne1.rgw.usage 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0 B
ncsa-ne1.rgw.users.email 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0 B
ncsa-ne1.rgw.users.keys 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0 B
ncsa-ne1.rgw.users.swift 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0 B
ncsa-ne1.rgw.users.uid 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0 B
total_objects 4668
total_used 32 GiB
total_avail 448 GiB
total_space 480 GiB
Cluster2:
[root@cloud-lab-rgw04 ~]# rados df
POOL_NAME USED OBJECTS CLONES COPIES
MISSING_ON_PRIMARY UNFOUND DEGRADED RD_OPS RD WR_OPS WR USED
COMPR UNDER COMPR
.rgw.root 3.2 MiB 18 0 54
0 0 0 5283 5.6 MiB 70 60 KiB 0 B 0
B
ncsa-ne1.rgw.buckets.data 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0
B
ncsa-ne1.rgw.buckets.extra 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0
B
ncsa-ne1.rgw.buckets.index 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0
B
ncsa-ne1.rgw.buckets.non-ec 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0
B
ncsa-ne1.rgw.control 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0
B
ncsa-ne1.rgw.data.root 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0
B
ncsa-ne1.rgw.gc 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0
B
ncsa-ne1.rgw.intent-log 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0
B
ncsa-ne1.rgw.log 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0
B
ncsa-ne1.rgw.meta 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0
B
ncsa-ne1.rgw.usage 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0
B
ncsa-ne1.rgw.users.email 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0
B
ncsa-ne1.rgw.users.keys 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0
B
ncsa-ne1.rgw.users.swift 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0
B
ncsa-ne1.rgw.users.uid 0 B 0 0 0
0 0 0 0 0 B 0 0 B 0 B 0
B
*secondary-zone.rgw.buckets.data 0 B 3935 0 11805*
0 0 0 242 140 KiB 55283 0 B 0 B
0 B
secondary-zone.rgw.buckets.index 2.9 MiB 26 0 78
0 0 0 35120 36 MiB 24034 15 MiB 0 B 0
B
secondary-zone.rgw.control 0 B 8 0 24
0 0 0 0 0 B 0 0 B 0 B 0
B
secondary-zone.rgw.log 37 MiB 766 0 2298
0 0 0 2307976 2.2 GiB 1459774 3.1 MiB 0 B 0
B
secondary-zone.rgw.meta 1.7 MiB 11 0 33
0 0 0 8868 6.6 MiB 131 44 KiB 0 B 0
B
total_objects 4764
total_used 31 GiB
total_avail 449 GiB
total_space 480 GiB
Is there a way I can fix this situation? make Cluster2 realize he still has
the objects related to ''testbucket'' ? or at least a way where i can re-do
the synchronization from scratch on that bucket? I already tried "data
sync init" and "metadata sync init" but it didn't do the trick.
_______________________________________________
ceph-users mailing list -- [email protected]
To unsubscribe send an email to [email protected]
