Hi Orit,
It appears it must have been one of the known bugs in 10.2.2. I just
upgraded to 10.2.3 and bi-directional syncing now works.
I am still seeing some errors when I run synch-related commands but they
don't seem to be affecting operations as of yet:
radosgw-admin sync status
2016-09-27 16:17:15.270331 7fe5e83ad9c0 0 error in read_id for id : (2)
No such file or directory
2016-09-27 16:17:15.270883 7fe5e83ad9c0 0 error in read_id for id : (2)
No such file or directory
realm 3af93a86-916a-490f-b38f-17922b472b19 (my_realm)
zonegroup 235b010c-22e2-4b43-8fcc-8ae01939273e (us)
zone 58aa3eef-fc1f-492c-a08e-9c6019e7c266 (us-phx)
metadata sync preparing for full sync
full sync: 0/64 shards
metadata is caught up with master
incremental sync: 64/64 shards
data sync source: 6c830b44-4e39-4e19-9bd8-03c37c2021f2 (us-dfw)
preparing for full sync
full sync: 18/128 shards
full sync: 0 buckets to sync
incremental sync: 110/128 shards
data is behind on 20 shards
oldest incremental change not applied: 2016-09-27
16:17:08.0.922757s
I've also verified all of the existing data within each bucket has sync'd
over as well
On Tue, Sep 27, 2016 at 9:21 AM John Rowe <john.r...@salesforce.com> wrote:
> Hi Orit,
>
> That was my failed attempt at sanitizing :)
>
> They are actually all identical:
>
> Periods:
> MD5 (cephrgw1-1-dfw-period.json) = 12ed481381c1f2937a27b57db0473d6d
> MD5 (cephrgw1-1-phx-period.json) = 12ed481381c1f2937a27b57db0473d6d
> MD5 (cephrgw1-2-dfw-period.json) = 12ed481381c1f2937a27b57db0473d6d
> MD5 (cephrgw1-2-phx-period.json) = 12ed481381c1f2937a27b57db0473d6d
> MD5 (cephrgw1-3-dfw-period.json) = 12ed481381c1f2937a27b57db0473d6d
> MD5 (cephrgw1-3-phx-period.json) = 12ed481381c1f2937a27b57db0473d6d
>
> Realms:
> MD5 (cephrgw1-1-dfw-realm.json) = 39a4e63bab64ed756961117d3629b109
> MD5 (cephrgw1-1-phx-realm.json) = 39a4e63bab64ed756961117d3629b109
> MD5 (cephrgw1-2-dfw-realm.json) = 39a4e63bab64ed756961117d3629b109
> MD5 (cephrgw1-2-phx-realm.json) = 39a4e63bab64ed756961117d3629b109
> MD5 (cephrgw1-3-dfw-realm.json) = 39a4e63bab64ed756961117d3629b109
> MD5 (cephrgw1-3-phx-realm.json) = 39a4e63bab64ed756961117d3629b109
>
>
> On Tue, Sep 27, 2016 at 5:32 AM Orit Wasserman <owass...@redhat.com>
> wrote:
>
>> see comment below
>>
>> On Mon, Sep 26, 2016 at 10:00 PM, John Rowe <john.r...@salesforce.com>
>> wrote:
>> > Hi Orit,
>> >
>> > Sure thing, please see below.
>> >
>> > Thanks!
>> >
>> >
>> > DFW (Primary)
>> > radosgw-admin zonegroupmap get
>> > {
>> > "zonegroups": [
>> > {
>> > "key": "235b010c-22e2-4b43-8fcc-8ae01939273e",
>> > "val": {
>> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e",
>> > "name": "us",
>> > "api_name": "us",
>> > "is_master": "true",
>> > "endpoints": [
>> > "http:\/\/ELB_FQDN:80"
>> > ],
>> > "hostnames": [],
>> > "hostnames_s3website": [],
>> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "zones": [
>> > {
>> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
>> > "name": "us-phx",
>> > "endpoints": [
>> > "http:\/\/cephrgw1-1:80"
>> > ],
>> > "log_meta": "false",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > },
>> > {
>> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "name": "us-dfw",
>> > "endpoints": [
>> > "http:\/\/cephrgw1-1-dfw:80"
>> > ],
>> > "log_meta": "true",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > }
>> > ],
>> > "placement_targets": [
>> > {
>> > "name": "default-placement",
>> > "tags": []
>> > }
>> > ],
>> > "default_placement": "default-placement",
>> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
>> > }
>> > }
>> > ],
>> > "master_zonegroup": "235b010c-22e2-4b43-8fcc-8ae01939273e",
>> > "bucket_quota": {
>> > "enabled": false,
>> > "max_size_kb": -1,
>> > "max_objects": -1
>> > },
>> > "user_quota": {
>> > "enabled": false,
>> > "max_size_kb": -1,
>> > "max_objects": -1
>> > }
>> > }
>> >
>> > PHX (Secondary)
>> > {
>> > "zonegroups": [
>> > {
>> > "key": "235b010c-22e2-4b43-8fcc-8ae01939273e",
>> > "val": {
>> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e",
>> > "name": "us",
>> > "api_name": "us",
>> > "is_master": "true",
>> > "endpoints": [
>> > "http:\/\/ELB_FQDN:80"
>> > ],
>> > "hostnames": [],
>> > "hostnames_s3website": [],
>> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "zones": [
>> > {
>> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
>> > "name": "us-phx",
>> > "endpoints": [
>> > "http:\/\/cephrgw1-1-phx:80"
>> > ],
>> > "log_meta": "false",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > },
>> > {
>> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "name": "us-dfw",
>> > "endpoints": [
>> > "http:\/\/cephrgw1-1-dfw:80"
>> > ],
>> > "log_meta": "true",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > }
>> > ],
>> > "placement_targets": [
>> > {
>> > "name": "default-placement",
>> > "tags": []
>> > }
>> > ],
>> > "default_placement": "default-placement",
>> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
>> > }
>> > }
>> > ],
>> > "master_zonegroup": "235b010c-22e2-4b43-8fcc-8ae01939273e",
>> > "bucket_quota": {
>> > "enabled": false,
>> > "max_size_kb": -1,
>> > "max_objects": -1
>> > },
>> > "user_quota": {
>> > "enabled": false,
>> > "max_size_kb": -1,
>> > "max_objects": -1
>> > }
>> > }
>> >
>> >
>> > Realms and Periods:
>> > DFW (Primary)
>> > rgw1-1-dfw
>> > radosgw-admin realm get
>> > {
>> > "id": "3af93a86-916a-490f-b38f-17922b472b19",
>> > "name": "my_realm",
>> > "current_period": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
>> > "epoch": 2
>> > }
>> > radosgw-admin period get
>> > {
>> > "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
>> > "epoch": 4,
>> > "predecessor_uuid": "ab3ffbbb-0fc9-49f7-a44f-e804f9ed7697",
>> > "sync_status": [
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > ""
>> > ],
>> > "period_map": {
>> > "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
>> > "zonegroups": [
>> > {
>> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e",
>> > "name": "us",
>> > "api_name": "us",
>> > "is_master": "true",
>> > "endpoints": [
>> > "http:\/\/ELB_FQDN:80"
>> > ],
>> > "hostnames": [],
>> > "hostnames_s3website": [],
>> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "zones": [
>> > {
>> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
>> > "name": "us-phx",
>> > "endpoints": [
>> > "http:\/\/cephrgw1-1-phx:80"
>> > ],
>> > "log_meta": "false",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > },
>> > {
>> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "name": "us-dfw",
>> > "endpoints": [
>> > "http:\/\/cephrgw1-1-dfw:80"
>> > ],
>> > "log_meta": "true",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > }
>> > ],
>> > "placement_targets": [
>> > {
>> > "name": "default-placement",
>> > "tags": []
>> > }
>> > ],
>> > "default_placement": "default-placement",
>> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
>> > }
>> > ],
>> > "short_zone_ids": [
>> > {
>> > "key": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
>> > "val": 1694436416
>> > },
>> > {
>> > "key": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "val": 2500379964
>> > }
>> > ]
>> > },
>> > "master_zonegroup": "235b010c-22e2-4b43-8fcc-8ae01939273e",
>> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "period_config": {
>> > "bucket_quota": {
>> > "enabled": false,
>> > "max_size_kb": -1,
>> > "max_objects": -1
>> > },
>> > "user_quota": {
>> > "enabled": false,
>> > "max_size_kb": -1,
>> > "max_objects": -1
>> > }
>> > },
>> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19",
>> > "realm_name": "my_realm",
>> > "realm_epoch": 2
>> > }
>> >
>> > rgw1-2-dfw
>> > radosgw-admin realm get
>> > {
>> > "id": "3af93a86-916a-490f-b38f-17922b472b19",
>> > "name": "my_realm",
>> > "current_period": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
>> > "epoch": 2
>> > }
>> >
>> > radosgw-admin period get
>> > {
>> > "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
>> > "epoch": 4,
>> > "predecessor_uuid": "ab3ffbbb-0fc9-49f7-a44f-e804f9ed7697",
>> > "sync_status": [
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > ""
>> > ],
>> > "period_map": {
>> > "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
>> > "zonegroups": [
>> > {
>> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e",
>> > "name": "us",
>> > "api_name": "us",
>> > "is_master": "true",
>> > "endpoints": [
>> > "http:\/\/ELB_FQDN:80"
>> > ],
>> > "hostnames": [],
>> > "hostnames_s3website": [],
>> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "zones": [
>> > {
>> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
>> > "name": "us-phx",
>> > "endpoints": [
>> > "http:\/\/cephrgw1-1-phx:80"
>> > ],
>> > "log_meta": "false",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > },
>> > {
>> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "name": "us-dfw",
>> > "endpoints": [
>> > "http:\/\/cephrgw1-1-dfw:80"
>> > ],
>> > "log_meta": "true",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > }
>> > ],
>> > "placement_targets": [
>> > {
>> > "name": "default-placement",
>> > "tags": []
>> > }
>> > ],
>> > "default_placement": "default-placement",
>> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
>> > }
>> > ],
>> > "short_zone_ids": [
>> > {
>> > "key": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
>> > "val": 1694436416
>> > },
>> > {
>> > "key": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "val": 2500379964
>> > }
>> > ]
>> > },
>> > "master_zonegroup": "235b010c-22e2-4b43-8fcc-8ae01939273e",
>> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "period_config": {
>> > "bucket_quota": {
>> > "enabled": false,
>> > "max_size_kb": -1,
>> > "max_objects": -1
>> > },
>> > "user_quota": {
>> > "enabled": false,
>> > "max_size_kb": -1,
>> > "max_objects": -1
>> > }
>> > },
>> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19",
>> > "realm_name": "my_realm",
>> > "realm_epoch": 2
>> > }
>> >
>> > rgw1-3-dfw
>> > radosgw-admin realm get
>> > radosgw-admin realm get
>> > {
>> > "id": "3af93a86-916a-490f-b38f-17922b472b19",
>> > "name": "my_realm",
>> > "current_period": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
>> > "epoch": 2
>> > }
>> >
>> > radosgw-admin period get
>> > {
>> > "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
>> > "epoch": 4,
>> > "predecessor_uuid": "ab3ffbbb-0fc9-49f7-a44f-e804f9ed7697",
>> > "sync_status": [
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > ""
>> > ],
>> > "period_map": {
>> > "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
>> > "zonegroups": [
>> > {
>> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e",
>> > "name": "us",
>> > "api_name": "us",
>> > "is_master": "true",
>> > "endpoints": [
>> > "http:\/\/ELB_FQDN:80"
>> > ],
>> > "hostnames": [],
>> > "hostnames_s3website": [],
>> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "zones": [
>> > {
>> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
>> > "name": "us-phx",
>> > "endpoints": [
>> > "http:\/\/cephrgw1-1-phx:80"
>> > ],
>> > "log_meta": "false",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > },
>> > {
>> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "name": "us-dfw",
>> > "endpoints": [
>> > "http:\/\/cephrgw1-1-dfw:80"
>> > ],
>> > "log_meta": "true",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > }
>> > ],
>> > "placement_targets": [
>> > {
>> > "name": "default-placement",
>> > "tags": []
>> > }
>> > ],
>> > "default_placement": "default-placement",
>> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
>> > }
>> > ],
>> > "short_zone_ids": [
>> > {
>> > "key": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
>> > "val": 1694436416
>> > },
>> > {
>> > "key": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "val": 2500379964
>> > }
>> > ]
>> > },
>> > "master_zonegroup": "235b010c-22e2-4b43-8fcc-8ae01939273e",
>> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "period_config": {
>> > "bucket_quota": {
>> > "enabled": false,
>> > "max_size_kb": -1,
>> > "max_objects": -1
>> > },
>> > "user_quota": {
>> > "enabled": false,
>> > "max_size_kb": -1,
>> > "max_objects": -1
>> > }
>> > },
>> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19",
>> > "realm_name": "my_realm",
>> > "realm_epoch": 2
>> > }
>> >
>> >
>> > PHX (secondary)
>> > radosgw-admin realm get
>> > {
>> > "id": "3af93a86-916a-490f-b38f-17922b472b19",
>> > "name": "my_realm",
>> > "current_period": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
>> > "epoch": 2
>> > }
>> > radosgw-admin period get
>> > {
>> > "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
>> > "epoch": 4,
>> > "predecessor_uuid": "ab3ffbbb-0fc9-49f7-a44f-e804f9ed7697",
>> > "sync_status": [
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > ""
>> > ],
>> > "period_map": {
>> > "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
>> > "zonegroups": [
>> > {
>> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e",
>> > "name": "us",
>> > "api_name": "us",
>> > "is_master": "true",
>> > "endpoints": [
>> > "http:\/\/ELB_FQDN:80"
>> > ],
>> > "hostnames": [],
>> > "hostnames_s3website": [],
>> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "zones": [
>> > {
>> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
>> > "name": "us-phx",
>> > "endpoints": [
>> > "http:\/\/cephrgw1-1-phx:80"
>> > ],
>> > "log_meta": "false",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > },
>> > {
>> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "name": "us-dfw",
>> > "endpoints": [
>> > "http:\/\/cephrgw1-1-dfw:80"
>> > ],
>> > "log_meta": "true",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > }
>> > ],
>> > "placement_targets": [
>> > {
>> > "name": "default-placement",
>> > "tags": []
>> > }
>> > ],
>> > "default_placement": "default-placement",
>> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
>> > }
>> > ],
>> > "short_zone_ids": [
>> > {
>> > "key": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
>> > "val": 1694436416
>> > },
>> > {
>> > "key": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "val": 2500379964
>> > }
>> > ]
>> > },
>> > "master_zonegroup": "235b010c-22e2-4b43-8fcc-8ae01939273e",
>> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "period_config": {
>> > "bucket_quota": {
>> > "enabled": false,
>> > "max_size_kb": -1,
>> > "max_objects": -1
>> > },
>> > "user_quota": {
>> > "enabled": false,
>> > "max_size_kb": -1,
>> > "max_objects": -1
>> > }
>> > },
>> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19",
>> > "realm_name": "my_realm",
>> > "realm_epoch": 2
>> > }
>> >
>> > cephrgw1-3-phx
>> > radosgw-admin realm get
>> > {
>> > "id": "3af93a86-916a-490f-b38f-17922b472b19",
>> > "name": "pardot",
>> > "current_period": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
>> > "epoch": 2
>> > }
>> > radosgw-admin period get
>> > {
>> > "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
>> > "epoch": 4,
>> > "predecessor_uuid": "ab3ffbbb-0fc9-49f7-a44f-e804f9ed7697",
>> > "sync_status": [
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > ""
>> > ],
>> > "period_map": {
>> > "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
>> > "zonegroups": [
>> > {
>> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e",
>> > "name": "us",
>> > "api_name": "us",
>> > "is_master": "true",
>> > "endpoints": [
>> > "http:\/\/ELB_FQDN:80"
>> > ],
>> > "hostnames": [],
>> > "hostnames_s3website": [],
>> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "zones": [
>> > {
>> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
>> > "name": "us-phx",
>> > "endpoints": [
>> > "http:\/\/cephrgw1-1-phx:80"
>> > ],
>> > "log_meta": "false",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > },
>> > {
>> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "name": "us-dfw",
>> > "endpoints": [
>> > "http:\/\/cephrgw1-1-dfw:80"
>> > ],
>> > "log_meta": "true",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > }
>> > ],
>> > "placement_targets": [
>> > {
>> > "name": "default-placement",
>> > "tags": []
>> > }
>> > ],
>> > "default_placement": "default-placement",
>> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
>> > }
>> > ],
>> > "short_zone_ids": [
>> > {
>> > "key": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
>> > "val": 1694436416
>> > },
>> > {
>> > "key": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "val": 2500379964
>> > }
>> > ]
>> > },
>> > "master_zonegroup": "235b010c-22e2-4b43-8fcc-8ae01939273e",
>> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "period_config": {
>> > "bucket_quota": {
>> > "enabled": false,
>> > "max_size_kb": -1,
>> > "max_objects": -1
>> > },
>> > "user_quota": {
>> > "enabled": false,
>> > "max_size_kb": -1,
>> > "max_objects": -1
>> > }
>> > },
>> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19",
>> > "realm_name": "my_realm",
>> > "realm_epoch": 2
>> > }
>> >
>> > cephrgw-1-3-phx
>> > radosgw-admin realm get
>> > {
>> > "id": "3af93a86-916a-490f-b38f-17922b472b19",
>> > "name": "pardot",
>>
>> In other nodes the name is "my_realm" , maybe that is the problem.
>> You can try fixing it by renaming it using realm rename or using realm
>> set command
>>
>> Orit
>> > "current_period": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
>> > "epoch": 2
>> > }
>> >
>> > radsogw-admin period get
>> > {
>> > "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
>> > "epoch": 4,
>> > "predecessor_uuid": "ab3ffbbb-0fc9-49f7-a44f-e804f9ed7697",
>> > "sync_status": [
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > "",
>> > ""
>> > ],
>> > "period_map": {
>> > "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
>> > "zonegroups": [
>> > {
>> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e",
>> > "name": "us",
>> > "api_name": "us",
>> > "is_master": "true",
>> > "endpoints": [
>> > "http:\/\/ELB_FQDN:80"
>> > ],
>> > "hostnames": [],
>> > "hostnames_s3website": [],
>> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "zones": [
>> > {
>> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
>> > "name": "us-phx",
>> > "endpoints": [
>> > "http:\/\/cephrgw1-1-phx:80"
>> > ],
>> > "log_meta": "false",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > },
>> > {
>> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "name": "us-dfw",
>> > "endpoints": [
>> > "http:\/\/cephrgw1-1-dfw:80"
>> > ],
>> > "log_meta": "true",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > }
>> > ],
>> > "placement_targets": [
>> > {
>> > "name": "default-placement",
>> > "tags": []
>> > }
>> > ],
>> > "default_placement": "default-placement",
>> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
>> > }
>> > ],
>> > "short_zone_ids": [
>> > {
>> > "key": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
>> > "val": 1694436416
>> > },
>> > {
>> > "key": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "val": 2500379964
>> > }
>> > ]
>> > },
>> > "master_zonegroup": "235b010c-22e2-4b43-8fcc-8ae01939273e",
>> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "period_config": {
>> > "bucket_quota": {
>> > "enabled": false,
>> > "max_size_kb": -1,
>> > "max_objects": -1
>> > },
>> > "user_quota": {
>> > "enabled": false,
>> > "max_size_kb": -1,
>> > "max_objects": -1
>> > }
>> > },
>> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19",
>> > "realm_name": "my_realm",
>> > "realm_epoch": 2
>> > }
>> >
>> >
>> > On Mon, Sep 26, 2016 at 5:27 AM Orit Wasserman <owass...@redhat.com>
>> wrote:
>> >>
>> >> Hi John,
>> >> Can you provide:
>> >> radosgw-admin zonegroupmap get on both us-dfw and us-phx?
>> >> radosgw-admin realm get and radosgw-admin period get on all the
>> gateways?
>> >>
>> >> Orit
>> >>
>> >> On Thu, Sep 22, 2016 at 4:37 PM, John Rowe <john.r...@salesforce.com>
>> >> wrote:
>> >> > Hello Orit, thanks.
>> >> >
>> >> > I will do all 6 just in case. Also as an FYI I originally had all 6
>> as
>> >> > endpoint (3 in each zone) but have it down to just the two "1"
>> servers
>> >> > talking to each other until I can get it working. Eventually I would
>> >> > like to
>> >> > have all 6 cross connecting again.
>> >> >
>> >> > rgw-primary-1:
>> >> > radosgw-admin zonegroup get
>> >> > {
>> >> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e",
>> >> > "name": "us",
>> >> > "api_name": "us",
>> >> > "is_master": "true",
>> >> > "endpoints": [
>> >> > "http:\/\/LB_FQDN:80"
>> >> > ],
>> >> > "hostnames": [],
>> >> > "hostnames_s3website": [],
>> >> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> >> > "zones": [
>> >> > {
>> >> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
>> >> > "name": "us-phx",
>> >> > "endpoints": [
>> >> > "http:\/\/PHX-RGW-1:80"
>> >> > ],
>> >> > "log_meta": "false",
>> >> > "log_data": "true",
>> >> > "bucket_index_max_shards": 0,
>> >> > "read_only": "false"
>> >> > },
>> >> > {
>> >> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> >> > "name": "us-dfw",
>> >> > "endpoints": [
>> >> > "http:\/\/DFW-RGW-1:80"
>> >> > ],
>> >> > "log_meta": "true",
>> >> > "log_data": "true",
>> >> > "bucket_index_max_shards": 0,
>> >> > "read_only": "false"
>> >> > }
>> >> > ],
>> >> > "placement_targets": [
>> >> > {
>> >> > "name": "default-placement",
>> >> > "tags": []
>> >> > }
>> >> > ],
>> >> > "default_placement": "default-placement",
>> >> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
>> >> > }
>> >> > radosgw-admin zone get
>> >> > {
>> >> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> >> > "name": "us-dfw",
>> >> > "domain_root": "us-dfw.rgw.data.root",
>> >> > "control_pool": "us-dfw.rgw.control",
>> >> > "gc_pool": "us-dfw.rgw.gc",
>> >> > "log_pool": "us-dfw.rgw.log",
>> >> > "intent_log_pool": "us-dfw.rgw.intent-log",
>> >> > "usage_log_pool": "us-dfw.rgw.usage",
>> >> > "user_keys_pool": "us-dfw.rgw.users.keys",
>> >> > "user_email_pool": "us-dfw.rgw.users.email",
>> >> > "user_swift_pool": "us-dfw.rgw.users.swift",
>> >> > "user_uid_pool": "us-dfw.rgw.users.uid",
>> >> > "system_key": {
>> >> > "access_key": "SYSTEM_ACCESS_KEY",
>> >> > "secret_key": "SYSTEM_SECRET_KEY"
>> >> > },
>> >> > "placement_pools": [
>> >> > {
>> >> > "key": "default-placement",
>> >> > "val": {
>> >> > "index_pool": "us-dfw.rgw.buckets.index",
>> >> > "data_pool": "us-dfw.rgw.buckets.data",
>> >> > "data_extra_pool": "us-dfw.rgw.buckets.non-ec",
>> >> > "index_type": 0
>> >> > }
>> >> > }
>> >> > ],
>> >> > "metadata_heap": "us-dfw.rgw.meta",
>> >> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
>> >> > }
>> >> >
>> >> > ----
>> >> > rgw-primary-2:
>> >> > radosgw-admin zonegroup get
>> >> > {
>> >> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e",
>> >> > "name": "us",
>> >> > "api_name": "us",
>> >> > "is_master": "true",
>> >> > "endpoints": [
>> >> > "http:\/\/LB_FQDN:80"
>> >> > ],
>> >> > "hostnames": [],
>> >> > "hostnames_s3website": [],
>> >> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> >> > "zones": [
>> >> > {
>> >> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
>> >> > "name": "us-phx",
>> >> > "endpoints": [
>> >> > "http:\/\/PHX-RGW-1:80"
>> >> > ],
>> >> > "log_meta": "false",
>> >> > "log_data": "true",
>> >> > "bucket_index_max_shards": 0,
>> >> > "read_only": "false"
>> >> > },
>> >> > {
>> >> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> >> > "name": "us-dfw",
>> >> > "endpoints": [
>> >> > "http:\/\/DFW-RGW-1:80"
>> >> > ],
>> >> > "log_meta": "true",
>> >> > "log_data": "true",
>> >> > "bucket_index_max_shards": 0,
>> >> > "read_only": "false"
>> >> > }
>> >> > ],
>> >> > "placement_targets": [
>> >> > {
>> >> > "name": "default-placement",
>> >> > "tags": []
>> >> > }
>> >> > ],
>> >> > "default_placement": "default-placement",
>> >> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
>> >> > }
>> >> >
>> >> > radosgw-admin zone get
>> >> > {
>> >> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> >> > "name": "us-dfw",
>> >> > "domain_root": "us-dfw.rgw.data.root",
>> >> > "control_pool": "us-dfw.rgw.control",
>> >> > "gc_pool": "us-dfw.rgw.gc",
>> >> > "log_pool": "us-dfw.rgw.log",
>> >> > "intent_log_pool": "us-dfw.rgw.intent-log",
>> >> > "usage_log_pool": "us-dfw.rgw.usage",
>> >> > "user_keys_pool": "us-dfw.rgw.users.keys",
>> >> > "user_email_pool": "us-dfw.rgw.users.email",
>> >> > "user_swift_pool": "us-dfw.rgw.users.swift",
>> >> > "user_uid_pool": "us-dfw.rgw.users.uid",
>> >> > "system_key": {
>> >> > "access_key": "SYSTEM_ACCESS_KEY",
>> >> > "secret_key": "SYSTEM_SECRET_KEY"
>> >> > },
>> >> > "placement_pools": [
>> >> > {
>> >> > "key": "default-placement",
>> >> > "val": {
>> >> > "index_pool": "us-dfw.rgw.buckets.index",
>> >> > "data_pool": "us-dfw.rgw.buckets.data",
>> >> > "data_extra_pool": "us-dfw.rgw.buckets.non-ec",
>> >> > "index_type": 0
>> >> > }
>> >> > }
>> >> > ],
>> >> > "metadata_heap": "us-dfw.rgw.meta",
>> >> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
>> >> > }
>> >> >
>> >> > ----
>> >> > rgw-primary-3:
>> >> > radosgw-admin zonegroup get
>> >> > {
>> >> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e",
>> >> > "name": "us",
>> >> > "api_name": "us",
>> >> > "is_master": "true",
>> >> > "endpoints": [
>> >> > "http:\/\/LB_FQDN:80"
>> >> > ],
>> >> > "hostnames": [],
>> >> > "hostnames_s3website": [],
>> >> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> >> > "zones": [
>> >> > {
>> >> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
>> >> > "name": "us-phx",
>>
>
_______________________________________________
ceph-users mailing list
ceph-users@lists.ceph.com
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
