sursingh opened a new issue #10860: URL: https://github.com/apache/pulsar/issues/10860
**Describe the bug** When doing operations on multiple brokers, the state is not always consistent. Sometimes the operations don't seem to get replicated to other brokers in the cluster. The queries from the brokers returns different results **To Reproduce** The steps below are for namespaces. But this should also be applicable to other global state. Steps to reproduce the behavior: 1. create a multi broker setup 2. create namespace on broker0 3. verify that namespace exists on all brokers 4. delete namespace on broker1 5. verify that namespace is removed from all brokers 5.1 The namespace is removed from broker1 5.2 The namespace still exists on broker0. This is not expected. The namespace should also be removed from broker0 **Expected behavior** namespace should be removed from all brokers. **Screenshots** ``` # Create on broker0 sh-3.2$ curl -H "Authorization: Bearer $token" -w "\n" -X PUT http://localhost:8080/admin/v2/namespaces/public/test-namespace-1 # namespace exists on broker0 sh-3.2$ curl -H "Authorization: Bearer $token" -w "\n" -X GET http://localhost:8080/admin/v2/namespaces/public/test-namespace-1 {"auth_policies":{"namespace_auth":{},"destination_auth":{},"subscription_auth_roles":{}},"replication_clusters":["play1.dev.us-west-2.splunk8s.io"],"bundles":{"boundaries":["0x00000000","0x04000000","0x08000000","0x0c000000","0x10000000","0x14000000","0x18000000","0x1c000000","0x20000000","0x24000000","0x28000000","0x2c000000","0x30000000","0x34000000","0x38000000","0x3c000000","0x40000000","0x44000000","0x48000000","0x4c000000","0x50000000","0x54000000","0x58000000","0x5c000000","0x60000000","0x64000000","0x68000000","0x6c000000","0x70000000","0x74000000","0x78000000","0x7c000000","0x80000000","0x84000000","0x88000000","0x8c000000","0x90000000","0x94000000","0x98000000","0x9c000000","0xa0000000","0xa4000000","0xa8000000","0xac000000","0xb0000000","0xb4000000","0xb8000000","0xbc000000","0xc0000000","0xc4000000","0xc8000000","0xcc000000","0xd0000000","0xd4000000","0xd8000000","0xdc000000","0xe0000000","0xe4000000","0xe8000000","0xec000000","0xf0000000","0xf4000000","0xf8000000","0 xfc000000","0xffffffff"],"numBundles":64},"backlog_quota_map":{},"clusterDispatchRate":{},"topicDispatchRate":{},"subscriptionDispatchRate":{},"replicatorDispatchRate":{},"clusterSubscribeRate":{},"publishMaxMessageRate":{},"latency_stats_sample_rate":{},"subscription_expiration_time_minutes":0,"deleted":false,"encryption_required":false,"subscription_auth_mode":"None","offload_threshold":-1,"schema_auto_update_compatibility_strategy":"Full","schema_compatibility_strategy":"UNDEFINED","is_allow_auto_update_schema":true,"schema_validation_enforced":false,"subscription_types_enabled":[],"properties":{}} # namespace exists on broker1 sh-3.2$ curl -H "Authorization: Bearer $token" -w "\n" -X GET http://localhost:8081/admin/v2/namespaces/public/test-namespace-1 {"auth_policies":{"namespace_auth":{},"destination_auth":{},"subscription_auth_roles":{}},"replication_clusters":["play1.dev.us-west-2.splunk8s.io"],"bundles":{"boundaries":["0x00000000","0x04000000","0x08000000","0x0c000000","0x10000000","0x14000000","0x18000000","0x1c000000","0x20000000","0x24000000","0x28000000","0x2c000000","0x30000000","0x34000000","0x38000000","0x3c000000","0x40000000","0x44000000","0x48000000","0x4c000000","0x50000000","0x54000000","0x58000000","0x5c000000","0x60000000","0x64000000","0x68000000","0x6c000000","0x70000000","0x74000000","0x78000000","0x7c000000","0x80000000","0x84000000","0x88000000","0x8c000000","0x90000000","0x94000000","0x98000000","0x9c000000","0xa0000000","0xa4000000","0xa8000000","0xac000000","0xb0000000","0xb4000000","0xb8000000","0xbc000000","0xc0000000","0xc4000000","0xc8000000","0xcc000000","0xd0000000","0xd4000000","0xd8000000","0xdc000000","0xe0000000","0xe4000000","0xe8000000","0xec000000","0xf0000000","0xf4000000","0xf8000000","0 xfc000000","0xffffffff"],"numBundles":64},"backlog_quota_map":{},"clusterDispatchRate":{},"topicDispatchRate":{},"subscriptionDispatchRate":{},"replicatorDispatchRate":{},"clusterSubscribeRate":{},"publishMaxMessageRate":{},"latency_stats_sample_rate":{},"subscription_expiration_time_minutes":0,"deleted":false,"encryption_required":false,"subscription_auth_mode":"None","offload_threshold":-1,"schema_auto_update_compatibility_strategy":"Full","schema_compatibility_strategy":"UNDEFINED","is_allow_auto_update_schema":true,"schema_validation_enforced":false,"subscription_types_enabled":[],"properties":{}} # delete on borker1 sh-3.2$ curl -H "Authorization: Bearer $token" -w "\n" -X DELETE http://localhost:8081/admin/v2/namespaces/public/test-namespace-1 # namespace removed on broker1 sh-3.2$ curl -H "Authorization: Bearer $token" -w "\n" -X GET http://localhost:8081/admin/v2/namespaces/public/test-namespace-1 {"reason":"Namespace does not exist"} # But it still exists on broker0 sh-3.2$ curl -H "Authorization: Bearer $token" -w "\n" -X GET http://localhost:8080/admin/v2/namespaces/public/test-namespace-1 {"auth_policies":{"namespace_auth":{},"destination_auth":{},"subscription_auth_roles":{}},"replication_clusters":["play1.dev.us-west-2.splunk8s.io"],"bundles":{"boundaries":["0x00000000","0x04000000","0x08000000","0x0c000000","0x10000000","0x14000000","0x18000000","0x1c000000","0x20000000","0x24000000","0x28000000","0x2c000000","0x30000000","0x34000000","0x38000000","0x3c000000","0x40000000","0x44000000","0x48000000","0x4c000000","0x50000000","0x54000000","0x58000000","0x5c000000","0x60000000","0x64000000","0x68000000","0x6c000000","0x70000000","0x74000000","0x78000000","0x7c000000","0x80000000","0x84000000","0x88000000","0x8c000000","0x90000000","0x94000000","0x98000000","0x9c000000","0xa0000000","0xa4000000","0xa8000000","0xac000000","0xb0000000","0xb4000000","0xb8000000","0xbc000000","0xc0000000","0xc4000000","0xc8000000","0xcc000000","0xd0000000","0xd4000000","0xd8000000","0xdc000000","0xe0000000","0xe4000000","0xe8000000","0xec000000","0xf0000000","0xf4000000","0xf8000000","0 xfc000000","0xffffffff"],"numBundles":64},"backlog_quota_map":{},"clusterDispatchRate":{},"topicDispatchRate":{},"subscriptionDispatchRate":{},"replicatorDispatchRate":{},"clusterSubscribeRate":{},"publishMaxMessageRate":{},"latency_stats_sample_rate":{},"subscription_expiration_time_minutes":0,"deleted":false,"encryption_required":false,"subscription_auth_mode":"None","offload_threshold":-1,"schema_auto_update_compatibility_strategy":"Full","schema_compatibility_strategy":"UNDEFINED","is_allow_auto_update_schema":true,"schema_validation_enforced":false,"subscription_types_enabled":[],"properties":{}} # this persists even after a long wait sh-3.2$ sleep 60 sh-3.2$ curl -H "Authorization: Bearer $token" -w "\n" -X GET http://localhost:8080/admin/v2/namespaces/public/test-namespace-1 {"auth_policies":{"namespace_auth":{},"destination_auth":{},"subscription_auth_roles":{}},"replication_clusters":["play1.dev.us-west-2.splunk8s.io"],"bundles":{"boundaries":["0x00000000","0x04000000","0x08000000","0x0c000000","0x10000000","0x14000000","0x18000000","0x1c000000","0x20000000","0x24000000","0x28000000","0x2c000000","0x30000000","0x34000000","0x38000000","0x3c000000","0x40000000","0x44000000","0x48000000","0x4c000000","0x50000000","0x54000000","0x58000000","0x5c000000","0x60000000","0x64000000","0x68000000","0x6c000000","0x70000000","0x74000000","0x78000000","0x7c000000","0x80000000","0x84000000","0x88000000","0x8c000000","0x90000000","0x94000000","0x98000000","0x9c000000","0xa0000000","0xa4000000","0xa8000000","0xac000000","0xb0000000","0xb4000000","0xb8000000","0xbc000000","0xc0000000","0xc4000000","0xc8000000","0xcc000000","0xd0000000","0xd4000000","0xd8000000","0xdc000000","0xe0000000","0xe4000000","0xe8000000","0xec000000","0xf0000000","0xf4000000","0xf8000000","0 xfc000000","0xffffffff"],"numBundles":64},"backlog_quota_map":{},"clusterDispatchRate":{},"topicDispatchRate":{},"subscriptionDispatchRate":{},"replicatorDispatchRate":{},"clusterSubscribeRate":{},"publishMaxMessageRate":{},"latency_stats_sample_rate":{},"subscription_expiration_time_minutes":0,"deleted":false,"encryption_required":false,"subscription_auth_mode":"None","offload_threshold":-1,"schema_auto_update_compatibility_strategy":"Full","schema_compatibility_strategy":"UNDEFINED","is_allow_auto_update_schema":true,"schema_validation_enforced":false,"subscription_types_enabled":[],"properties":{}} sh-3.2$ curl -H "Authorization: Bearer $token" -w "\n" -X GET http://localhost:8081/admin/v2/namespaces/public/test-namespace-1 {"reason":"Namespace does not exist"} ``` If applicable, add screenshots to help explain your problem. **Desktop (please complete the following information):** - OS: [e.g. iOS] **Additional context** Add any other context about the problem here. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
