anant-ahuja commented on issue #14082:
URL: https://github.com/apache/pulsar/issues/14082#issuecomment-1028472063
Issue happened just now to a topic with just 1 message. Message isn't being
consumed by the consumers.
Topic stats:
`{
"msgRateIn" : 0.0,
"msgThroughputIn" : 0.0,
"msgRateOut" : 0.0,
"msgThroughputOut" : 0.0,
"bytesInCounter" : 2071,
"msgInCounter" : 1,
"bytesOutCounter" : 2071,
"msgOutCounter" : 1,
"averageMsgSize" : 0.0,
"msgChunkPublished" : false,
"storageSize" : 4142,
"backlogSize" : 2071,
"offloadedStorageSize" : 0,
"publishers" : [ ],
"subscriptions" : {
"connector-manager" : {
"msgRateOut" : 0.0,
"msgThroughputOut" : 0.0,
"bytesOutCounter" : 2071,
"msgOutCounter" : 1,
"msgRateRedeliver" : 0.0,
"chuckedMessageRate" : 0,
"msgBacklog" : 1,
"backlogSize" : 0,
"msgBacklogNoDelayed" : 1,
"blockedSubscriptionOnUnackedMsgs" : false,
"msgDelayed" : 0,
"unackedMessages" : 1,
"type" : "Shared",
"msgRateExpired" : 0.0,
"totalMsgExpired" : 0,
"lastExpireTimestamp" : 0,
"lastConsumedFlowTimestamp" : 1643841867623,
"lastConsumedTimestamp" : 1643845638551,
"lastAckedTimestamp" : 0,
"lastMarkDeleteAdvancedTimestamp" : 0,
"consumers" : [ {
"msgRateOut" : 0.0,
"msgThroughputOut" : 0.0,
"bytesOutCounter" : 2071,
"msgOutCounter" : 1,
"msgRateRedeliver" : 0.0,
"chuckedMessageRate" : 0.0,
"consumerName" : "",
"availablePermits" : 0,
"unackedMessages" : 1,
"avgMessagesPerEntry" : 900,
"blockedConsumerOnUnackedMsgs" : false,
"lastAckedTimestamp" : 0,
"lastConsumedTimestamp" : 1643845638551,
"metadata" : { },
"connectedSince" : "2022-02-02T22:44:27.556Z",
"address" : "/10.32.8.9:37096"
}, {
"msgRateOut" : 0.0,
"msgThroughputOut" : 0.0,
"bytesOutCounter" : 0,
"msgOutCounter" : 0,
"msgRateRedeliver" : 0.0,
"chuckedMessageRate" : 0.0,
"consumerName" : "",
"availablePermits" : 1,
"unackedMessages" : 0,
"avgMessagesPerEntry" : 1000,
"blockedConsumerOnUnackedMsgs" : false,
"lastAckedTimestamp" : 0,
"lastConsumedTimestamp" : 0,
"metadata" : { },
"connectedSince" : "2022-02-02T22:44:27.557Z",
"address" : "/10.32.8.9:37084"
}, {
"msgRateOut" : 0.0,
"msgThroughputOut" : 0.0,
"bytesOutCounter" : 0,
"msgOutCounter" : 0,
"msgRateRedeliver" : 0.0,
"chuckedMessageRate" : 0.0,
"consumerName" : "",
"availablePermits" : 1,
"unackedMessages" : 0,
"avgMessagesPerEntry" : 1000,
"blockedConsumerOnUnackedMsgs" : false,
"lastAckedTimestamp" : 0,
"lastConsumedTimestamp" : 0,
"metadata" : { },
"connectedSince" : "2022-02-02T22:44:27.558Z",
"address" : "/10.32.8.9:37144"
}, {
"msgRateOut" : 0.0,
"msgThroughputOut" : 0.0,
"bytesOutCounter" : 0,
"msgOutCounter" : 0,
"msgRateRedeliver" : 0.0,
"chuckedMessageRate" : 0.0,
"consumerName" : "",
"availablePermits" : 1,
"unackedMessages" : 0,
"avgMessagesPerEntry" : 1000,
"blockedConsumerOnUnackedMsgs" : false,
"lastAckedTimestamp" : 0,
"lastConsumedTimestamp" : 0,
"metadata" : { },
"connectedSince" : "2022-02-02T22:44:27.558Z",
"address" : "/10.32.8.9:37090"
}, {
"msgRateOut" : 0.0,
"msgThroughputOut" : 0.0,
"bytesOutCounter" : 0,
"msgOutCounter" : 0,
"msgRateRedeliver" : 0.0,
"chuckedMessageRate" : 0.0,
"consumerName" : "",
"availablePermits" : 1,
"unackedMessages" : 0,
"avgMessagesPerEntry" : 1000,
"blockedConsumerOnUnackedMsgs" : false,
"lastAckedTimestamp" : 0,
"lastConsumedTimestamp" : 0,
"metadata" : { },
"connectedSince" : "2022-02-02T22:44:27.558Z",
"address" : "/10.32.8.10:56112"
}, {
"msgRateOut" : 0.0,
"msgThroughputOut" : 0.0,
"bytesOutCounter" : 0,
"msgOutCounter" : 0,
"msgRateRedeliver" : 0.0,
"chuckedMessageRate" : 0.0,
"consumerName" : "",
"availablePermits" : 1,
"unackedMessages" : 0,
"avgMessagesPerEntry" : 1000,
"blockedConsumerOnUnackedMsgs" : false,
"lastAckedTimestamp" : 0,
"lastConsumedTimestamp" : 0,
"metadata" : { },
"connectedSince" : "2022-02-02T22:44:27.559Z",
"address" : "/10.32.8.10:56200"
}, {
"msgRateOut" : 0.0,
"msgThroughputOut" : 0.0,
"bytesOutCounter" : 0,
"msgOutCounter" : 0,
"msgRateRedeliver" : 0.0,
"chuckedMessageRate" : 0.0,
"consumerName" : "",
"availablePermits" : 1,
"unackedMessages" : 0,
"avgMessagesPerEntry" : 1000,
"blockedConsumerOnUnackedMsgs" : false,
"lastAckedTimestamp" : 0,
"lastConsumedTimestamp" : 0,
"metadata" : { },
"connectedSince" : "2022-02-02T22:44:27.559Z",
"address" : "/10.32.8.10:56108"
}, {
"msgRateOut" : 0.0,
"msgThroughputOut" : 0.0,
"bytesOutCounter" : 0,
"msgOutCounter" : 0,
"msgRateRedeliver" : 0.0,
"chuckedMessageRate" : 0.0,
"consumerName" : "",
"availablePermits" : 1,
"unackedMessages" : 0,
"avgMessagesPerEntry" : 1000,
"blockedConsumerOnUnackedMsgs" : false,
"lastAckedTimestamp" : 0,
"lastConsumedTimestamp" : 0,
"metadata" : { },
"connectedSince" : "2022-02-02T22:44:27.559Z",
"address" : "/10.32.12.9:52124"
}, {
"msgRateOut" : 0.0,
"msgThroughputOut" : 0.0,
"bytesOutCounter" : 0,
"msgOutCounter" : 0,
"msgRateRedeliver" : 0.0,
"chuckedMessageRate" : 0.0,
"consumerName" : "",
"availablePermits" : 1,
"unackedMessages" : 0,
"avgMessagesPerEntry" : 1000,
"blockedConsumerOnUnackedMsgs" : false,
"lastAckedTimestamp" : 0,
"lastConsumedTimestamp" : 0,
"metadata" : { },
"connectedSince" : "2022-02-02T22:44:27.559Z",
"address" : "/10.32.12.9:52440"
}, {
"msgRateOut" : 0.0,
"msgThroughputOut" : 0.0,
"bytesOutCounter" : 0,
"msgOutCounter" : 0,
"msgRateRedeliver" : 0.0,
"chuckedMessageRate" : 0.0,
"consumerName" : "",
"availablePermits" : 1,
"unackedMessages" : 0,
"avgMessagesPerEntry" : 1000,
"blockedConsumerOnUnackedMsgs" : false,
"lastAckedTimestamp" : 0,
"lastConsumedTimestamp" : 0,
"metadata" : { },
"connectedSince" : "2022-02-02T22:44:27.566Z",
"address" : "/10.32.8.10:56184"
} ],
"isDurable" : true,
"isReplicated" : false,
"consumersAfterMarkDeletePosition" : { },
"nonContiguousDeletedMessagesRanges" : 0,
"nonContiguousDeletedMessagesRangesSerializedSize" : 0
}
},
"replication" : { },
"deduplicationStatus" : "Disabled",
"nonContiguousDeletedMessagesRanges" : 0,
"nonContiguousDeletedMessagesRangesSerializedSize" : 0
}`
Topic internal stats:
`{
"entriesAddedCounter" : 1,
"numberOfEntries" : 2,
"totalSize" : 4142,
"currentLedgerEntries" : 1,
"currentLedgerSize" : 2071,
"lastLedgerCreatedTimestamp" : "2022-02-02T22:44:27.548Z",
"waitingCursorsCount" : 1,
"pendingAddEntriesCount" : 0,
"lastConfirmedEntry" : "1060:0",
"state" : "LedgerOpened",
"ledgers" : [ {
"ledgerId" : 1015,
"entries" : 1,
"size" : 2071,
"offloaded" : false,
"underReplicated" : false
}, {
"ledgerId" : 1060,
"entries" : 0,
"size" : 0,
"offloaded" : false,
"underReplicated" : false
} ],
"cursors" : {
"connector-manager" : {
"markDeletePosition" : "1015:0",
"readPosition" : "1060:1",
"waitingReadOp" : true,
"pendingReadOps" : 0,
"messagesConsumedCounter" : 0,
"cursorLedger" : 1063,
"cursorLedgerLastEntry" : 23,
"individuallyDeletedMessages" : "[]",
"lastLedgerSwitchTimestamp" : "2022-02-02T22:44:27.551Z",
"state" : "Open",
"numberOfEntriesSinceFirstNotAckedMessage" : 2,
"totalNonContiguousDeletedMessagesRange" : 0,
"properties" : { }
}
},
"schemaLedgers" : [ ],
"compactedLedger" : {
"ledgerId" : -1,
"entries" : -1,
"size" : -1,
"offloaded" : false,
"underReplicated" : false
}
}`
I know it's just for 1 message right now but it's also happened with a
backlog of 30-40k messages, where consumers stop receiving backlog messages.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
