This is an automated email from the ASF dual-hosted git repository. shuwenwei pushed a commit to branch supports_subscription in repository https://gitbox.apache.org/repos/asf/iotdb-client-go.git
commit a9a95eacd4a33a2710d19d7bf68eed151b94698f Author: shuwenwei <[email protected]> AuthorDate: Mon Nov 16 18:15:51 2026 +0800 Fix subscription client parity gaps --- client/protocol.go | 443 ++++++++++++++++++++++++++------- subscription/README.md | 9 +- subscription/consumer/config.go | 6 +- subscription/consumer/provider.go | 82 ++++-- subscription/consumer/pull_consumer.go | 29 ++- subscription/consumer/pull_flow.go | 41 +-- subscription/consumer/pull_progress.go | 21 +- subscription/errors.go | 46 +++- 8 files changed, 521 insertions(+), 156 deletions(-) diff --git a/client/protocol.go b/client/protocol.go index 111e642..fe3dc0f 100644 --- a/client/protocol.go +++ b/client/protocol.go @@ -114,92 +114,175 @@ const ( // TSStatusCode const ( - SuccessStatus int32 = 200 + SuccessStatus int32 = 200 + + // System level IncompatibleVersion int32 = 201 ConfigurationError int32 = 202 StartUpError int32 = 203 ShutDownError int32 = 204 - UnsupportedOperation int32 = 300 - ExecuteStatementError int32 = 301 - MultipleError int32 = 302 - IllegalParameter int32 = 303 - OverlapWithExistingTask int32 = 304 - InternalServerError int32 = 305 + UnsupportedSqlDialect int32 = 205 + + PortOccupied int32 = 206 + // General Error + UnsupportedOperation int32 = 300 + ExecuteStatementError int32 = 301 + MultipleError int32 = 302 + IllegalParameter int32 = 303 + OverlapWithExistingTask int32 = 304 + InternalServerError int32 = 305 + DispatchError int32 = 306 + LicenseError int32 = 307 + ThriftFrameOversize int32 = 308 + RepeatedWithExistingTask int32 = 309 + + // Client RedirectionRecommend int32 = 400 - DatabaseNotExist int32 = 500 - DatabaseAlreadyExists int32 = 501 - SeriesOverflow int32 = 502 - TimeseriesAlreadyExist int32 = 503 - TimeseriesInBlackList int32 = 504 - AliasAlreadyExist int32 = 505 - PathAlreadyExist int32 = 506 - MetadataError int32 = 507 - PathNotExist int32 = 508 - IllegalPath int32 = 509 - CreateTemplateError int32 = 510 - DuplicatedTemplate int32 = 511 - UndefinedTemplate int32 = 512 - TemplateNotSet int32 = 513 - DifferentTemplate int32 = 514 - TemplateIsInUse int32 = 515 - TemplateIncompatible int32 = 516 - SegmentNotFound int32 = 517 - PageOutOfSpace int32 = 518 - RecordDuplicated int32 = 519 - SegmentOutOfSpace int32 = 520 - SchemaFileNotExists int32 = 521 - OversizeRecord int32 = 522 - SchemaFileRedoLogBroken int32 = 523 - TemplateNotActivated int32 = 524 - - SystemReadOnly int32 = 600 - StorageEngineError int32 = 601 - StorageEngineNotReady int32 = 602 - DataregionProcessError int32 = 603 - TsfileProcessorError int32 = 604 - WriteProcessError int32 = 605 - WriteProcessReject int32 = 606 - OutOfTtl int32 = 607 - CompactionError int32 = 608 - AlignedTimeseriesError int32 = 609 - WalError int32 = 610 - DiskSpaceInsufficient int32 = 611 - - SqlParseError int32 = 700 - SemanticError int32 = 701 - GenerateTimeZoneError int32 = 702 - SetTimeZoneError int32 = 703 - QueryNotAllowed int32 = 704 - LogicalOperatorError int32 = 705 - LogicalOptimizeError int32 = 706 - UnsupportedFillType int32 = 707 - QueryProcessError int32 = 708 - MppMemoryNotEnough int32 = 709 - CloseOperationError int32 = 710 - TsblockSerializeError int32 = 711 - InternalRequestTimeOut int32 = 712 - InternalRequestRetryError int32 = 713 - - InitAuthError int32 = 800 - WrongLoginPassword int32 = 801 - NotLogin int32 = 802 - NoPermission int32 = 803 - UserNotExist int32 = 804 - UserAlreadyExist int32 = 805 - UserAlreadyHasRole int32 = 806 - UserNotHasRole int32 = 807 - RoleNotExist int32 = 808 - RoleAlreadyExist int32 = 809 - AlreadyHasPrivilege int32 = 810 - NotHasPrivilege int32 = 811 - ClearPermissionCacheError int32 = 812 - UnknownAuthPrivilege int32 = 813 - UnsupportedAuthOperation int32 = 814 - AuthIoException int32 = 815 + UseOrDropDb int32 = 401 + + // Schema Engine + DatabaseNotExist int32 = 500 + DatabaseAlreadyExists int32 = 501 + SeriesOverflow int32 = 502 + TimeseriesAlreadyExist int32 = 503 + TimeseriesInBlackList int32 = 504 + AliasAlreadyExist int32 = 505 + PathAlreadyExist int32 = 506 + MetadataError int32 = 507 + PathNotExist int32 = 508 + IllegalPath int32 = 509 + CreateTemplateError int32 = 510 + DuplicatedTemplate int32 = 511 + UndefinedTemplate int32 = 512 + TemplateNotSet int32 = 513 + DifferentTemplate int32 = 514 + TemplateIsInUse int32 = 515 + TemplateIncompatible int32 = 516 + SegmentNotFound int32 = 517 + PageOutOfSpace int32 = 518 + RecordDuplicated int32 = 519 + SegmentOutOfSpace int32 = 520 + SchemaFileNotExists int32 = 521 + OversizeRecord int32 = 522 + SchemaFileRedoLogBroken int32 = 523 + TemplateNotActivated int32 = 524 + DatabaseConfigError int32 = 525 + SchemaQuotaExceeded int32 = 526 + MeasurementAlreadyExistsInTemplate int32 = 527 + TypeNotFound int32 = 528 + DatabaseConflict int32 = 529 + DatabaseModel int32 = 530 + TemplateTimeseriesCannotCreateAlias int32 = 531 + ViewTimeseriesCannotCreateAlias int32 = 532 + InvalidTimeseriesCannotCreateAlias int32 = 533 + TimeseriesAlreadyRenaming int32 = 534 + MetadataLeaseFenced int32 = 535 + MetadataLeaseFencedRetryRequired int32 = 536 + + TableNotExists int32 = 550 + TableAlreadyExists int32 = 551 + ColumnAlreadyExists int32 = 552 + TableIsLost int32 = 553 + TableIncompatible int32 = 554 + TableInPreDelete int32 = 555 + ColumnInPreDelete int32 = 556 + ColumnInPreAlter int32 = 557 + OnlyLogicalView int32 = 560 + + // Storage Engine + SystemReadOnly int32 = 600 + StorageEngineError int32 = 601 + StorageEngineNotReady int32 = 602 + DataregionProcessError int32 = 603 + TsfileProcessorError int32 = 604 + WriteProcessError int32 = 605 + WriteProcessReject int32 = 606 + OutOfTtl int32 = 607 + CompactionError int32 = 608 + AlignedTimeseriesError int32 = 609 + WalError int32 = 610 + DiskSpaceInsufficient int32 = 611 + OversizeTtl int32 = 612 + TtlConfigError int32 = 613 + DataTypeMismatch int32 = 614 + ColumnCategoryMismatch int32 = 615 + ColumnNotExists int32 = 616 + MeasurementNameConflict int32 = 617 + + WalEntryTooLarge int32 = 620 + // Query Engine + SqlParseError int32 = 700 + SemanticError int32 = 701 + GenerateTimeZoneError int32 = 702 + SetTimeZoneError int32 = 703 + QueryNotAllowed int32 = 704 + LogicalOperatorError int32 = 705 + LogicalOptimizeError int32 = 706 + UnsupportedFillType int32 = 707 + QueryProcessError int32 = 708 + MppMemoryNotEnough int32 = 709 + CloseOperationError int32 = 710 + TsblockSerializeError int32 = 711 + InternalRequestTimeOut int32 = 712 + InternalRequestRetryError int32 = 713 + NoSuchQuery int32 = 714 + QueryWasKilled int32 = 715 + ExplainAnalyzeFetchError int32 = 716 + TooManyConcurrentQueriesError int32 = 717 + + OperatorNotFound int32 = 718 + + QueryExecutionMemoryNotEnough int32 = 719 + QueryTimeout int32 = 720 + PlanFailedNetworkPartition int32 = 721 + CannotFetchFiState int32 = 722 + RepeatedRpcCall int32 = 723 + CannotReadTsfile int32 = 724 + CopyToWriteError int32 = 725 + DeviceEntrySpillNotFound int32 = 726 + + // OBJECT + ObjectNotExists int32 = 740 + ObjectInsertError int32 = 741 + ObjectReadError int32 = 742 + + // Arithmetic + NumericValueOutOfRange int32 = 750 + DivisionByZero int32 = 751 + DateOutOfRange int32 = 752 + + // Authentication + InitAuthError int32 = 800 + WrongLoginPassword int32 = 801 + NotLogin int32 = 802 + NoPermission int32 = 803 + UserNotExist int32 = 804 + UserAlreadyExist int32 = 805 + UserAlreadyHasRole int32 = 806 + UserNotHasRole int32 = 807 + RoleNotExist int32 = 808 + RoleAlreadyExist int32 = 809 + NotHasPrivilege int32 = 811 + ClearPermissionCacheError int32 = 812 + UnknownAuthPrivilege int32 = 813 + UnsupportedAuthOperation int32 = 814 + AuthIoException int32 = 815 + IllegalPrivilege int32 = 816 + NotHasPrivilegeGrantopt int32 = 817 + AuthOperateException int32 = 818 + OptimizerTimeout int32 = 819 + IllegalPassword int32 = 820 + SessionNumsExceeded int32 = 821 + UserLoginLocked int32 = 822 + TotpCodeRequired int32 = 823 + TotpCodeInvalid int32 = 824 + TotpAuthServiceUnavailable int32 = 825 + + // Partition Error MigrateRegionError int32 = 900 CreateRegionError int32 = 901 DeleteRegionError int32 = 902 @@ -207,33 +290,56 @@ const ( ConsensusNotInitialized int32 = 904 RegionLeaderChangeError int32 = 905 NoAvailableRegionGroup int32 = 906 + LackPartitionAllocation int32 = 907 + ReconstructRegionError int32 = 908 + ExtendRegionError int32 = 909 + RemoveRegionPeerError int32 = 910 + RegionAlreadyExists int32 = 911 + RegionNotExist int32 = 912 - DatanodeAlreadyRegistered int32 = 1000 - NoEnoughDatanode int32 = 1001 - AddConfignodeError int32 = 1002 - RemoveConfignodeError int32 = 1003 + // Cluster Manager + AddConfignodeError int32 = 1000 + RemoveConfignodeError int32 = 1001 + RejectNodeStart int32 = 1002 + NoEnoughDatanode int32 = 1003 DatanodeNotExist int32 = 1004 DatanodeStopError int32 = 1005 RemoveDatanodeError int32 = 1006 - RegisterRemovedDatanode int32 = 1007 - CanNotConnectDatanode int32 = 1008 - - LoadFileError int32 = 1100 - LoadPieceOfTsfileError int32 = 1101 - DeserializePieceOfTsfileError int32 = 1102 - SyncConnectionError int32 = 1103 - SyncFileRedirectionError int32 = 1104 - SyncFileError int32 = 1105 - CreatePipeSinkError int32 = 1106 - PipeError int32 = 1107 - PipeserverError int32 = 1108 - VerifyMetadataError int32 = 1109 + CanNotConnectDatanode int32 = 1007 + TransferLeaderError int32 = 1008 + GetClusterIdError int32 = 1009 + CanNotConnectConfignode int32 = 1010 + CanNotConnectAinode int32 = 1011 + NoAvailableReplica int32 = 1012 + NoAvailableAinode int32 = 1013 + ConfigNodeLeaderWarmingUp int32 = 1014 + + // Sync, Load TsFile + LoadFileError int32 = 1100 + LoadPieceOfTsfileError int32 = 1101 + DeserializePieceOfTsfileError int32 = 1102 + SyncConnectionError int32 = 1103 + SyncFileRedirectionError int32 = 1104 + SyncFileError int32 = 1105 + CreatePipeSinkError int32 = 1106 + PipeError int32 = 1107 + PipeserverError int32 = 1108 + VerifyMetadataError int32 = 1109 + LoadTemporaryUnavailableException int32 = 1110 + LoadIdempotentConflictException int32 = 1111 + LoadUserConflictException int32 = 1112 + // UDF UdfLoadClassError int32 = 1200 UdfDownloadError int32 = 1201 CreateUdfOnDatanodeError int32 = 1202 DropUdfOnDatanodeError int32 = 1203 + CreateUdfError int32 = 1204 + DropUdfError int32 = 1205 + ExecuteUdfError int32 = 1206 + UdfAlreadyExists int32 = 1207 + // Trigger CreateTriggerError int32 = 1300 DropTriggerError int32 = 1301 TriggerFireError int32 = 1302 @@ -244,10 +350,159 @@ const ( DropTriggerInstanceError int32 = 1307 UpdateTriggerLocationError int32 = 1308 + // Continuous Query NoSuchCq int32 = 1400 CqAlreadyActive int32 = 1401 CqAlreadyExist int32 = 1402 CqUpdateLastExecTimeError int32 = 1403 + NoPermissionError int32 = 1404 + + // AI + NoRegisteredAiNodeError int32 = 1500 + RegisterAiNodeError int32 = 1501 + RemoveAiNodeError int32 = 1502 + ModelExistedError int32 = 1503 + ModelNotExistError int32 = 1504 + CreateModelError int32 = 1505 + DropBuiltinModelError int32 = 1506 + DropModelError int32 = 1507 + UnavailableAiDeviceError int32 = 1508 + + AinodeInternalError int32 = 1599 // In case somebody too lazy to add a new error code + + // Pipe Plugin + CreatePipePluginError int32 = 1600 + DropPipePluginError int32 = 1601 + PipePluginLoadClassError int32 = 1602 + PipePluginDownloadError int32 = 1603 + CreatePipePluginOnDatanodeError int32 = 1604 + DropPipePluginOnDatanodeError int32 = 1605 + + // Quota + SpaceQuotaExceeded int32 = 1700 + NumRequestsExceeded int32 = 1701 + RequestSizeExceeded int32 = 1702 + NumReadRequestsExceeded int32 = 1703 + NumWriteRequestsExceeded int32 = 1704 + WriteSizeExceeded int32 = 1705 + ReadSizeExceeded int32 = 1706 + QuotaMemQueryNotEnough int32 = 1707 + QueryCpuQueryNotEnough int32 = 1708 + QuotaTempDiskQueryNotEnough int32 = 1709 + + // Pipe + PipeVersionError int32 = 1800 + PipeTypeError int32 = 1801 + PipeHandshakeError int32 = 1802 + PipeTransferFileOffsetReset int32 = 1803 + PipeTransferFileError int32 = 1804 + PipeTransferExecuteStatementError int32 = 1805 + PipeNotExistError int32 = 1806 + PipePushMetaError int32 = 1807 + PipeReceiverTemporaryUnavailableException int32 = 1808 + PipeReceiverIdempotentConflictException int32 = 1809 + PipeReceiverUserConflictException int32 = 1810 + PipeConfigReceiverHandshakeNeeded int32 = 1811 + PipeTransferSliceOutOfOrder int32 = 1812 + PipePushMetaTimeout int32 = 1813 + PipePushMetaNotEnoughMemory int32 = 1814 + PipeReceiverParallelOrUserConflictException int32 = 1815 + + // Timecho Pipe + PipeTransferTabletObjectBatchSequenceReset int32 = -1 + + // Subscription + SubscriptionVersionError int32 = 1900 + SubscriptionTypeError int32 = 1901 + SubscriptionHandshakeError int32 = 1902 + SubscriptionHeartbeatError int32 = 1903 + SubscriptionPollError int32 = 1904 + SubscriptionCommitError int32 = 1905 + SubscriptionCloseError int32 = 1906 + SubscriptionSubscribeError int32 = 1907 + SubscriptionUnsubscribeError int32 = 1908 + SubscriptionMissingConsumer int32 = 1909 + ShowSubscriptionError int32 = 1910 + SubscriptionPipeTimeoutError int32 = 1911 + SubscriptionNotEnabledError int32 = 1912 + SubscriptionSeekError int32 = 1913 + SubscriptionOwnerFenced int32 = 1914 + SubscriptionOwnerRequired int32 = 1915 + SubscriptionOwnerEpochRequired int32 = 1916 + SubscriptionOwnerLeaseExpired int32 = 1917 + SubscriptionOwnerEpochConflict int32 = 1918 + SubscriptionConsumerFenced int32 = 1919 + SubscriptionOversizedEntry int32 = 1920 + + // Topic + CreateTopicError int32 = 2000 + DropTopicError int32 = 2001 + AlterTopicError int32 = 2002 + ShowTopicError int32 = 2003 + TopicPushMetaError int32 = 2004 + TopicNotExistError int32 = 2005 + + // Consumer + CreateConsumerError int32 = 2100 + DropConsumerError int32 = 2101 + AlterConsumerError int32 = 2102 + ConsumerPushMetaError int32 = 2103 + + // Consensus Exception + IotConsensusV2ConnectorRestartError int32 = 2200 + IotConsensusV2VersionError int32 = 2201 + IotConsensusV2DeprecatedRequest int32 = 2202 + IotConsensusV2TransferFileOffsetReset int32 = 2203 + IotConsensusV2TransferFileError int32 = 2204 + IotConsensusV2TypeError int32 = 2205 + ConsensusGroupNotExist int32 = 2206 + RatisReadUnavailable int32 = 2207 + IotConsensusV2CloseError int32 = 2208 + IotConsensusV2WaitOrderTimeout int32 = 2209 + + // ExternalService + NoSuchExternalService int32 = 2300 + ExternalServiceAlreadyExist int32 = 2301 + GetBuiltinExternalServiceError int32 = 2302 + ExternalServiceInstanceCreateError int32 = 2303 + CannotDropBuiltinExternalService int32 = 2304 + CannotDropRunningExternalService int32 = 2305 + + // LBAC + LbacAccessDenied int32 = 2400 + ComponentNotExists int32 = 2401 + ComponentAlreadyExists int32 = 2402 + PolicyNotExists int32 = 2403 + PolicyAlreadyExists int32 = 2404 + LabelNotExists int32 = 2405 + LabelAlreadyExists int32 = 2406 + ElementNotExists int32 = 2407 + ElementAlreadyExists int32 = 2408 + InvalidComponentType int32 = 2409 + InvalidElementClause int32 = 2410 + InvalidCommentTarget int32 = 2411 + ComponentIsReferenced int32 = 2412 + PolicyIsReferenced int32 = 2413 + LabelIsReferenced int32 = 2414 + PolicyNotSet int32 = 2415 + PolicyAlreadySet int32 = 2416 + PolicyConflict int32 = 2417 + LabelNotSet int32 = 2418 + InvalidLabelValue int32 = 2419 + + // STREAM + NoRegisteredStreamNodeError int32 = 9400 + RegisterStreamNodeError int32 = 9401 + RemoveStreamNodeError int32 = 9402 + StreamNodeNotExist int32 = 9403 + StreamNodeStopError int32 = 9404 + StreamTaskExistOnStreamnode int32 = 9405 + + // Stream + StreamAlreadyExists int32 = -3000 + StreamNotExist int32 = -3001 + StreamStale int32 = -3002 + StreamStartError int32 = -3003 ) const ( diff --git a/subscription/README.md b/subscription/README.md index 8710ecd..c03c95d 100644 --- a/subscription/README.md +++ b/subscription/README.md @@ -175,8 +175,8 @@ for _, message := range messages { return consumer.Commit(ctx, messages...) ``` -- A positive Poll timeout starts at entry and covers waiting for the serialized foreground gate and the network budget. Processor callbacks must return promptly. Negative timeouts return argument errors. A zero timeout does not wait for a busy foreground gate or perform synchronous provider recovery; it sends one round of requests to available Providers with server timeout 0. Network I/O can still wait and is bounded by the caller's context. Caller context cancellation always returns an error. -- Multiple topics are partitioned according to `MaxPollParallelism`. When only Poll's own budget expires, already completed healthy partitions may still be delivered while incomplete sibling results are discarded. TsFile or current-consumer fencing makes the entire round fail, without passing records to processors, automatic commit tracking, or fetch positions. Custom processor code must still avoid blocking indefinitely. +- A positive Poll timeout starts at entry and covers waiting for the serialized foreground gate, polling, and termination unsubscribe cleanup. Processor callbacks must return promptly. Negative timeouts return argument errors. A zero timeout does not wait for a busy foreground gate or perform synchronous provider recovery; it sends one round of requests to available Providers with server timeout 0. Network I/O can still wait and is bounded by the caller's context. Caller context cancella [...] +- Poll work is serial even when `MaxPollParallelism` is greater than one, matching the current Java client. TsFile or current-consumer fencing makes the entire round fail without passing records to processors, automatic commit tracking, or fetch positions. Custom processor code must still avoid blocking indefinitely. - Heartbeats renew records buffered by Pull processors. Automatic commits track only records delivered to the caller and wait at least one configured interval. Manual `Commit` can run concurrently with Poll; `CommitAsync` reports its result once through a buffered channel. - Commit and seek server requests and local progress updates are mutually exclusive. Partial acceptance is applied first and accepted contexts leave the pending queue; only the remainder is retried. NACKs are not replayed. A writer ACK pending after node loss is redirected only after the same Topic/Region reappears through a healthy node. - Heartbeats are scheduled independently per Provider. Endpoint synchronization and connection failure trigger recovery. Background recovery attempts are at least five seconds apart. Late old-connection responses cannot reactivate an old Provider or incorrectly fence the replacement connection's Consumer. @@ -373,8 +373,11 @@ Alignment reads inputs through the existing public RecordBatch API, copies recor | `ErrDataReleased` | Record access after user data release | | `ErrIncompatibleMessage` | Incompatible payload access represented by `SubscriptionIncompatibleHandlerException` | | `ErrConsumerFenced`, `ErrOwnerFenced` | `SubscriptionConsumerFencedException`, `SubscriptionOwnerFencedException` | +| `*SubscriptionError` with `Kind` and `Code` | Server status categories: connection setup, pipe timeout, non-critical/critical runtime, payload limit, or fencing | -Operation errors can wrap these sentinels; callers identify them with `errors.Is`. Unsupported TsFile errors must not trigger connection retries or file-payload acknowledgments. Pull returns an error and no messages for that round, including records obtained by parallel branches. That round must not advance positions or enter processor or automatic-commit buffers. Previously delivered records and confirmed progress are not rolled back. Callers can unsubscribe the offending topic and poll again. +Operation errors can wrap these sentinels; callers identify them with `errors.Is`. Unsupported TsFile errors must not trigger connection retries or file-payload acknowledgments. Pull returns an error and no messages for a failed round. That round must not advance positions or enter processor or automatic-commit buffers. Previously delivered records and confirmed progress are not rolled back. Callers can unsubscribe the offending topic and poll again. + +`SubscriptionError` is exposed for `errors.As`. Its `Kind`, `Code`, `Critical`, and `Message` fields retain the server classification; wrapped transport causes remain available through `errors.Is` and `errors.As`. Connection setup errors redact configured plaintext and encrypted passwords before returning. `MaxPollParallelism` remains accepted for Java configuration compatibility, while poll work currently runs serially to match Java behavior. The built-in Watermark/ColumnAlign processors and standalone tree/table examples are implemented. The existing 19 unit-test cases now live in `internal/protocol` (protocol and real-message lifecycle cases) and the external `processor_test` package under `processor` (processor behavior). This package migration moves those cases without expanding their scenarios; runtime verification is pending. Management queries reuse the root module's decoder; this work does not change its handling of m [...] diff --git a/subscription/consumer/config.go b/subscription/consumer/config.go index 8958c10..1fd2cd2 100644 --- a/subscription/consumer/config.go +++ b/subscription/consumer/config.go @@ -49,9 +49,9 @@ type SubscriptionConsumerConfig struct { // endpoints. Java consumers default to two minutes and enforce a five-second // minimum. EndpointsSyncInterval time.Duration - // MaxPollParallelism limits parallel polling work. The Java default and - // minimum are both one, allowing polling to use a separate worker when - // resources are available so its timeout can be enforced. + // MaxPollParallelism is retained for Java configuration compatibility. The + // current Java client executes poll work serially even when this is greater + // than one, so Go also polls serially. MaxPollParallelism int } diff --git a/subscription/consumer/provider.go b/subscription/consumer/provider.go index 9386378..e7ec4b9 100644 --- a/subscription/consumer/provider.go +++ b/subscription/consumer/provider.go @@ -28,7 +28,6 @@ import ( "context" "errors" - "fmt" "strconv" "strings" "sync/atomic" @@ -64,6 +63,12 @@ func openSubscriptionProvider(ctx context.Context, config SubscriptionConsumerCo if instanceID == "" { return nil, errors.New("subscription: consumer instance ID must not be empty") } + defer func() { + if err == nil { + return + } + err = classifyAndRedactConnectionError(err, config.Connection.Password, config.Connection.EncryptedPassword) + }() openingCtx, cancel := context.WithTimeout(ctx, config.Connection.ConnectTimeout) defer cancel() connectionConfig := config.Connection @@ -232,7 +237,7 @@ func (p *subscriptionProvider) sendOne(ctx context.Context, request *rpc.TPipeSu } if err := verifySubscriptionProviderStatus(response.Status); err != nil { var statusErr *subscriptionStatusError - if errors.As(err, &statusErr) && statusErr.Code == 1920 { + if errors.As(err, &statusErr) && statusErr.Code == client.SubscriptionOversizedEntry { _ = p.connection.Abort() } return nil, err @@ -363,7 +368,7 @@ func (p *subscriptionProvider) pollPayloads(ctx context.Context, request *rpc.TP response, err := p.call(ctx, request) if err != nil { var statusErr *subscriptionStatusError - if errors.As(err, &statusErr) && statusErr.Code == 1904 { + if errors.As(err, &statusErr) && statusErr.Code == client.SubscriptionPollError { // Broker loss is a server-reported connection failure. Rebuilding the // provider and handshaking again is required before another poll. group := "[" + p.groupID + "]" @@ -419,22 +424,7 @@ func partitionSubscriptionContexts(contexts []payload.CommitContext, limit, fixe return append(batches, contexts[start:]), nil } -// subscriptionStatusError retains server status and its retry classification -// without expanding the public error API before Consumer error handling exists. -type subscriptionStatusError struct { - Code int32 - Message string - Critical bool - cause error -} - -func (e *subscriptionStatusError) Error() string { - return fmt.Sprintf("subscription: server status %d: %s", e.Code, e.Message) -} - -func (e *subscriptionStatusError) Unwrap() error { - return e.cause -} +type subscriptionStatusError = subscription.SubscriptionError func verifySubscriptionProviderStatus(status *common.TSStatus) error { if status == nil { @@ -443,16 +433,58 @@ func verifySubscriptionProviderStatus(status *common.TSStatus) error { if status.Code == client.SuccessStatus { return nil } - failure := &subscriptionStatusError{Code: status.Code, Message: status.GetMessage()} + failure := &subscription.SubscriptionError{Code: status.Code, Message: status.GetMessage()} switch status.Code { - case 1902, 1903, 1904, 1905, 1906, 1907, 1908, 1911, 1913, 1920: - case 1914, 1915, 1916, 1917, 1918: - failure.cause = subscription.ErrOwnerFenced - case 1919: - failure.cause = subscription.ErrConsumerFenced + case client.SubscriptionHandshakeError, + client.SubscriptionHeartbeatError, + client.SubscriptionPollError, + client.SubscriptionCommitError, + client.SubscriptionCloseError, + client.SubscriptionSubscribeError, + client.SubscriptionUnsubscribeError, + client.SubscriptionSeekError: + failure.Kind = subscription.ErrorKindRuntimeNonCritical + case client.SubscriptionPipeTimeoutError: + failure.Kind = subscription.ErrorKindPipeTimeout + case client.SubscriptionOwnerFenced, + client.SubscriptionOwnerRequired, + client.SubscriptionOwnerEpochRequired, + client.SubscriptionOwnerLeaseExpired, + client.SubscriptionOwnerEpochConflict: + failure.Kind = subscription.ErrorKindOwnerFenced + failure.Cause = subscription.ErrOwnerFenced + case client.SubscriptionConsumerFenced: + failure.Kind = subscription.ErrorKindConsumerFenced + failure.Cause = subscription.ErrConsumerFenced failure.Critical = true + case client.SubscriptionOversizedEntry: + failure.Kind = subscription.ErrorKindPayloadExceed default: + failure.Kind = subscription.ErrorKindRuntimeCritical failure.Critical = true } return failure } + +func classifyAndRedactConnectionError(err error, secrets ...string) error { + var status *subscription.SubscriptionError + if errors.As(err, &status) { + copy := *status + copy.Message = redactSubscriptionSecrets(copy.Message, secrets) + return © + } + return &subscription.SubscriptionError{ + Kind: subscription.ErrorKindConnection, + Message: redactSubscriptionSecrets(err.Error(), secrets), + Cause: err, + } +} + +func redactSubscriptionSecrets(message string, secrets []string) string { + for _, secret := range secrets { + if secret != "" { + message = strings.ReplaceAll(message, secret, "[REDACTED]") + } + } + return message +} diff --git a/subscription/consumer/pull_consumer.go b/subscription/consumer/pull_consumer.go index 04b419c..4b16ee1 100644 --- a/subscription/consumer/pull_consumer.go +++ b/subscription/consumer/pull_consumer.go @@ -20,6 +20,7 @@ package consumer import ( + "github.com/apache/iotdb-client-go/v2/client" "github.com/apache/iotdb-client-go/subscription" "github.com/apache/iotdb-client-go/subscription/payload" messageprocessor "github.com/apache/iotdb-client-go/subscription/processor" @@ -454,6 +455,21 @@ func (c *subscriptionPullConsumer) changeSubscriptions(ctx context.Context, topi providers = c.providerSnapshot() } var failures []error + clearRedirectAcks := func() { + if !unsubscribe { + return + } + c.mu.Lock() + for key, commitContext := range c.redirectAcks { + for _, topic := range topics { + if commitContext.Topic == topic { + delete(c.redirectAcks, key) + break + } + } + } + c.mu.Unlock() + } for _, provider := range providers { if provider.connection.Closed() { continue @@ -462,16 +478,8 @@ func (c *subscriptionPullConsumer) changeSubscriptions(ctx context.Context, topi if err == nil { c.mu.Lock() c.topics = current - if unsubscribe { - for key, commitContext := range c.redirectAcks { - for _, topic := range topics { - if commitContext.Topic == topic { - delete(c.redirectAcks, key) - } - } - } - } c.mu.Unlock() + clearRedirectAcks() return nil } err = c.handleProviderError(provider, err) @@ -479,9 +487,10 @@ func (c *subscriptionPullConsumer) changeSubscriptions(ctx context.Context, topi return err } var status *subscriptionStatusError - if errors.As(err, &status) && status.Code == 1911 { + if errors.As(err, &status) && status.Code == client.SubscriptionPipeTimeoutError { // As in Java, a pipe procedure can finish after its timeout. A later // heartbeat reconciles the server's subscription configuration. + clearRedirectAcks() return nil } failures = append(failures, err) diff --git a/subscription/consumer/pull_flow.go b/subscription/consumer/pull_flow.go index 5f076ca..e0a832d 100644 --- a/subscription/consumer/pull_flow.go +++ b/subscription/consumer/pull_flow.go @@ -32,6 +32,7 @@ import ( "fmt" "math/rand/v2" "sort" + "strings" "time" ) @@ -214,8 +215,8 @@ func (c *subscriptionPullConsumer) pollLocked(operationCtx, pollCtx context.Cont } // The caller/lifetime context aborts the entire round. Exhausting only the // polling budget preserves complete healthy partitions, as Java does for - // completed sibling futures. Keep rollback and empty-delivery ACKs tied to - // the poll budget; a received termination is handled using operationCtx below. + // completed sibling futures. Do not start cleanup RPCs after that budget has + // expired; undelivered events remain unconfirmed. c.rollbackFlow(pollCtx, failedContexts) if pollCtx.Err() == nil && len(emptyContexts) != 0 { if err := c.commitContexts(pollCtx, emptyContexts, false); err != nil { @@ -225,11 +226,13 @@ func (c *subscriptionPullConsumer) pollLocked(operationCtx, pollCtx context.Cont } } } - if operationCtx.Err() == nil && len(terminations) != 0 { + if pollCtx.Err() == nil && len(terminations) != 0 { names := uniqueFlowTopics(terminations) - if err := c.changeSubscriptions(operationCtx, names, true); err != nil { - c.rollbackFlow(operationCtx, allContexts) - return nil, err + if err := c.changeSubscriptions(pollCtx, names, true); err != nil { + if !(operationCtx.Err() == nil && pollCtx.Err() != nil && errors.Is(err, context.DeadlineExceeded)) { + c.rollbackFlow(pollCtx, allContexts) + return nil, err + } } } processed := messages @@ -318,10 +321,7 @@ func uniqueFlowTopics(topics []string) []string { } func (c *subscriptionPullConsumer) pollFlowPartitions(ctx context.Context, topics []string, immediate bool) []flowPollResult { - parallelism := min(c.config.Consumer.MaxPollParallelism, len(topics)) - if parallelism < 1 { - parallelism = 1 - } + parallelism := 1 partitions := make([][]string, parallelism) for index, topic := range topics { partitions[index%parallelism] = append(partitions[index%parallelism], topic) @@ -436,7 +436,7 @@ func (c *subscriptionPullConsumer) pollFlowPartition(ctx context.Context, topics if isOutdatedFlowResponse(response) { continue } - failure := &protocol.PollError{Message: response.ErrorMessage, Critical: response.Critical} + failure := newSubscriptionPollError(response.ErrorMessage, response.Critical) if response.Critical { result.err = failure return result @@ -660,12 +660,19 @@ func isOutdatedFlowResponse(response protocol.PollResponse) bool { } func flowCriticalPollError(err error) bool { - var status *subscriptionStatusError - if errors.As(err, &status) { - return status.Critical + var status *subscription.SubscriptionError + return errors.As(err, &status) && status.Critical +} + +func newSubscriptionPollError(message string, critical bool) *subscription.SubscriptionError { + kind := subscription.ErrorKindRuntimeNonCritical + if critical { + kind = subscription.ErrorKindRuntimeCritical + } + if strings.Contains(message, "SUBSCRIPTION_OVERSIZED_ENTRY") { + kind = subscription.ErrorKindPayloadExceed } - var payload *protocol.PollError - return errors.As(err, &payload) && payload.Critical + return &subscription.SubscriptionError{Kind: kind, Message: message, Critical: critical} } func flowFatalPollError(err error) bool { @@ -704,7 +711,7 @@ func (c *subscriptionPullConsumer) completeFlowRecord(ctx context.Context, provi return nil, true, nil } if next.Type == protocol.ResponseError { - return nil, false, &protocol.PollError{Message: next.ErrorMessage, Critical: next.Critical} + return nil, false, newSubscriptionPollError(next.ErrorMessage, next.Critical) } if next.NextOffset > 0 && next.NextOffset <= response.NextOffset { return nil, false, errors.New("subscription: tablet continuation offset did not advance") diff --git a/subscription/consumer/pull_progress.go b/subscription/consumer/pull_progress.go index 21c89c0..1628c52 100644 --- a/subscription/consumer/pull_progress.go +++ b/subscription/consumer/pull_progress.go @@ -249,7 +249,7 @@ func (c *subscriptionPullConsumer) commitNode(ctx context.Context, node int32, c return failure } var status *subscriptionStatusError - if errors.As(failure, &status) && status.Code != 1920 { + if errors.As(failure, &status) && status.Code != client.SubscriptionOversizedEntry { return failure } } @@ -533,14 +533,21 @@ func (c *subscriptionPullConsumer) seekProgress(ctx context.Context, topic strin providers = c.providerSnapshot() } available := 0 + type seekFailure struct { + provider *subscriptionProvider + err error + } + var providerFailures []seekFailure for _, provider := range providers { if provider.connection.Closed() { continue } available++ if err := provider.seek(operationCtx, name, kind, progress); err != nil { - err = c.handleProviderError(provider, err) - return err + if operationCtx.Err() != nil { + return operationCtx.Err() + } + providerFailures = append(providerFailures, seekFailure{provider: provider, err: err}) } } if available == 0 { @@ -549,6 +556,14 @@ func (c *subscriptionPullConsumer) seekProgress(ctx context.Context, topic strin if err := operationCtx.Err(); err != nil { return err } + var failures []error + for _, failure := range providerFailures { + err := c.handleProviderError(failure.provider, failure.err) + failures = append(failures, fmt.Errorf("subscription: seek provider %s: %w", failure.provider.endpoint, err)) + } + if len(failures) != 0 { + return errors.Join(failures...) + } c.mu.Lock() if err := c.stateErrorLocked(false); err != nil { c.mu.Unlock() diff --git a/subscription/errors.go b/subscription/errors.go index 45e9caf..03d37ce 100644 --- a/subscription/errors.go +++ b/subscription/errors.go @@ -19,7 +19,51 @@ package subscription -import "errors" +import ( + "errors" + "fmt" +) + +// ErrorKind identifies the Java subscription exception category represented by +// an operation error. Transport errors remain compatible with errors.Is and +// errors.As through SubscriptionError.Unwrap. +type ErrorKind uint8 + +const ( + ErrorKindUnknown ErrorKind = iota + ErrorKindConnection + ErrorKindPipeTimeout + ErrorKindRuntimeNonCritical + ErrorKindRuntimeCritical + ErrorKindParameterNotValid + ErrorKindPayloadExceed + ErrorKindOwnerFenced + ErrorKindConsumerFenced +) + +// SubscriptionError preserves a server status and its Java-compatible error +// category. Code is zero for transport or locally classified errors. +type SubscriptionError struct { + Kind ErrorKind + Code int32 + Message string + Critical bool + Cause error +} + +func (e *SubscriptionError) Error() string { + if e.Code != 0 { + return fmt.Sprintf("subscription: server status %d: %s", e.Code, e.Message) + } + if e.Message != "" { + return "subscription: " + e.Message + } + return "subscription: operation failed" +} + +func (e *SubscriptionError) Unwrap() error { + return e.Cause +} // Sentinel errors can be matched through wrapped operation errors with errors.Is. var (
