This is an automated email from the ASF dual-hosted git repository. shuwenwei pushed a commit to branch supports_subscription in repository https://gitbox.apache.org/repos/asf/iotdb-client-go.git
commit 3ca3fef70c0de56b37ffd5c7802110136694157e Author: shuwenwei <[email protected]> AuthorDate: Tue Nov 17 10:52:35 2026 +0800 Stabilize subscription completion and E2E tests --- Makefile | 9 +- subscription/README.md | 6 +- subscription/consumer/consumer.go | 12 +- subscription/consumer/provider.go | 7 +- subscription/consumer/pull_consumer.go | 73 ++++- subscription/consumer/pull_flow.go | 19 +- subscription/consumer/pull_progress.go | 8 +- subscription/consumer/pull_providers.go | 4 +- subscription/go.mod | 2 + test/e2e/README.md | 25 +- test/e2e/config_test.go | 165 +++++++++++ test/e2e/docker-compose.yml | 5 + test/e2e/e2e_table_test.go | 54 +--- test/e2e/e2e_test.go | 14 +- test/e2e/start-1c1d.sh | 2 + test/e2e/subscription_table_test.go | 504 ++++++++++++++++++++++++++++++-- test/e2e/subscription_tree_test.go | 103 +++---- 17 files changed, 831 insertions(+), 181 deletions(-) diff --git a/Makefile b/Makefile index 8336331..7b79ac9 100644 --- a/Makefile +++ b/Makefile @@ -72,11 +72,16 @@ generate: fi @rm -rf rpc/i_client_r_p_c_service-remote -.PHONY: generate all test e2e_test e2e_test_clean +.PHONY: generate all test test_client test_subscription e2e_test e2e_test_clean -test: +test: test_client test_subscription + +test_client: go test -v ./client/... +test_subscription: + cd subscription && go test -v ./... + e2e_test: sh -c "cd /tmp/ && rm -rf iotdb && git clone https://github.com/apache/iotdb.git && cd iotdb && mvn clean package -pl distribution -am -DskipTests" mkdir -p target/iotdb diff --git a/subscription/README.md b/subscription/README.md index c17f922..fa49061 100644 --- a/subscription/README.md +++ b/subscription/README.md @@ -177,7 +177,7 @@ return consumer.Commit(ctx, messages...) - A positive Poll timeout starts at entry and covers waiting for the serialized foreground gate, polling, and termination unsubscribe cleanup. Processor callbacks must return promptly. Negative timeouts return argument errors. A zero timeout does not wait for a busy foreground gate or perform synchronous provider recovery; it sends one round of requests to available Providers with server timeout 0. Network I/O can still wait and is bounded by the caller's context. Caller context cancella [...] - Poll work is serial even when `MaxPollParallelism` is greater than one, matching the current Java client. TsFile or current-consumer fencing makes the entire round fail without passing records to processors, automatic commit tracking, or fetch positions. Custom processor code must still avoid blocking indefinitely. -- Heartbeats renew records buffered by Pull processors. Automatic commits track only records delivered to the caller and use an expiry heap so each tick processes due entries instead of scanning the full backlog. Manual `Commit` can run concurrently with Poll; `CommitAsync` reports its result once through a buffered channel. +- Heartbeats renew records buffered by Pull processors. Automatic commits track only records delivered to the caller and use an expiry heap so each tick processes due entries instead of scanning the full backlog. `LastAutoCommitError` exposes the last background ACK failure; it is cleared by a later successful due batch. Manual `Commit` can run concurrently with Poll; `CommitAsync` reports its result once through a buffered channel. - Commit and seek server requests and local progress updates are mutually exclusive. Partial acceptance is applied first and accepted contexts leave the pending queue; only the remainder is retried. NACKs are not replayed. A writer ACK pending after node loss is redirected only after the same Topic/Region reappears through a healthy node. - Heartbeats are scheduled independently per Provider. Endpoint synchronization and connection failure trigger recovery. Background recovery attempts are at least five seconds apart. Late old-connection responses cannot reactivate an old Provider or incorrectly fence the replacement connection's Consumer. - `Close` interrupts blocked polling and stops background work. In manual mode, buffered processor messages make closing return an error while preserving the drain path. Call `DrainBufferedMessages`, process and commit its records, then close again. Automatic mode makes a best-effort commit of delivered and drained records during closing. Commit failures return errors; a successful close does not imply that all historical messages were processed. @@ -270,7 +270,7 @@ Consumer interfaces, configurations, defaults, constructors, acknowledgment type | `Subscribe(ctx, topics...)`, `Unsubscribe(ctx, topics...)` | Single-topic, array, and collection overloads | Variadic topic names | | `Info()`, `SubscriptionConsumerInfo` | `getConsumerId/getConsumerGroupId/getOwnerId/getOwnerEpoch` | Returns a credential-free identity snapshot, including server-assigned IDs | | `LatestWatermark()` | `getLatestWatermarkTimestamp()` | An additional `ok` distinguishes an unavailable watermark from a valid timestamp | -| `AllConsumed()` | `allTopicMessagesHaveBeenConsumed()` | True without subscriptions; an active unbounded topic prevents completion | +| `AllConsumed()` | `allTopicMessagesHaveBeenConsumed()` | True without subscriptions or after every subscribed topic emits termination; unsubscribe cleanup may still be pending | | `SubscriptionPullConsumer.Poll(ctx, timeout, topics...)` | Four `poll` overloads | `time.Duration` replaces millisecond and Java Duration overloads; omitted topics mean all current subscriptions | | `PollWithInfo(ctx, timeout, topics...)`, `PollResult` | Concrete Pull classes' `pollWithInfo`, `payload.PollResult` | Returns messages, buffered count, and processor watermark together | | `DrainBufferedMessages(ctx)` | `drainBufferedMessages()` | Explicitly drains processors or pending delivery buffers | @@ -375,7 +375,7 @@ Decoded subscription records use typed column storage and are aligned in place u | `ErrConsumerFenced`, `ErrOwnerFenced` | `SubscriptionConsumerFencedException`, `SubscriptionOwnerFencedException` | | `*SubscriptionError` with `Kind` and `Code` | Server status categories: connection setup, pipe timeout, non-critical/critical runtime, payload limit, or fencing | -Operation errors can wrap these sentinels; callers identify them with `errors.Is`. Unsupported TsFile errors must not trigger connection retries or file-payload acknowledgments. Pull returns an error and no messages for a failed round. That round must not advance positions or enter processor or automatic-commit buffers. Previously delivered records and confirmed progress are not rolled back. Callers can unsubscribe the offending topic and poll again. +Operation errors can wrap these sentinels; callers identify them with `errors.Is`. Unsupported TsFile errors must not trigger connection retries or file-payload acknowledgments. Pull returns an error and no messages for a failed round. That round must not advance positions or enter processor or automatic-commit buffers. Previously delivered records and confirmed progress are not rolled back. Callers can unsubscribe the offending topic and poll again. A termination received from the serve [...] `SubscriptionError` is exposed for `errors.As`. Its `Kind`, `Code`, `Critical`, and `Message` fields retain the server classification; wrapped transport causes remain available through `errors.Is` and `errors.As`. Connection setup errors redact configured plaintext and encrypted passwords before returning. `MaxPollParallelism` remains accepted for Java configuration compatibility, while poll work currently runs serially to match Java behavior. diff --git a/subscription/consumer/consumer.go b/subscription/consumer/consumer.go index fc6a35b..502f2e9 100644 --- a/subscription/consumer/consumer.go +++ b/subscription/consumer/consumer.go @@ -59,10 +59,11 @@ type SubscriptionConsumer interface { // LatestWatermark reports the latest received server watermark. The boolean // is false until a watermark is observed. This is not a processor watermark. LatestWatermark() (timestamp int64, ok bool) - // AllConsumed reports whether all subscribed topics have completed and been - // automatically unsubscribed. An active unbounded topic prevents completion; - // no subscriptions means true. This does not assert that delivered messages - // were committed or that client-side processor buffers have been drained. + // AllConsumed reports whether every currently subscribed topic has emitted a + // server completion termination. A bounded unsubscribe may still be pending + // cleanup after this becomes true. An active unbounded topic prevents + // completion; no subscriptions means true. This does not assert that delivered + // messages were committed or that client-side processor buffers were drained. AllConsumed() bool } @@ -122,6 +123,9 @@ type SubscriptionPullConsumer interface { // never blocks cleanup. Context cancellation does not undo accepted commits. CommitAsync(ctx context.Context, messages ...payload.SubscriptionMessage) <-chan error + // LastAutoCommitError returns the most recent background auto-commit error. + // It is cleared after a later due auto-commit batch succeeds completely. + LastAutoCommitError() error // AddProcessor appends a processor to the pipeline. Each poll passes messages // through processors in their addition order before making records visible. AddProcessor(processor messageprocessor.SubscriptionMessageProcessor) error diff --git a/subscription/consumer/provider.go b/subscription/consumer/provider.go index e7ec4b9..f6a3384 100644 --- a/subscription/consumer/provider.go +++ b/subscription/consumer/provider.go @@ -392,11 +392,14 @@ func (p *subscriptionProvider) close(ctx context.Context, preserveSubscriptions if p.connection.Closed() { return nil } + if err := ctx.Err(); err != nil { + return errors.Join(err, p.connection.Abort()) + } select { case p.operations <- struct{}{}: defer func() { <-p.operations }() - default: - return p.connection.Abort() + case <-ctx.Done(): + return errors.Join(ctx.Err(), p.connection.Abort()) } _, requestErr := p.sendLocked(ctx, protocol.EncodeClose()) return errors.Join(requestErr, p.connection.Close(ctx)) diff --git a/subscription/consumer/pull_consumer.go b/subscription/consumer/pull_consumer.go index 0a59eaf..226a150 100644 --- a/subscription/consumer/pull_consumer.go +++ b/subscription/consumer/pull_consumer.go @@ -86,6 +86,8 @@ func newSubscriptionPullConsumer(resolved SubscriptionPullConsumerConfig) (*subs uncommitted: make(map[subscriptionCommitKey]subscriptionPendingCommit), uncommittedDeadlines: make(subscriptionCommitDeadlineHeap, 0), redirectAcks: make(map[subscriptionCommitKey]payload.CommitContext), + pendingTerminations: make(map[string]struct{}), + pendingUnsubscribes: make(map[string]struct{}), identity: SubscriptionConsumerInfo{ ConsumerID: resolved.Consumer.ConsumerID, GroupID: resolved.Consumer.GroupID, @@ -128,6 +130,9 @@ type subscriptionPullConsumer struct { uncommitted map[subscriptionCommitKey]subscriptionPendingCommit uncommittedDeadlines subscriptionCommitDeadlineHeap redirectAcks map[subscriptionCommitKey]payload.CommitContext + pendingTerminations map[string]struct{} + pendingUnsubscribes map[string]struct{} + lastAutoCommitError error watermark int64 hasWatermark bool pendingCritical error @@ -404,7 +409,61 @@ func (c *subscriptionPullConsumer) LatestWatermark() (int64, bool) { func (c *subscriptionPullConsumer) AllConsumed() bool { c.mu.RLock() defer c.mu.RUnlock() - return len(c.topics) == 0 + for topic := range c.topics { + if _, completed := c.pendingTerminations[topic]; !completed { + return false + } + } + return true +} + +func (c *subscriptionPullConsumer) LastAutoCommitError() error { + c.mu.RLock() + defer c.mu.RUnlock() + return c.lastAutoCommitError +} + +func (c *subscriptionPullConsumer) stagePendingTerminations(topics []string) { + c.mu.Lock() + defer c.mu.Unlock() + for _, topic := range topics { + if topic != "" { + c.pendingTerminations[topic] = struct{}{} + c.pendingUnsubscribes[topic] = struct{}{} + } + } +} + +func (c *subscriptionPullConsumer) pendingTerminationTopics() []string { + c.mu.RLock() + topics := make([]string, 0, len(c.pendingTerminations)) + for topic := range c.pendingTerminations { + topics = append(topics, topic) + } + c.mu.RUnlock() + sort.Strings(topics) + return topics +} + +func (c *subscriptionPullConsumer) reconcileTopicStateLocked() { + for topic := range c.pendingTerminations { + if _, exists := c.topics[topic]; !exists { + delete(c.pendingTerminations, topic) + } + } + for topic := range c.pendingUnsubscribes { + if _, exists := c.topics[topic]; !exists { + delete(c.pendingUnsubscribes, topic) + } + } +} + +func (c *subscriptionPullConsumer) flushPendingTerminations(ctx context.Context) error { + topics := c.pendingTerminationTopics() + if len(topics) == 0 { + return nil + } + return c.changeSubscriptions(ctx, topics, true) } func (c *subscriptionPullConsumer) Subscribe(ctx context.Context, topics ...string) error { @@ -443,6 +502,11 @@ func (c *subscriptionPullConsumer) changeSubscriptions(ctx context.Context, topi } c.mu.Lock() c.topicRevision++ + if unsubscribe { + for _, topic := range topics { + c.pendingUnsubscribes[topic] = struct{}{} + } + } c.mu.Unlock() defer func() { c.mu.Lock() @@ -480,6 +544,13 @@ func (c *subscriptionPullConsumer) changeSubscriptions(ctx context.Context, topi if err == nil { c.mu.Lock() c.topics = current + if !unsubscribe { + for _, topic := range topics { + delete(c.pendingTerminations, topic) + delete(c.pendingUnsubscribes, topic) + } + } + c.reconcileTopicStateLocked() c.mu.Unlock() clearRedirectAcks() return nil diff --git a/subscription/consumer/pull_flow.go b/subscription/consumer/pull_flow.go index 07e25ae..bb8a760 100644 --- a/subscription/consumer/pull_flow.go +++ b/subscription/consumer/pull_flow.go @@ -138,6 +138,11 @@ func (c *subscriptionPullConsumer) beginFlowPoll(ctx context.Context, timeout ti } func (c *subscriptionPullConsumer) pollLocked(operationCtx, pollCtx context.Context, immediate bool, topics []string) ([]payload.SubscriptionMessage, error) { + if !immediate { + if err := c.flushPendingTerminations(pollCtx); err != nil { + return nil, err + } + } selected, err := c.flowTopics(topics) if err != nil { return nil, err @@ -239,12 +244,14 @@ func (c *subscriptionPullConsumer) pollLocked(operationCtx, pollCtx context.Cont } } } - if pollCtx.Err() == nil && len(terminations) != 0 { - names := uniqueFlowTopics(terminations) - if err := c.changeSubscriptions(pollCtx, names, true); err != nil { - if !(operationCtx.Err() == nil && pollCtx.Err() != nil && errors.Is(err, context.DeadlineExceeded)) { - c.rollbackFlow(pollCtx, allContexts) - return nil, err + if len(terminations) != 0 { + c.stagePendingTerminations(terminations) + if pollCtx.Err() == nil { + if err := c.flushPendingTerminations(pollCtx); err != nil { + if !(operationCtx.Err() == nil && pollCtx.Err() != nil && errors.Is(err, context.DeadlineExceeded)) { + c.rollbackFlow(pollCtx, allContexts) + return nil, err + } } } } diff --git a/subscription/consumer/pull_progress.go b/subscription/consumer/pull_progress.go index cd54a51..c33f307 100644 --- a/subscription/consumer/pull_progress.go +++ b/subscription/consumer/pull_progress.go @@ -20,6 +20,7 @@ package consumer import ( + "github.com/apache/iotdb-client-go/v2/client" "github.com/apache/iotdb-client-go/subscription" "github.com/apache/iotdb-client-go/subscription/payload" "github.com/apache/iotdb-client-go/subscription/internal/protocol" @@ -505,7 +506,10 @@ func (c *subscriptionPullConsumer) autoCommit(ctx context.Context, now time.Time defer func() { <-c.progressOperations }() contexts := c.pendingCommitContexts(false, now) if len(contexts) != 0 { - _ = c.commitUniqueContextsLocked(ctx, contexts, false) + err := c.commitUniqueContextsLocked(ctx, contexts, false) + c.mu.Lock() + c.lastAutoCommitError = err + c.mu.Unlock() c.requeuePendingCommitContexts(contexts, now) } } @@ -675,6 +679,8 @@ func (c *subscriptionPullConsumer) seekProgress(ctx context.Context, topic strin delete(c.redirectAcks, key) } } + delete(c.pendingTerminations, name) + delete(c.pendingUnsubscribes, name) c.mu.Unlock() c.resetProcessors(name) return nil diff --git a/subscription/consumer/pull_providers.go b/subscription/consumer/pull_providers.go index e3d715c..9eb017b 100644 --- a/subscription/consumer/pull_providers.go +++ b/subscription/consumer/pull_providers.go @@ -297,6 +297,7 @@ func (c *subscriptionPullConsumer) providerHeartbeat(ctx context.Context, provid // snapshot, even if no second revision increment has happened yet. if c.topicRevision == revision && revision%2 == 0 { c.topics = response.Topics + c.reconcileTopicStateLocked() } else { response.Unsubscribe = nil } @@ -347,6 +348,7 @@ func (c *subscriptionPullConsumer) runProviderHeartbeat(provider *subscriptionPr return } if len(response.Unsubscribe) != 0 { + c.stagePendingTerminations(response.Unsubscribe) ctx, cancel := context.WithTimeout(c.lifeCtx, c.config.Consumer.Connection.ConnectTimeout) defer cancel() operationCtx, finish, err := c.beginOperation(ctx) @@ -358,7 +360,7 @@ func (c *subscriptionPullConsumer) runProviderHeartbeat(provider *subscriptionPr current := c.providers[provider.dataNodeID] == provider && c.topicRevision == revision c.mu.RUnlock() if current { - _ = c.changeSubscriptions(operationCtx, response.Unsubscribe, true) + _ = c.flushPendingTerminations(operationCtx) } } } diff --git a/subscription/go.mod b/subscription/go.mod index fb9f187..ee94356 100644 --- a/subscription/go.mod +++ b/subscription/go.mod @@ -6,3 +6,5 @@ require ( github.com/apache/iotdb-client-go/v2 v2.0.0-20260914074454-a25f68aa9681 github.com/apache/thrift v0.24.0 ) + +replace github.com/apache/iotdb-client-go/v2 => .. diff --git a/test/e2e/README.md b/test/e2e/README.md index 0b3c6a1..9608898 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -35,11 +35,15 @@ cd test/e2e go test -v ./... ``` -The existing Session suites use the Docker hostname `iotdb` and their existing fixed connection settings. For a locally managed server, select only the subscription cases: +The existing Session suites use the Docker hostname `iotdb` and their existing fixed connection settings. A normal `go test -v ./...` discovers subscription cases as well; without the opt-in environment variable, every subscription case reports `SKIP`. For a locally managed server, enable only the subscription cases with explicit connection credentials: ```sh cd test/e2e -IOTDB_SUBSCRIPTION_E2E=1 IOTDB_SUBSCRIPTION_NODE_URLS=127.0.0.1:6667 go test -v -run '^TestSubscription' ./... +IOTDB_SUBSCRIPTION_E2E=1 \ +IOTDB_E2E_NODE_URLS=iotdb:6668,iotdb:6667,iotdb:6669 \ +IOTDB_E2E_USERNAME=root \ +IOTDB_E2E_PASSWORD=root \ +go test -v -run '^TestSubscription' ./... ``` The commands above are instructions for the user; they were not executed during this change. @@ -52,17 +56,18 @@ The subscription module's existing 19 unit-test cases were moved with the implem ## Subscription server requirements -Use a server with subscription support enabled. Incremental cases additionally require WAL subscriptions and a compatible non-Ratis DataRegion consensus protocol, such as the supported IoTConsensus configuration of the target server. Merely exposing the normal Session RPC endpoint does not establish that these subscription features are available. +Use a server with subscription support enabled. Incremental cases additionally require WAL subscriptions and a compatible non-Ratis DataRegion consensus protocol, such as the supported IoTConsensus configuration of the target server. The E2E Docker startup enables `subscription_enabled` and gives the Subscription component a nonzero ninth share in `chunk_timeseriesmeta_free_memory_proportion`; both settings take effect after DataNode restart. A locally managed server needs equivalent set [...] -Subscription cases skip unless `IOTDB_SUBSCRIPTION_E2E=1` is set. Once selected, connection errors, disabled subscription support, unsupported topic modes, and unavailable WAL features fail the tests; an empty poll does not count as successful delivery. +Session and subscription cases share one [config_test.go](config_test.go) containing both connection settings and subscription runtime settings. Subscription cases are disabled by default: exactly `IOTDB_SUBSCRIPTION_E2E=1` enables them, while an unset, empty, or different value skips them. Once selected, connection errors, disabled subscription support, unsupported topic modes, and unavailable WAL features fail the tests; an empty poll does not count as successful delivery. | Environment variable | Behavior | | --- | --- | -| `IOTDB_SUBSCRIPTION_E2E` | Set exactly to `1` to enable subscription tests. | -| `IOTDB_SUBSCRIPTION_NODE_URLS` | Comma-separated `host:port` seeds; defaults to the existing E2E `nodeUrls` value (`iotdb:6668,iotdb:6667,iotdb:6669`). Whitespace around seeds is removed. | -| `IOTDB_SUBSCRIPTION_USERNAME` | Subscription and data-writer username; defaults to `root` when unset or empty. | -| `IOTDB_SUBSCRIPTION_PASSWORD` | Password when present, including an explicitly empty value. Otherwise falls back to `IOTDB_PASSWORD`, then `root`. | -| `IOTDB_PASSWORD` | Password fallback when `IOTDB_SUBSCRIPTION_PASSWORD` is absent; an explicitly empty value is preserved. | +| `IOTDB_SUBSCRIPTION_E2E` | Required opt-in: exactly `1` enables subscription tests; unset, empty, or any other value skips them. | +| `IOTDB_E2E_NODE_URLS` | Comma-separated `host:port` seeds shared by Session and subscription E2E tests; defaults to `iotdb:6668,iotdb:6667,iotdb:6669`. Whitespace around seeds is removed. | +| `IOTDB_E2E_USERNAME` | Shared Session and subscription username; defaults to `root`. `IOTDB_USERNAME` is used as a fallback. | +| `IOTDB_E2E_PASSWORD` | Shared Session and subscription password; defaults to `root` and preserves an explicitly empty value. `IOTDB_PASSWORD` is used as a fallback. | +| `IOTDB_USERNAME` | Username fallback when `IOTDB_E2E_USERNAME` is absent. | +| `IOTDB_PASSWORD` | Password fallback when `IOTDB_E2E_PASSWORD` is absent. | Each test uses unique resource names and closes consumers before dropping topics and data. Subscription operations have bounded contexts, with independent contexts for topic cleanup. The existing Session and `ITableSession` data preparation/cleanup APIs do not accept per-statement contexts, so those calls do not inherit the subscription test deadline. @@ -78,6 +83,6 @@ Comments next to each case identify the relevant Java IT file and method under ` The existing `make e2e_test` and `make e2e_test_for_parent_git_repo` workflows still prepare the IoTDB distribution and start `test/e2e/docker-compose.yml`. The client container runs tests from this independent module using Go 1.25. -The current Compose file does not forward subscription environment variables or configure the server for subscription/WAL support. Its default execution therefore runs the legacy Session suites and skips subscription cases. To run subscription cases in containers, supply your own compatible server configuration and a Compose override that explicitly forwards `IOTDB_SUBSCRIPTION_E2E=1`, the seed list, and any credentials to the client service. Setting these variables only in the host shel [...] +The Compose client passes through `IOTDB_E2E_*` and `IOTDB_SUBSCRIPTION_E2E` when they are set in the host environment. Unset variables stay unset, so the in-container tests use the defaults from `config_test.go`; subscription remains disabled by default. The server image still needs subscription/WAL support for those cases. To enable subscription cases in containers, provide a compatible server configuration and set `IOTDB_SUBSCRIPTION_E2E=1` when invoking Compose. No compilation, tests, examples, formatting tools, or compiler/syntax checks were run during this package migration. Runtime results remain unverified. diff --git a/test/e2e/config_test.go b/test/e2e/config_test.go new file mode 100644 index 0000000..37d5be8 --- /dev/null +++ b/test/e2e/config_test.go @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package e2e + +import ( + "net" + "os" + "strings" + "testing" + "time" + + "github.com/apache/iotdb-client-go/subscription" + subsession "github.com/apache/iotdb-client-go/subscription/session" + "github.com/apache/iotdb-client-go/v2/client" +) + +const ( + e2eDefaultNodeURLs = "iotdb:6668,iotdb:6667,iotdb:6669" + e2eDefaultUsername = "root" + e2eDefaultPassword = "root" +) + +var ( + e2eNodeURLs = e2eDefaultValue("IOTDB_E2E_NODE_URLS", "", e2eDefaultNodeURLs) + e2eUsername = e2eDefaultValue("IOTDB_E2E_USERNAME", "IOTDB_USERNAME", e2eDefaultUsername) + e2ePassword = e2eValueOrFallback("IOTDB_E2E_PASSWORD", "IOTDB_PASSWORD", e2eDefaultPassword) +) + +// Subscription cases are discovered by the normal go test run but require an +// explicit environment opt-in. Endpoint and credential defaults come from the +// shared E2E configuration. +const ( + subscriptionE2EDefaultEnabled = false + + subscriptionE2ETestTimeout = 20 * time.Second + subscriptionE2EOpenTimeout = 10 * time.Second + subscriptionE2ECleanupTimeout = 10 * time.Second + subscriptionE2EWriterTimeoutMs = 10000 + + subscriptionE2EHeartbeatInterval = time.Second + subscriptionE2EEndpointsSyncInterval = 5 * time.Second + subscriptionE2EAutoCommitInterval = 500 * time.Millisecond + subscriptionE2EPollTimeout = time.Second + subscriptionE2EPushPollInterval = 100 * time.Millisecond + subscriptionE2EMaxPollRounds = 40 + subscriptionE2EQuietPollRounds = 3 + + subscriptionTreeE2EProgressPollInterval = 20 * time.Millisecond + subscriptionTableE2EProgressPollInterval = 100 * time.Millisecond +) + +// Subscription tests require IOTDB_SUBSCRIPTION_E2E=1 and a subscription-capable +// server. Incremental cases require WAL subscriptions with a compatible +// DataRegion consensus protocol; Ratis is not supported. Opting in makes +// unavailable server features fail rather than silently skip these cases. +// Java topic/path/time/snapshot ITs provide design references, but some are +// @Ignore in that checkout; their presence is not evidence these Go tests ran. +func subscriptionE2EConnection(t *testing.T) subscription.ConnectionConfig { + t.Helper() + enabled := subscriptionE2EDefaultEnabled + if value, supplied := os.LookupEnv("IOTDB_SUBSCRIPTION_E2E"); supplied { + enabled = value == "1" + } + if !enabled { + t.Skip("set IOTDB_SUBSCRIPTION_E2E=1 to run subscription server tests") + } + shared := e2eClusterConfig() + if len(shared.NodeUrls) == 0 { + t.Fatal("subscription endpoint list is empty") + } + config := subsession.DefaultSubscriptionSessionConfig().Connection + config.NodeURLs = shared.NodeUrls + config.Username = shared.UserName + config.Password = shared.Password + return config +} + +func e2eDefaultValue(primary, fallbackEnv, defaultValue string) string { + value := e2eValueOrFallback(primary, fallbackEnv, defaultValue) + if value == "" { + return defaultValue + } + return value +} + +func e2eValueOrFallback(primary, fallbackEnv, defaultValue string) string { + if value, supplied := os.LookupEnv(primary); supplied { + return value + } + if fallbackEnv != "" { + if value, supplied := os.LookupEnv(fallbackEnv); supplied { + return value + } + } + return defaultValue +} + +func e2eAddresses() []string { + var addresses []string + for _, address := range strings.Split(e2eNodeURLs, ",") { + if address = strings.TrimSpace(address); address != "" { + addresses = append(addresses, address) + } + } + return addresses +} + +func e2eClusterConfig() client.ClusterConfig { + return client.ClusterConfig{ + NodeUrls: e2eAddresses(), + UserName: e2eUsername, + Password: e2ePassword, + } +} + +func e2eTableEndpoint() (host, port string) { + addresses := e2eAddresses() + if len(addresses) == 0 { + return "", "" + } + host, port, _ = net.SplitHostPort(addresses[0]) + if host == "" { + host = addresses[0] + } + return host, port +} + +func e2eTableConfig(database string) *client.Config { + host, port := e2eTableEndpoint() + return &client.Config{ + Host: host, + Port: port, + UserName: e2eUsername, + Password: e2ePassword, + Database: database, + } +} + +func e2eTablePoolConfig(database string) *client.PoolConfig { + host, port := e2eTableEndpoint() + return &client.PoolConfig{ + Host: host, + Port: port, + UserName: e2eUsername, + Password: e2ePassword, + Database: database, + } +} diff --git a/test/e2e/docker-compose.yml b/test/e2e/docker-compose.yml index 7a27713..bd851fb 100644 --- a/test/e2e/docker-compose.yml +++ b/test/e2e/docker-compose.yml @@ -36,6 +36,11 @@ services: build: context: ../.. dockerfile: test/e2e/Dockerfile.iotdb-client-go + environment: + IOTDB_E2E_NODE_URLS: + IOTDB_E2E_USERNAME: + IOTDB_E2E_PASSWORD: + IOTDB_SUBSCRIPTION_E2E: depends_on: iotdb: condition: service_healthy \ No newline at end of file diff --git a/test/e2e/e2e_table_test.go b/test/e2e/e2e_table_test.go index 6355e48..7708a9c 100644 --- a/test/e2e/e2e_table_test.go +++ b/test/e2e/e2e_table_test.go @@ -23,7 +23,6 @@ import ( "bytes" "log" "strconv" - "strings" "sync" "sync/atomic" "testing" @@ -34,14 +33,7 @@ import ( "github.com/apache/iotdb-client-go/v2/client" ) -var ( - nodeUrls = "iotdb:6668,iotdb:6667,iotdb:6669" - host = "iotdb" - port = "6667" - username = "root" - password = "root" - database = "test_db" -) +var database = "test_db" type e2eTableTestSuite struct { suite.Suite @@ -53,11 +45,7 @@ func TestE2ETableTestSuite(t *testing.T) { } func (s *e2eTableTestSuite) SetupSuite() { - clusterConfig := client.ClusterConfig{ - NodeUrls: strings.Split(nodeUrls, ","), - UserName: username, - Password: password, - } + clusterConfig := e2eClusterConfig() session, err := client.NewClusterTableSession(&clusterConfig, false) s.Require().NoError(err) s.session = session @@ -84,12 +72,7 @@ func (s *e2eTableTestSuite) TearDownTest() { func (s *e2eTableTestSuite) Test_CreateTableSession() { assert := s.Require() - config := &client.Config{ - Host: host, - Port: port, - UserName: username, - Password: password, - } + config := e2eTableConfig("") session, err := client.NewTableSession(config, false, 3000) assert.NoError(err) defer session.Close() @@ -98,13 +81,7 @@ func (s *e2eTableTestSuite) Test_CreateTableSession() { func (s *e2eTableTestSuite) Test_CreateTableSessionWithDatabase() { assert := s.Require() - config := &client.Config{ - Host: host, - Port: port, - UserName: username, - Password: password, - Database: database, - } + config := e2eTableConfig(database) session, err := client.NewTableSession(config, false, 3000) defer session.Close() assert.NoError(err) @@ -115,12 +92,7 @@ func (s *e2eTableTestSuite) Test_CreateTableSessionWithDatabase() { func (s *e2eTableTestSuite) Test_GetSessionFromTableSessionPool() { assert := s.Require() - poolConfig := &client.PoolConfig{ - Host: host, - Port: port, - UserName: username, - Password: password, - } + poolConfig := e2eTablePoolConfig("") sessionPool := client.NewTableSessionPool(poolConfig, 3, 10000, 3000, false) defer sessionPool.Close() @@ -174,13 +146,7 @@ func (s *e2eTableTestSuite) Test_GetSessionFromTableSessionPool() { func (s *e2eTableTestSuite) Test_GetSessionFromSessionPoolWithSpecificDatabase() { assert := s.Require() - poolConfig := &client.PoolConfig{ - Host: host, - Port: port, - UserName: username, - Password: password, - Database: database, - } + poolConfig := e2eTablePoolConfig(database) sessionPool := client.NewTableSessionPool(poolConfig, 3, 10000, 3000, false) defer sessionPool.Close() @@ -248,13 +214,7 @@ func (s *e2eTableTestSuite) Test_GetSessionFromSessionPoolWithSpecificDatabase() func (s *e2eTableTestSuite) Test_ErrInTableSessionPool() { assert := s.Require() - poolConfig := &client.PoolConfig{ - Host: host, - Port: port, - UserName: username, - Password: password, - Database: database, - } + poolConfig := e2eTablePoolConfig(database) sessionPool := client.NewTableSessionPool(poolConfig, 3, 10000, 3000, false) defer sessionPool.Close() diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index aef1dbd..c2d2d2a 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -24,7 +24,6 @@ import ( "fmt" "log" "math/rand" - "strings" "testing" "time" @@ -43,11 +42,7 @@ func TestE2ETestSuite(t *testing.T) { } func (s *e2eTestSuite) SetupSuite() { - clusterConfig := client.ClusterConfig{ - NodeUrls: strings.Split("iotdb:6668,iotdb:6667,iotdb:6669", ","), - UserName: "root", - Password: "root", - } + clusterConfig := e2eClusterConfig() session, err := client.NewClusterSession(&clusterConfig) s.Require().NoError(err) s.session = session @@ -74,11 +69,8 @@ func (s *e2eTestSuite) checkError(err error) { } func (s *e2eTestSuite) Test_WrongURL() { - clusterConfig := client.ClusterConfig{ - NodeUrls: strings.Split("iotdb1:6667", ","), - UserName: "root", - Password: "root", - } + clusterConfig := e2eClusterConfig() + clusterConfig.NodeUrls = []string{"iotdb1:6667"} _, err := client.NewClusterSession(&clusterConfig) s.Require().Error(err) } diff --git a/test/e2e/start-1c1d.sh b/test/e2e/start-1c1d.sh index b74e6c5..88c2c3c 100755 --- a/test/e2e/start-1c1d.sh +++ b/test/e2e/start-1c1d.sh @@ -18,6 +18,8 @@ # under the License. # sed -i 's/dn_rpc_address=127.0.0.1/dn_rpc_address=0.0.0.0/g' /usr/local/iotdb/conf/iotdb-system.properties +sed -i 's/^subscription_enabled=.*/subscription_enabled=true/' /usr/local/iotdb/conf/iotdb-system.properties +sed -i 's/^chunk_timeseriesmeta_free_memory_proportion=.*/chunk_timeseriesmeta_free_memory_proportion=1:100:200:50:200:200:200:50:250/' /usr/local/iotdb/conf/iotdb-system.properties nohup /usr/local/iotdb/sbin/start-confignode.sh > /dev/null 2>&1 & sleep 5 ls /usr/local/iotdb/sbin diff --git a/test/e2e/subscription_table_test.go b/test/e2e/subscription_table_test.go index a79b296..b86c90e 100644 --- a/test/e2e/subscription_table_test.go +++ b/test/e2e/subscription_table_test.go @@ -107,6 +107,56 @@ func subscriptionTableInsert(t *testing.T, writer client.ITableSession, database database, table, timestamp, timestamp, timestamp*10, timestamp)) } subscriptionTableSQL(t, writer, "flush") + if len(timestamps) != 0 { + unique := make(map[int64]struct{}, len(timestamps)) + start, end := timestamps[0], timestamps[0] + for _, timestamp := range timestamps { + unique[timestamp] = struct{}{} + start = min(start, timestamp) + end = max(end, timestamp) + } + // Flush completion does not guarantee that a newly created snapshot queue can + // immediately observe every write. Wait for the same rows through the read path. + subscriptionTableWaitVisibleRows(t, writer, database, table, start, end, len(unique)) + } +} + +func subscriptionTableInsertSameDevice(t *testing.T, writer client.ITableSession, database, table, tag string, timestamps ...int64) { + t.Helper() + for _, timestamp := range timestamps { + subscriptionTableSQL(t, writer, fmt.Sprintf( + "insert into %s.%s(time,tag1,s1,s2,s3) values (%d,'%s',%d,%d.5,true)", + database, table, timestamp, tag, timestamp*10, timestamp)) + } + subscriptionTableSQL(t, writer, "flush") + if len(timestamps) != 0 { + start, end := timestamps[0], timestamps[0] + unique := make(map[int64]struct{}, len(timestamps)) + for _, timestamp := range timestamps { + unique[timestamp] = struct{}{} + start = min(start, timestamp) + end = max(end, timestamp) + } + subscriptionTableWaitVisibleRows(t, writer, database, table, start, end, len(unique)) + } +} + +func subscriptionTableExpectedSameDevice(database, table, tag string, timestamps ...int64) map[int64]subscriptionTableRecord { + result := make(map[int64]subscriptionTableRecord, len(timestamps)) + for _, timestamp := range timestamps { + result[timestamp] = subscriptionTableRecord{ + Database: database, + Table: table, + Timestamp: timestamp, + Values: map[string]any{ + "tag1": tag, + "s1": timestamp * 10, + "s2": float64(timestamp) + 0.5, + "s3": true, + }, + } + } + return result } type subscriptionTableRecord struct { @@ -186,14 +236,21 @@ func subscriptionTableExpected(database, table string, projected bool, timestamp } func subscriptionTablePollExact(t *testing.T, ctx context.Context, consumer subconsumer.SubscriptionPullConsumer, - want map[int64]subscriptionTableRecord, manualCommit, snapshot, timeSelected bool) { + want map[int64]subscriptionTableRecord, manualCommit, snapshot, timeSelected bool, stage string) { t.Helper() seen := make(map[int64]subscriptionTableRecord) emptyAfterRows := 0 + startedAt := time.Now() + lastProgressLog := startedAt + polls := 0 + deadline, hasDeadline := ctx.Deadline() + t.Logf("[%s] polling for table records: want=%+v deadline=%v", stage, want, deadline) for { - messages, err := consumer.Poll(ctx, time.Second) + messages, err := consumer.Poll(ctx, subscriptionE2EPollTimeout) + polls++ if err != nil { - t.Fatalf("poll table records: %v", err) + t.Fatalf("[%s] poll failed after %s (%d polls), seen=%+v want=%+v: %v", + stage, time.Since(startedAt), polls, seen, want, err) } for _, message := range messages { records, readErr := subscriptionTableRead(message, timeSelected) @@ -203,15 +260,44 @@ func subscriptionTablePollExact(t *testing.T, ctx context.Context, consumer subc } for _, record := range records { expected, exists := want[record.Timestamp] - if !exists || !reflect.DeepEqual(record, expected) { + if !exists || record.Database != expected.Database || record.Table != expected.Table { message.Release() - t.Fatalf("unexpected table record: got=%+v, want=%+v", record, expected) + t.Fatalf("unexpected table record identity: got=%+v, want=%+v", record, expected) } - if _, duplicate := seen[record.Timestamp]; duplicate { - message.Release() - t.Fatalf("duplicate table record at %d", record.Timestamp) + combined, found := seen[record.Timestamp] + if !found { + combined = subscriptionTableRecord{ + Database: record.Database, + Table: record.Table, + Timestamp: record.Timestamp, + Values: make(map[string]any, len(record.Values)), + } + } + for column, value := range record.Values { + wantedValue, selected := expected.Values[column] + if !selected || !reflect.DeepEqual(value, wantedValue) { + message.Release() + t.Fatalf( + "unexpected table value at time %d column %s: got=%v, want=%v", + record.Timestamp, + column, + value, + wantedValue, + ) + } + if previous, duplicate := combined.Values[column]; duplicate && !reflect.DeepEqual(previous, value) { + message.Release() + t.Fatalf( + "conflicting table value at time %d column %s: got=%v, previous=%v", + record.Timestamp, + column, + value, + previous, + ) + } + combined.Values[column] = value } - seen[record.Timestamp] = record + seen[record.Timestamp] = combined } if manualCommit { if err := consumer.Commit(ctx, message); err != nil { @@ -227,20 +313,57 @@ func subscriptionTablePollExact(t *testing.T, ctx context.Context, consumer subc } return } - if !snapshot && len(seen) == len(want) { + if !snapshot && subscriptionTableRecordsComplete(seen, want) { if len(messages) == 0 { emptyAfterRows++ } else { emptyAfterRows = 0 } - if emptyAfterRows == 3 { + if emptyAfterRows == subscriptionE2EQuietPollRounds { + t.Logf("[%s] table records complete after %s (%d polls): seen=%+v", + stage, time.Since(startedAt), polls, seen) return } } if err := ctx.Err(); err != nil { - t.Fatalf("table delivery incomplete: got=%+v, want=%+v: %v", seen, want, err) + t.Fatalf("[%s] table delivery incomplete after %s (%d polls): got=%+v, want=%+v: %v", + stage, time.Since(startedAt), polls, seen, want, err) } + if time.Since(lastProgressLog) >= 5*time.Second { + remaining := time.Duration(0) + if hasDeadline { + remaining = time.Until(deadline) + } + t.Logf("[%s] still waiting after %s (%d polls, %s deadline remaining): seen=%+v want=%+v", + stage, time.Since(startedAt), polls, remaining, seen, want) + lastProgressLog = time.Now() + } + } +} + +func subscriptionTableRecordsComplete(seen, want map[int64]subscriptionTableRecord) bool { + if len(seen) != len(want) { + return false } + for timestamp, expected := range want { + if !reflect.DeepEqual(seen[timestamp], expected) { + return false + } + } + return true +} + +func subscriptionTableRecordMatchesFragment(got, want subscriptionTableRecord) bool { + if got.Database != want.Database || got.Table != want.Table || + got.Timestamp != want.Timestamp || len(got.Values) == 0 { + return false + } + for column, value := range got.Values { + if expected, selected := want.Values[column]; !selected || !reflect.DeepEqual(value, expected) { + return false + } + } + return true } // Java reference: local/tablemodel/IoTDBSubscriptionIsolationIT.testTopicIsolation. @@ -295,7 +418,7 @@ func TestSubscriptionTableSnapshotFiltersAndAutoCommitCompletion(t *testing.T) { "format": "SubscriptionRecordHandler", "start-time": "100", "end-time": "102", }) consumer := subscriptionE2EPull(t, ctx, connection, subscription.TableDialect, topic, true) - subscriptionTablePollExact(t, ctx, consumer, subscriptionTableExpected(database, "events", false, 100, 101, 102), false, true, true) + subscriptionTablePollExact(t, ctx, consumer, subscriptionTableExpected(database, "events", false, 100, 101, 102), false, true, true, "snapshot") } // Java reference: consensus/local/tablemodel/IoTDBConsensusSubscriptionFilterTableIT.testColumnFiltering. @@ -315,39 +438,55 @@ func TestSubscriptionTableIncrementalColumnProjection(t *testing.T) { }) consumer := subscriptionE2EPull(t, ctx, connection, subscription.TableDialect, topic, false) subscriptionTableInsert(t, writer, database, "events", 101, 102, 103) - subscriptionTablePollExact(t, ctx, consumer, subscriptionTableExpected(database, "events", true, 101, 102, 103), true, false, false) + subscriptionTablePollExact(t, ctx, consumer, subscriptionTableExpected(database, "events", true, 101, 102, 103), true, false, false, "incremental column projection") } // Java references: consensus/local/IoTDBConsensusSubscriptionSeekIT.testSeekAfterCheckpointReplaysExactTail // and consensus/local/tablemodel/IoTDBConsensusSubscriptionBasicTableIT.testRealtimeOnlyAfterSubscribe. // Exact per-writer checkpoint replay in the table model is a Go extension. func TestSubscriptionTableSeekAfterReplaysOnlyTail(t *testing.T) { + t.Log("seek-after: begin test") connection := subscriptionE2EConnection(t) ctx := subscriptionE2EContext(t) writer := subscriptionTableWriter(t, connection) database := subscriptionTableDatabase(t, connection, writer) subscriptionTableCreate(t, writer, database, "events") - subscriptionTableInsert(t, writer, database, "events", 0) + const seekDevice = "seek_device" + t.Logf("seek-after: created table %s.events; inserting bootstrap row", database) + subscriptionTableInsertSameDevice(t, writer, database, "events", seekDevice, 0) + t.Log("seek-after: bootstrap insert and flush completed; opening subscription manager") manager := subscriptionE2EManager(t, connection, subscription.TableDialect) topic := subscriptionE2EName("sub_table_seek") subscriptionE2ETopic(t, ctx, manager, topic, map[string]string{ "database": database, "table": "events", "mode": "incremental", "format": "SubscriptionRecordHandler", "order-mode": "per-writer", }) + t.Logf("seek-after: topic %s created; opening consumer", topic) consumer := subscriptionE2EPull(t, ctx, connection, subscription.TableDialect, topic, false) - subscriptionTableInsert(t, writer, database, "events", 101, 102) - subscriptionTablePollExact(t, ctx, consumer, subscriptionTableExpected(database, "events", false, 101, 102), true, false, true) + t.Log("seek-after: consumer subscribed; inserting checkpoint rows 101/102") + subscriptionTableInsertSameDevice(t, writer, database, "events", seekDevice, 101, 102) + t.Log("seek-after: checkpoint rows inserted and flushed; polling initial delivery") + subscriptionTablePollExact(t, ctx, consumer, subscriptionTableExpectedSameDevice(database, "events", seekDevice, 101, 102), true, false, true, "seek initial checkpoint rows") + t.Log("seek-after: initial rows committed; reading committed checkpoint") checkpoint, err := consumer.CommittedPositions(topic) if err != nil || len(checkpoint) == 0 { t.Fatalf("committed prefix has no resumable checkpoint: %v, %v", checkpoint, err) } - subscriptionTableInsert(t, writer, database, "events", 201, 202) - tail := subscriptionTableExpected(database, "events", false, 201, 202) - subscriptionTablePollExact(t, ctx, consumer, tail, true, false, true) + t.Logf("seek-after: captured checkpoint=%+v", checkpoint) + t.Log("seek-after: inserting tail rows 201/202") + subscriptionTableInsertSameDevice(t, writer, database, "events", seekDevice, 201, 202) + t.Log("seek-after: tail rows inserted and flushed") + tail := subscriptionTableExpectedSameDevice(database, "events", seekDevice, 201, 202) + t.Log("seek-after: polling and committing tail before seek") + subscriptionTablePollExact(t, ctx, consumer, tail, true, false, true, "seek tail before seek") + t.Logf("seek-after: checkpoint=%+v", checkpoint) + seekStartedAt := time.Now() + t.Log("seek-after: calling SeekAfter") if err := consumer.SeekAfter(ctx, topic, checkpoint); err != nil { - t.Fatalf("seek after committed prefix: %v", err) + t.Fatalf("seek after committed prefix after %s: %v", time.Since(seekStartedAt), err) } - subscriptionTablePollExact(t, ctx, consumer, tail, true, false, true) + t.Logf("seek-after: SeekAfter returned after %s; polling replay tail", time.Since(seekStartedAt)) + subscriptionTablePollExact(t, ctx, consumer, tail, true, false, true, "seek replay tail") } func subscriptionTablePush(t *testing.T, ctx context.Context, connection subscription.ConnectionConfig, @@ -359,7 +498,7 @@ func subscriptionTablePush(t *testing.T, ctx context.Context, connection subscri config.Consumer.ConsumerID = subscriptionE2EName("sub_table_consumer") config.Consumer.GroupID = subscriptionE2EName("sub_table_group") config.AckStrategy = subconsumer.AckAfterConsume - config.PollTimeout = time.Second + config.PollTimeout = subscriptionE2EPollTimeout config.Handler = handler consumer, err := subconsumer.NewSubscriptionPushConsumer(config) if err != nil { @@ -419,9 +558,14 @@ func TestSubscriptionTablePushRetriesFailedCallback(t *testing.T) { for index := 0; index < 2; index++ { select { case got := <-attempts: - if got.err != nil || !reflect.DeepEqual(got.records, []subscriptionTableRecord{want}) { + if got.err != nil || len(got.records) == 0 { t.Fatalf("callback received wrong table data: %+v, %v", got.records, got.err) } + for _, record := range got.records { + if !subscriptionTableRecordMatchesFragment(record, want) { + t.Fatalf("callback received wrong table data: %+v, %v", got.records, got.err) + } + } if index == 0 { first = got.context } else if !reflect.DeepEqual(first, got.context) { @@ -438,7 +582,7 @@ func TestSubscriptionTablePushRetriesFailedCallback(t *testing.T) { } // Observe confirmation after callback success, rather than treating the // callback itself or absence of a terminal error as proof of acknowledgment. - ticker := time.NewTicker(100 * time.Millisecond) + ticker := time.NewTicker(subscriptionTableE2EProgressPollInterval) defer ticker.Stop() for { progress, err := consumer.CommittedPositions(topic) @@ -459,6 +603,9 @@ func TestSubscriptionTablePushRetriesFailedCallback(t *testing.T) { case <-ticker.C: } } + if err := consumer.Unsubscribe(ctx, topic); err != nil { + t.Fatalf("unsubscribe push retry topic: %v", err) + } if err := consumer.Close(); err != nil { t.Fatal(err) } @@ -508,3 +655,310 @@ func TestSubscriptionTableTsFilePushFailsBeforeCallback(t *testing.T) { t.Fatalf("TsFile payload reached record callback %d times", calls.Load()) } } + +// These cases separate snapshot completion into observable stages. Existing +// permissive snapshot coverage remains unchanged; these strict cases allow +// repeated TAGs in disjoint field fragments, reject repeated fields or tag-only +// redeliveries, and verify writer progress before termination-driven completion. +func TestSubscriptionTableSnapshotStrictAutoCommitCompletion(t *testing.T) { + runSubscriptionTableStrictSnapshot(t, true) +} + +func TestSubscriptionTableSnapshotStrictManualCommitCompletion(t *testing.T) { + runSubscriptionTableStrictSnapshot(t, false) +} + +func runSubscriptionTableStrictSnapshot(t *testing.T, autoCommit bool) { + t.Helper() + connection := subscriptionE2EConnection(t) + ctx := subscriptionE2EContext(t) + writer := subscriptionTableWriter(t, connection) + database := subscriptionTableDatabase(t, connection, writer) + subscriptionTableCreate(t, writer, database, "events") + subscriptionTableInsert(t, writer, database, "events", 99, 100, 101, 102, 103) + + manager := subscriptionE2EManager(t, connection, subscription.TableDialect) + topic := subscriptionE2EName("sub_table_strict_snapshot") + subscriptionE2ETopic(t, ctx, manager, topic, map[string]string{ + "database": database, "table": "events", "mode": "snapshot", + "format": "SubscriptionRecordHandler", "start-time": "100", "end-time": "102", + }) + consumer := subscriptionE2EPull(t, ctx, connection, subscription.TableDialect, topic, autoCommit) + want := subscriptionTableExpected(database, "events", false, 100, 101, 102) + tagColumns := map[string]struct{}{"tag1": {}} + subscriptionTablePollStrictSnapshot(t, ctx, consumer, topic, want, tagColumns, !autoCommit) +} + +func subscriptionTableWaitVisibleRows(t *testing.T, writer client.ITableSession, database, table string, start, end int64, want int) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for { + timeout := int64(subscriptionE2EWriterTimeoutMs) + dataSet, err := writer.ExecuteQueryStatement(fmt.Sprintf( + "select time from %s.%s where time >= %d and time <= %d order by time", + database, table, start, end), &timeout) + if err != nil { + t.Fatalf("query table rows before snapshot topic: %v", err) + } + count := 0 + for { + hasNext, nextErr := dataSet.Next() + if nextErr != nil { + _ = dataSet.Close() + t.Fatalf("read table rows before snapshot topic: %v", nextErr) + } + if !hasNext { + break + } + count++ + } + if closeErr := dataSet.Close(); closeErr != nil { + t.Fatalf("close visibility query: %v", closeErr) + } + if count == want { + return + } + if time.Now().After(deadline) { + t.Fatalf("snapshot source rows not visible before topic creation: got=%d want=%d", count, want) + } + time.Sleep(50 * time.Millisecond) + } +} + +func subscriptionTablePollStrictSnapshot( + t *testing.T, + ctx context.Context, + consumer subconsumer.SubscriptionPullConsumer, + topic string, + want map[int64]subscriptionTableRecord, + tagColumns map[string]struct{}, + manualCommit bool, +) { + t.Helper() + seen := make(map[int64]subscriptionTableRecord) + startedAt := time.Now() + polls := 0 + positionChecked := false + requiredProgress := make(subpayload.TopicProgress) + var manualCommitContexts []subpayload.CommitContext + + for { + messages, err := consumer.Poll(ctx, subscriptionE2EPollTimeout) + polls++ + if err != nil { + t.Fatalf("strict snapshot poll %d failed after %s: %v", polls, time.Since(startedAt), err) + } + recordCount := 0 + contexts := make([]string, 0, len(messages)) + for _, message := range messages { + context := message.CommitContext() + contexts = append(contexts, fmt.Sprintf("topic=%s group=%s node=%d reboot=%d commit=%d seek=%d region=%s physical=%d writer=%v", + context.Topic, context.GroupID, context.DataNodeID, context.RebootTimes, context.CommitID, context.SeekGeneration, context.RegionID, context.PhysicalTime, context.Writer)) + subscriptionTableTrackWriterProgress(requiredProgress, context) + records, readErr := subscriptionTableRead(message, true) + if readErr != nil { + message.Release() + t.Fatalf("strict snapshot read after %d polls: %v", polls, readErr) + } + recordCount += len(records) + if mergeErr := subscriptionTableMergeStrict(seen, want, records, tagColumns); mergeErr != nil { + message.Release() + t.Fatal(mergeErr) + } + if manualCommit { + if subscriptionTableContextSeen(manualCommitContexts, context) { + t.Logf("[strict snapshot] skipped duplicate manual ACK at poll=%d context=%+v", polls, context) + } else { + if err := consumer.Commit(ctx, message); err != nil { + message.Release() + t.Fatalf("strict snapshot manual commit after %d polls: %v", polls, err) + } + manualCommitContexts = append(manualCommitContexts, context) + } + } + message.Release() + } + + if !manualCommit { + if autoCommitErr := consumer.LastAutoCommitError(); autoCommitErr != nil { + t.Fatalf("strict snapshot auto-commit failed after %d polls: %v", polls, autoCommitErr) + } + } + complete := subscriptionTableRecordsComplete(seen, want) + t.Logf("[strict snapshot] poll=%d messages=%d records=%d seen=%d/%d complete=%t consumed=%t manualCommit=%t contexts=%v", + polls, len(messages), recordCount, len(seen), len(want), complete, consumer.AllConsumed(), manualCommit, contexts) + + if complete && len(requiredProgress) == 0 && !positionChecked { + t.Logf("[strict snapshot] delivered contexts use legacy commit metadata; writer-position assertion unavailable") + positionChecked = true + } + if complete && len(requiredProgress) != 0 && !positionChecked { + if manualCommit { + subscriptionTableRequireCommittedPosition(t, consumer, topic, requiredProgress) + } else { + subscriptionTableWaitCommittedPosition(t, ctx, consumer, topic, requiredProgress) + } + positionChecked = true + } + if consumer.AllConsumed() && !complete { + t.Fatalf("snapshot reported completion before all records arrived: seen=%+v want=%+v", seen, want) + } + if complete && consumer.AllConsumed() { + if !reflect.DeepEqual(seen, want) { + t.Fatalf("strict snapshot completed with records=%+v, want=%+v", seen, want) + } + return + } + if err := ctx.Err(); err != nil { + positions, positionErr := consumer.CommittedPositions(topic) + t.Fatalf("strict snapshot incomplete after %s (%d polls): complete=%t requiredProgress=%+v positionChecked=%t positions=%+v positionError=%v consumed=%t error=%v", + time.Since(startedAt), polls, complete, requiredProgress, positionChecked, positions, positionErr, consumer.AllConsumed(), err) + } + } +} + +func subscriptionTableMergeStrict( + seen map[int64]subscriptionTableRecord, + want map[int64]subscriptionTableRecord, + records []subscriptionTableRecord, + tagColumns map[string]struct{}, +) error { + for _, record := range records { + expected, exists := want[record.Timestamp] + if !exists || record.Database != expected.Database || record.Table != expected.Table { + return fmt.Errorf("strict snapshot unexpected identity: got=%+v, want=%+v", record, expected) + } + combined, found := seen[record.Timestamp] + if !found { + combined = subscriptionTableRecord{ + Database: record.Database, + Table: record.Table, + Timestamp: record.Timestamp, + Values: make(map[string]any, len(record.Values)), + } + } + newFields := 0 + for column, value := range record.Values { + wantedValue, selected := expected.Values[column] + if !selected || !reflect.DeepEqual(value, wantedValue) { + return fmt.Errorf("strict snapshot unexpected value at time %d column %s: got=%v, want=%v", record.Timestamp, column, value, wantedValue) + } + previous, duplicate := combined.Values[column] + if _, tag := tagColumns[column]; tag { + // Table events can repeat the same TAG in separate disjoint field + // fragments. The TAG must stay identical, but its repetition alone is + // not evidence of redelivery. + if duplicate && !reflect.DeepEqual(previous, value) { + return fmt.Errorf("strict snapshot conflicting TAG at time %d column %s: got=%v, previous=%v", record.Timestamp, column, value, previous) + } + } else { + if duplicate { + return fmt.Errorf("strict snapshot duplicate field delivery at time %d column %s: got=%v, previous=%v", record.Timestamp, column, value, previous) + } + newFields++ + } + combined.Values[column] = value + } + if newFields == 0 { + return fmt.Errorf("strict snapshot redelivered time %d without a new field fragment", record.Timestamp) + } + seen[record.Timestamp] = combined + } + return nil +} + +func subscriptionTableContextSeen(seen []subpayload.CommitContext, context subpayload.CommitContext) bool { + for _, existing := range seen { + if reflect.DeepEqual(existing, context) { + return true + } + } + return false +} + +func subscriptionTableTrackWriterProgress(required subpayload.TopicProgress, context subpayload.CommitContext) { + if context.Writer == nil { + return + } + region := context.Writer.ID.RegionID + if region == "" { + return + } + regionProgress := required[region] + if regionProgress == nil { + regionProgress = make(subpayload.RegionProgress) + required[region] = regionProgress + } + previous, exists := regionProgress[context.Writer.ID] + if !exists || context.Writer.Progress.PhysicalTime > previous.PhysicalTime || + (context.Writer.Progress.PhysicalTime == previous.PhysicalTime && context.Writer.Progress.LocalSeq > previous.LocalSeq) { + regionProgress[context.Writer.ID] = context.Writer.Progress + } +} + +func subscriptionTableProgressCovers(committed, required subpayload.TopicProgress) bool { + for region, requiredWriters := range required { + committedWriters := committed[region] + for writer, requiredProgress := range requiredWriters { + committedProgress, exists := committedWriters[writer] + if !exists || committedProgress.PhysicalTime < requiredProgress.PhysicalTime || + (committedProgress.PhysicalTime == requiredProgress.PhysicalTime && committedProgress.LocalSeq < requiredProgress.LocalSeq) { + return false + } + } + } + return true +} + +func subscriptionTableRequireCommittedPosition( + t *testing.T, + consumer subconsumer.SubscriptionPullConsumer, + topic string, + required subpayload.TopicProgress, +) { + t.Helper() + positions, err := consumer.CommittedPositions(topic) + if err != nil { + t.Fatalf("read committed positions after manual commit: %v", err) + } + if !subscriptionTableProgressCovers(positions, required) { + t.Fatalf("manual commit did not cover delivered writer progress: committed=%+v required=%+v", positions, required) + } +} + +func subscriptionTableWaitCommittedPosition( + t *testing.T, + ctx context.Context, + consumer subconsumer.SubscriptionPullConsumer, + topic string, + required subpayload.TopicProgress, +) { + t.Helper() + waitCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + ticker := time.NewTicker(50 * time.Millisecond) + defer ticker.Stop() + var lastPositions subpayload.TopicProgress + var lastErr error + for { + if autoCommitErr := consumer.LastAutoCommitError(); autoCommitErr != nil { + t.Fatalf("auto-commit failed before writer position advanced: %v", autoCommitErr) + } + positions, err := consumer.CommittedPositions(topic) + if err != nil { + lastErr = err + } else { + lastPositions = positions + if subscriptionTableProgressCovers(positions, required) { + t.Logf("[strict snapshot] auto-commit position covered delivered progress: committed=%+v required=%+v", positions, required) + return + } + } + select { + case <-waitCtx.Done(): + t.Fatalf("auto-commit did not cover delivered writer progress for topic %s: committed=%+v required=%+v lastError=%v waitError=%v", + topic, lastPositions, required, lastErr, waitCtx.Err()) + case <-ticker.C: + } + } +} diff --git a/test/e2e/subscription_tree_test.go b/test/e2e/subscription_tree_test.go index bd2cae3..a175f6c 100644 --- a/test/e2e/subscription_tree_test.go +++ b/test/e2e/subscription_tree_test.go @@ -23,7 +23,6 @@ import ( "context" "errors" "fmt" - "os" "reflect" "strings" "sync/atomic" @@ -37,50 +36,9 @@ import ( "github.com/apache/iotdb-client-go/v2/client" ) -// Subscription tests require explicit opt-in and a subscription-capable server. -// Set IOTDB_SUBSCRIPTION_E2E=1 and optionally IOTDB_SUBSCRIPTION_NODE_URLS to a -// comma-separated seed list. Incremental cases require WAL subscriptions with a -// compatible DataRegion consensus protocol; Ratis is not supported. Opting in -// makes unavailable server features fail rather than silently skip these cases. -// Java topic/path/time/snapshot ITs provide design references, but some are -// @Ignore in that checkout; their presence is not evidence these Go tests ran. -func subscriptionE2EConnection(t *testing.T) subscription.ConnectionConfig { - t.Helper() - if os.Getenv("IOTDB_SUBSCRIPTION_E2E") != "1" { - t.Skip("set IOTDB_SUBSCRIPTION_E2E=1 to enable subscription server tests") - } - nodes := os.Getenv("IOTDB_SUBSCRIPTION_NODE_URLS") - if nodes == "" { - nodes = nodeUrls - } - var addresses []string - for _, address := range strings.Split(nodes, ",") { - if address = strings.TrimSpace(address); address != "" { - addresses = append(addresses, address) - } - } - if len(addresses) == 0 { - t.Fatal("subscription endpoint list is empty") - } - user := os.Getenv("IOTDB_SUBSCRIPTION_USERNAME") - if user == "" { - user = "root" - } - secret, supplied := os.LookupEnv("IOTDB_SUBSCRIPTION_PASSWORD") - if !supplied { - secret, supplied = os.LookupEnv("IOTDB_PASSWORD") - } - if !supplied { - secret = "root" - } - config := subsession.DefaultSubscriptionSessionConfig().Connection - config.NodeURLs, config.Username, config.Password = addresses, user, secret - return config -} - func subscriptionE2EContext(t *testing.T) context.Context { t.Helper() - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), subscriptionE2ETestTimeout) t.Cleanup(cancel) return ctx } @@ -104,7 +62,7 @@ func subscriptionE2EManager(t *testing.T, connection subscription.ConnectionConf t.Errorf("close subscription manager: %v", err) } }) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), subscriptionE2EOpenTimeout) defer cancel() if err := manager.Open(ctx); err != nil { t.Fatal(err) @@ -118,10 +76,19 @@ func subscriptionE2ETopic(t *testing.T, ctx context.Context, manager subsession. t.Fatal(err) } t.Cleanup(func() { - cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + cleanupCtx, cancel := context.WithTimeout(context.Background(), subscriptionE2ECleanupTimeout) defer cancel() - if err := manager.DropTopic(cleanupCtx, name, subsession.DropOptions{IfExists: true}); err != nil { - t.Errorf("drop topic %s: %v", name, err) + for { + err := manager.DropTopic(cleanupCtx, name, subsession.DropOptions{IfExists: true}) + if err == nil { + return + } + select { + case <-cleanupCtx.Done(): + t.Errorf("drop topic %s: %v", name, err) + return + case <-time.After(100 * time.Millisecond): + } } }) } @@ -130,9 +97,9 @@ func subscriptionE2EPull(t *testing.T, ctx context.Context, connection subscript t.Helper() config := subconsumer.DefaultSubscriptionPullConsumerConfig() config.Consumer.Connection, config.Consumer.Dialect = connection, dialect - config.Consumer.HeartbeatInterval = time.Second - config.Consumer.EndpointsSyncInterval = 5 * time.Second - config.AutoCommit, config.AutoCommitInterval = &autoCommit, 500*time.Millisecond + config.Consumer.HeartbeatInterval = subscriptionE2EHeartbeatInterval + config.Consumer.EndpointsSyncInterval = subscriptionE2EEndpointsSyncInterval + config.AutoCommit, config.AutoCommitInterval = &autoCommit, subscriptionE2EAutoCommitInterval consumer, err := subconsumer.NewSubscriptionPullConsumer(config) if err != nil { t.Fatal(err) @@ -162,7 +129,7 @@ func subscriptionTreeWriter(t *testing.T, connection subscription.ConnectionConf if err != nil { t.Fatal(err) } - if err := session.Open(false, 10000); err != nil { + if err := session.Open(false, subscriptionE2EWriterTimeoutMs); err != nil { _ = session.Close() t.Fatal(err) } @@ -176,9 +143,9 @@ func subscriptionTreeWriter(t *testing.T, connection subscription.ConnectionConf t.Errorf("close tree writer: %v", err) } }) - subscriptionTreeSQL(t, &session, "CREATE DATABASE " + database) + subscriptionTreeSQL(t, &session, "CREATE DATABASE "+database) for _, series := range []string{".d1.s1", ".d1.s2", ".d2.s1"} { - subscriptionTreeSQL(t, &session, "CREATE TIMESERIES " + database + series + " WITH DATATYPE=INT64, ENCODING=RLE") + subscriptionTreeSQL(t, &session, "CREATE TIMESERIES "+database+series+" WITH DATATYPE=INT64, ENCODING=RLE") } return &session, database } @@ -191,10 +158,10 @@ func subscriptionTreeSQL(t *testing.T, writer *client.Session, sql string) { } type subscriptionTreeRow struct { - target string - column string + target string + column string timestamp int64 - value int64 + value int64 } func subscriptionTreeInsert(t *testing.T, writer *client.Session, database, device, column string, timestamp, value int64) subscriptionTreeRow { @@ -244,8 +211,8 @@ func subscriptionTreeCollect(t *testing.T, ctx context.Context, consumer subcons t.Helper() seen := make(map[subscriptionTreeRow]int) empty := 0 - for round := 0; round < 40; round++ { - messages, err := consumer.Poll(ctx, time.Second) + for round := 0; round < subscriptionE2EMaxPollRounds; round++ { + messages, err := consumer.Poll(ctx, subscriptionE2EPollTimeout) if err != nil { t.Fatal(err) } @@ -265,7 +232,7 @@ func subscriptionTreeCollect(t *testing.T, ctx context.Context, consumer subcons if consumer.AllConsumed() { return seen } - } else if len(seen) >= expected && empty >= 3 { + } else if len(seen) >= expected && empty >= subscriptionE2EQuietPollRounds { return seen } } @@ -462,10 +429,10 @@ func TestSubscriptionTreeIncrementalSeekAfterReplaysExactTail(t *testing.T) { } type subscriptionTreePushResult struct { - rows []subscriptionTreeRow + rows []subscriptionTreeRow context subpayload.CommitContext - failed bool - err error + failed bool + err error } // This supplements Java's local/IoTDBSubscriptionBasicIT record Push scenarios @@ -485,8 +452,8 @@ func TestSubscriptionTreePushAfterConsumeRedeliversOneFailure(t *testing.T) { results := make(chan subscriptionTreePushResult, 8) config := subconsumer.DefaultSubscriptionPushConsumerConfig() config.Consumer.Connection = connection - config.Consumer.HeartbeatInterval = time.Second - config.PollTimeout, config.PollInterval = time.Second, 100*time.Millisecond + config.Consumer.HeartbeatInterval = subscriptionE2EHeartbeatInterval + config.PollTimeout, config.PollInterval = subscriptionE2EPollTimeout, subscriptionE2EPushPollInterval config.AckStrategy = subconsumer.AckAfterConsume config.Handler = func(callbackCtx context.Context, message subpayload.SubscriptionMessage) error { rows, err := subscriptionTreeRows(callbackCtx, message) @@ -513,7 +480,7 @@ func TestSubscriptionTreePushAfterConsumeRedeliversOneFailure(t *testing.T) { if err := consumer.Close(); err != nil { t.Errorf("close push consumer: %v", err) } - cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + cleanupCtx, cancel := context.WithTimeout(context.Background(), subscriptionE2ECleanupTimeout) defer cancel() for { select { @@ -558,7 +525,7 @@ func TestSubscriptionTreePushAfterConsumeRedeliversOneFailure(t *testing.T) { if confirmed.Writer == nil { t.Fatal("incremental delivery did not expose writer progress") } - ticker := time.NewTicker(20 * time.Millisecond) + ticker := time.NewTicker(subscriptionTreeE2EProgressPollInterval) defer ticker.Stop() for { progress, err := consumer.CommittedPositions(topic) @@ -594,8 +561,8 @@ func TestSubscriptionTreeTsFilePollIsExplicitlyUnsupported(t *testing.T) { topic := subscriptionE2EName("tree_tsfile") subscriptionE2ETopic(t, ctx, manager, topic, map[string]string{"format": "SubscriptionTsFileHandler", "mode": "initial", "path": database + ".d1.s1"}) consumer := subscriptionE2EPull(t, ctx, connection, subscription.TreeDialect, topic, false) - for round := 0; round < 40; round++ { - messages, err := consumer.Poll(ctx, time.Second) + for round := 0; round < subscriptionE2EMaxPollRounds; round++ { + messages, err := consumer.Poll(ctx, subscriptionE2EPollTimeout) if len(messages) != 0 { for _, message := range messages { message.Release()
