This is an automated email from the ASF dual-hosted git repository.

shuwenwei pushed a commit to branch supports_subscription
in repository https://gitbox.apache.org/repos/asf/iotdb-client-go.git

commit cc734d4508a39007d9a5d22428c100d153caa1f7
Author: shuwenwei <[email protected]>
AuthorDate: Fri Nov 13 12:43:36 2026 +0800

    Add subscription protocol and processor behavior unit tests
---
 subscription/column_align_processor_test.go | 215 +++++++++++++++++
 subscription/processor_fixture_test.go      | 263 +++++++++++++++++++++
 subscription/subscription_poll_test.go      | 347 ++++++++++++++++++++++++++++
 subscription/watermark_processor_test.go    | 204 ++++++++++++++++
 4 files changed, 1029 insertions(+)

diff --git a/subscription/column_align_processor_test.go 
b/subscription/column_align_processor_test.go
new file mode 100644
index 0000000..7a63c4a
--- /dev/null
+++ b/subscription/column_align_processor_test.go
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package subscription_test
+
+import (
+       "bytes"
+       "errors"
+       "testing"
+
+       "github.com/apache/iotdb-client-go/subscription"
+       "github.com/apache/iotdb-client-go/v2/client"
+)
+
+func newTestColumnAlignProcessor(t *testing.T, dialect subscription.Dialect) 
*subscription.ColumnAlignProcessor {
+       t.Helper()
+       processor, err := subscription.NewColumnAlignProcessor(dialect)
+       if err != nil {
+               t.Fatal(err)
+       }
+       t.Cleanup(processor.Reset)
+       return processor
+}
+
+func processAlignment(t *testing.T, processor 
*subscription.ColumnAlignProcessor, input ...subscription.SubscriptionMessage) 
[]subscription.SubscriptionMessage {
+       t.Helper()
+       messages, err := processor.Process(input)
+       if err != nil {
+               t.Fatal(err)
+       }
+       t.Cleanup(func() { releaseProcessorMessages(messages) })
+       return messages
+}
+
+func TestColumnAlignNeverFillsFromEqualOrFutureTimestamps(t *testing.T) {
+       processor := newTestColumnAlignProcessor(t, subscription.TreeDialect)
+       seed := processorRecord("topic", "region", 1, 
processorTreeBatch([]int64{10}, int64(100)))
+       processAlignment(t, processor, seed)
+       input := processorRecord("topic", "region", 2, processorTreeBatch(
+               []int64{10, 5, 6, 11, 12, 13}, nil, nil, int64(60), nil, 
int64(120), nil,
+       ))
+       messages := processAlignment(t, processor, input)
+       for row, expected := range []any{nil, nil, int64(60), int64(100), 
int64(120), int64(120)} {
+               if value := processorOutputValue(t, messages, row, 0); value != 
expected {
+                       t.Fatalf("raw row %d value=%v, want %v", row, value, 
expected)
+               }
+       }
+       if _, err := input.Records(); !errors.Is(err, 
subscription.ErrDataReleased) {
+               t.Fatalf("consumed input was not released: %v", err)
+       }
+}
+
+func TestColumnAlignMatchesColumnNamesAfterSchemaReordering(t *testing.T) {
+       processor := newTestColumnAlignProcessor(t, subscription.TreeDialect)
+       seed := processorTreeBatch([]int64{1})
+       seed.columns = []subscription.Column{
+               {Name: "left", DataType: client.INT64, Category: client.FIELD},
+               {Name: "right", DataType: client.INT64, Category: client.FIELD},
+       }
+       seed.values = [][]any{{int64(11), int64(22)}}
+       processAlignment(t, processor, processorRecord("topic", "region", 1, 
seed))
+       input := processorTreeBatch([]int64{2})
+       input.columns = []subscription.Column{
+               {Name: "right", DataType: client.INT64, Category: client.FIELD},
+               {Name: "left", DataType: client.INT64, Category: client.FIELD},
+               {Name: "new", DataType: client.INT64, Category: client.FIELD},
+       }
+       input.values = [][]any{{nil, nil, nil}}
+       messages := processAlignment(t, processor, processorRecord("topic", 
"region", 2, input))
+       for column, expected := range []any{int64(22), int64(11), nil} {
+               if value := processorOutputValue(t, messages, 0, column); value 
!= expected {
+                       t.Fatalf("column %d reused another column's value: 
got=%v want=%v", column, value, expected)
+               }
+       }
+}
+
+func TestColumnAlignTableIdentityCannotLeakValuesAcrossDevices(t *testing.T) {
+       for _, test := range []struct {
+               name string
+               topic string
+               database string
+               table string
+               seedTags []any
+               otherTags []any
+       }{
+               {name: "topic", topic: "other", database: "db", table: "table", 
seedTags: []any{"a"}, otherTags: []any{"a"}},
+               {name: "database", topic: "topic", database: "other", table: 
"table", seedTags: []any{"a"}, otherTags: []any{"a"}},
+               {name: "table", topic: "topic", database: "db", table: "other", 
seedTags: []any{"a"}, otherTags: []any{"a"}},
+               {name: "tag", topic: "topic", database: "db", table: "table", 
seedTags: []any{"a"}, otherTags: []any{"b"}},
+               {name: "null_and_empty", topic: "topic", database: "db", table: 
"table", seedTags: []any{nil, "x"}, otherTags: []any{"", "x"}},
+               {name: "embedded_delimiters", topic: "topic", database: "db", 
table: "table", seedTags: []any{"a|b", "c"}, otherTags: []any{"a", "b|c"}},
+       } {
+               t.Run(test.name, func(t *testing.T) {
+                       processor := newTestColumnAlignProcessor(t, 
subscription.TableDialect)
+                       processAlignment(t, processor, 
processorTableRecord("topic", "db", "table", test.seedTags, 1, int64(101)))
+                       other := processAlignment(t, processor, 
processorTableRecord(test.topic, test.database, test.table, test.otherTags, 2, 
nil))
+                       if value := processorOutputValue(t, other, 0, 
len(test.otherTags)); value != nil {
+                               t.Fatalf("another device inherited the seed 
value: %v", value)
+                       }
+                       same := processAlignment(t, processor, 
processorTableRecord("topic", "db", "table", test.seedTags, 3, nil))
+                       if value := processorOutputValue(t, same, 0, 
len(test.seedTags)); value != int64(101) {
+                               t.Fatalf("same device lost its value: %v", 
value)
+                       }
+               })
+       }
+}
+
+func TestColumnAlignTableIdentityIgnoresTrailingNullTags(t *testing.T) {
+       for _, test := range []struct {
+               name string
+               seedTags []any
+               nextTags []any
+       }{
+               {name: "trailing_null", seedTags: []any{"device"}, nextTags: 
[]any{"device", nil}},
+               {name: "no_tags_and_all_null", seedTags: nil, nextTags: 
[]any{nil, nil}},
+       } {
+               t.Run(test.name, func(t *testing.T) {
+                       processor := newTestColumnAlignProcessor(t, 
subscription.TableDialect)
+                       processAlignment(t, processor, 
processorTableRecord("topic", "db", "table", test.seedTags, 1, int64(101)))
+                       messages := processAlignment(t, processor, 
processorTableRecord("topic", "db", "table", test.nextTags, 2, nil))
+                       if value := processorOutputValue(t, messages, 0, 
len(test.nextTags)); value != int64(101) {
+                               t.Fatalf("equivalent device identity did not 
fill its field: %v", value)
+                       }
+               })
+       }
+}
+
+func TestColumnAlignResetTopicDoesNotDiscardOtherTopicValues(t *testing.T) {
+       processor := newTestColumnAlignProcessor(t, subscription.TreeDialect)
+       processAlignment(t, processor,
+               processorRecord("a", "region", 1, 
processorTreeBatch([]int64{10}, int64(11))),
+               processorRecord("b", "region", 2, 
processorTreeBatch([]int64{10}, int64(22))),
+       )
+       processor.ResetTopic("a")
+       a := processAlignment(t, processor, processorRecord("a", "region", 3, 
processorTreeBatch([]int64{20})))
+       if value := processorOutputValue(t, a, 0, 0); value != nil {
+               t.Fatalf("reset topic reused its obsolete cached value: %v", 
value)
+       }
+       b := processAlignment(t, processor, processorRecord("b", "region", 4, 
processorTreeBatch([]int64{20})))
+       if value := processorOutputValue(t, b, 0, 0); value != int64(22) {
+               t.Fatalf("reset discarded another topic's value: %v", value)
+       }
+}
+
+func TestColumnAlignCopiedBytesSurviveInputAndOutputRelease(t *testing.T) {
+       processor := newTestColumnAlignProcessor(t, subscription.TreeDialect)
+       seedBatch := processorTreeBatch([]int64{1}, []byte{1, 2, 3})
+       seedBatch.columns[0].DataType = client.BLOB
+       seed := processorRecord("topic", "region", 1, seedBatch)
+       messages := processAlignment(t, processor, seed)
+       if _, err := seed.Records(); !errors.Is(err, 
subscription.ErrDataReleased) {
+               t.Fatalf("input storage was not reclaimed: %v", err)
+       }
+       if value, ok := processorOutputValue(t, messages, 0, 0).([]byte); !ok 
|| !bytes.Equal(value, []byte{1, 2, 3}) {
+               t.Fatalf("output borrowed storage reclaimed by input Release: 
%v", value)
+       }
+       releaseProcessorMessages(messages)
+       for timestamp := int64(2); timestamp <= 3; timestamp++ {
+               batch := processorTreeBatch([]int64{timestamp})
+               batch.columns[0].DataType = client.BLOB
+               messages = processAlignment(t, processor, 
processorRecord("topic", "region", timestamp, batch))
+               if value, ok := processorOutputValue(t, messages, 0, 
0).([]byte); !ok || !bytes.Equal(value, []byte{1, 2, 3}) {
+                       t.Fatalf("cached bytes failed after earlier Release: 
%v", value)
+               }
+               releaseProcessorMessages(messages)
+       }
+}
+
+func TestColumnAlignFailedInputCannotPolluteEarlierValues(t *testing.T) {
+       for _, failure := range []string{"dialect", "read"} {
+               t.Run(failure, func(t *testing.T) {
+                       processor := newTestColumnAlignProcessor(t, 
subscription.TreeDialect)
+                       processAlignment(t, processor, processorRecord("topic", 
"region", 1, processorTreeBatch([]int64{10}, int64(11))))
+                       good := processorRecord("topic", "region", 2, 
processorTreeBatch([]int64{20}, int64(99)))
+                       brokenBatch := processorTreeBatch([]int64{21}, 
int64(123))
+                       readFailure := errors.New("input value unavailable")
+                       if failure == "dialect" {
+                               brokenBatch.dialect = subscription.TableDialect
+                       } else {
+                               brokenBatch.valueErr = readFailure
+                       }
+                       broken := processorRecord("topic", "region", 3, 
brokenBatch)
+                       messages, err := 
processor.Process([]subscription.SubscriptionMessage{good, broken})
+                       if err == nil || len(messages) != 0 || (failure == 
"read" && !errors.Is(err, readFailure)) {
+                               t.Fatalf("failed round delivered data or lost 
the read failure: messages=%v error=%v", messages, err)
+                       }
+                       for _, input := range []*processorInputMessage{good, 
broken} {
+                               if _, err := input.Records(); err != nil {
+                                       t.Fatalf("failed round consumed 
caller-owned input: %v", err)
+                               }
+                               defer input.Release()
+                       }
+                       messages = processAlignment(t, processor, 
processorRecord("topic", "region", 4, processorTreeBatch([]int64{30})))
+                       if value := processorOutputValue(t, messages, 0, 0); 
value != int64(11) {
+                               t.Fatalf("failed round changed the cache: %v", 
value)
+                       }
+               })
+       }
+}
diff --git a/subscription/processor_fixture_test.go 
b/subscription/processor_fixture_test.go
new file mode 100644
index 0000000..aae4ab1
--- /dev/null
+++ b/subscription/processor_fixture_test.go
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package subscription_test
+
+import (
+       "fmt"
+       "iter"
+       "sort"
+       "testing"
+
+       "github.com/apache/iotdb-client-go/subscription"
+       "github.com/apache/iotdb-client-go/v2/client"
+)
+
+// processorInputMessage is a caller-owned implementation of the public message
+// contract. Release reclaims its borrowed buffers to exercise ownership 
transfer.
+type processorInputMessage struct {
+       context subscription.CommitContext
+       batches []*processorInputBatch
+       watermark *int64
+       released bool
+}
+
+func processorRecord(topic, region string, commitID int64, batches 
...*processorInputBatch) *processorInputMessage {
+       return &processorInputMessage{
+               context: subscription.CommitContext{Topic: topic, GroupID: 
"group", DataNodeID: 1, RegionID: region, CommitID: commitID},
+               batches: batches,
+       }
+}
+
+func processorWatermark(topic, region string, timestamp int64) 
*processorInputMessage {
+       message := processorRecord(topic, region, subscription.InvalidCommitID)
+       message.watermark = &timestamp
+       return message
+}
+
+func (m *processorInputMessage) Kind() subscription.SubscriptionMessageKind {
+       if m.watermark != nil {
+               return subscription.WatermarkMessage
+       }
+       return subscription.RecordMessage
+}
+
+func (m *processorInputMessage) CommitContext() subscription.CommitContext {
+       result := m.context
+       if result.Writer != nil {
+               writer := *result.Writer
+               result.Writer = &writer
+       }
+       return result
+}
+
+func (m *processorInputMessage) Records() ([]subscription.RecordBatch, error) {
+       if m.watermark != nil {
+               return nil, subscription.ErrIncompatibleMessage
+       }
+       if m.released {
+               return nil, subscription.ErrDataReleased
+       }
+       result := make([]subscription.RecordBatch, len(m.batches))
+       for index, batch := range m.batches {
+               result[index] = batch
+       }
+       return result, nil
+}
+
+func (m *processorInputMessage) Watermark() (int64, bool) {
+       if m.watermark == nil {
+               return 0, false
+       }
+       return *m.watermark, true
+}
+
+func (m *processorInputMessage) EstimatedSize() int64 { return 64 }
+
+func (m *processorInputMessage) Release() {
+       if m.released {
+               return
+       }
+       m.released = true
+       for _, batch := range m.batches {
+               for _, row := range batch.values {
+                       for _, value := range row {
+                               if binary, ok := value.([]byte); ok {
+                                       clear(binary)
+                               }
+                       }
+               }
+               batch.released = true
+               batch.values = nil
+               batch.timestamps = nil
+       }
+       m.batches = nil
+}
+
+// processorInputBatch stores rows, independently of the processor's column
+// storage. Optional read errors represent failures in a caller-provided 
reader.
+type processorInputBatch struct {
+       dialect subscription.Dialect
+       database string
+       target string
+       columns []subscription.Column
+       timestamps []int64
+       values [][]any
+       timestampErr error
+       valueErr error
+       released bool
+}
+
+func processorTreeBatch(timestamps []int64, values ...any) 
*processorInputBatch {
+       rows := make([][]any, len(timestamps))
+       for row := range rows {
+               var value any
+               if row < len(values) {
+                       value = values[row]
+               }
+               rows[row] = []any{value}
+       }
+       return &processorInputBatch{
+               dialect: subscription.TreeDialect,
+               target: "root.db.device",
+               columns: []subscription.Column{{Name: "value", DataType: 
client.INT64, Category: client.FIELD}},
+               timestamps: timestamps,
+               values: rows,
+       }
+}
+
+func processorTableRecord(topic, database, table string, tags []any, timestamp 
int64, value any) *processorInputMessage {
+       columns := make([]subscription.Column, 0, len(tags)+1)
+       for index := range tags {
+               columns = append(columns, subscription.Column{Name: 
fmt.Sprintf("tag%d", index), DataType: client.STRING, Category: client.TAG})
+       }
+       columns = append(columns, subscription.Column{Name: "value", DataType: 
client.INT64, Category: client.FIELD})
+       row := append([]any(nil), tags...)
+       row = append(row, value)
+       return processorRecord(topic, "region", timestamp, &processorInputBatch{
+               dialect: subscription.TableDialect,
+               database: database,
+               target: table,
+               columns: columns,
+               timestamps: []int64{timestamp},
+               values: [][]any{row},
+       })
+}
+
+func (b *processorInputBatch) Dialect() subscription.Dialect { return 
b.dialect }
+func (b *processorInputBatch) Database() string { return b.database }
+func (b *processorInputBatch) Target() string { return b.target }
+func (b *processorInputBatch) Columns() []subscription.Column { return 
b.columns }
+func (b *processorInputBatch) TimeSelected() bool { return true }
+func (b *processorInputBatch) Len() int { return len(b.timestamps) }
+
+func (b *processorInputBatch) Timestamp(row int) (int64, error) {
+       if b.released {
+               return 0, subscription.ErrDataReleased
+       }
+       if b.timestampErr != nil {
+               return 0, b.timestampErr
+       }
+       if row < 0 || row >= len(b.timestamps) {
+               return 0, fmt.Errorf("input row %d out of range", row)
+       }
+       return b.timestamps[row], nil
+}
+
+func (b *processorInputBatch) Value(row, column int) (any, error) {
+       if b.released {
+               return nil, subscription.ErrDataReleased
+       }
+       if b.valueErr != nil {
+               return nil, b.valueErr
+       }
+       if row < 0 || row >= len(b.values) || column < 0 || column >= 
len(b.columns) {
+               return nil, fmt.Errorf("input cell (%d,%d) out of range", row, 
column)
+       }
+       return b.values[row][column], nil
+}
+
+func (b *processorInputBatch) Rows() iter.Seq2[subscription.Row, error] {
+       return func(yield func(subscription.Row, error) bool) {
+               if b.released {
+                       yield(subscription.Row{}, subscription.ErrDataReleased)
+                       return
+               }
+               order := make([]int, b.Len())
+               for index := range order {
+                       order[index] = index
+               }
+               sort.SliceStable(order, func(i, j int) bool { return 
b.timestamps[order[i]] < b.timestamps[order[j]] })
+               for _, index := range order {
+                       timestamp, err := b.Timestamp(index)
+                       if err != nil {
+                               yield(subscription.Row{}, err)
+                               return
+                       }
+                       values := make([]any, len(b.columns))
+                       for column := range values {
+                               values[column], err = b.Value(index, column)
+                               if err != nil {
+                                       yield(subscription.Row{}, err)
+                                       return
+                               }
+                       }
+                       if !yield(subscription.Row{Timestamp: &timestamp, 
Values: values}, nil) {
+                               return
+                       }
+               }
+       }
+}
+
+func releaseProcessorMessages(messages []subscription.SubscriptionMessage) {
+       for _, message := range messages {
+               message.Release()
+       }
+}
+
+func requireProcessorCommitIDs(t *testing.T, messages 
[]subscription.SubscriptionMessage, expected ...int64) {
+       t.Helper()
+       if len(messages) != len(expected) {
+               t.Fatalf("delivered %d messages, want %d", len(messages), 
len(expected))
+       }
+       for index, message := range messages {
+               if id := message.CommitContext().CommitID; id != 
expected[index] {
+                       t.Fatalf("message %d commit ID = %d, want %d", index, 
id, expected[index])
+               }
+               if _, err := message.Records(); err != nil {
+                       t.Fatalf("delivered message %d is unreadable: %v", 
index, err)
+               }
+       }
+}
+
+func processorOutputValue(t *testing.T, messages 
[]subscription.SubscriptionMessage, row, column int) any {
+       t.Helper()
+       if len(messages) != 1 {
+               t.Fatalf("delivered %d messages, want one", len(messages))
+       }
+       batches, err := messages[0].Records()
+       if err != nil || len(batches) != 1 {
+               t.Fatalf("output batches = %d, error = %v", len(batches), err)
+       }
+       value, err := batches[0].Value(row, column)
+       if err != nil {
+               t.Fatal(err)
+       }
+       return value
+}
diff --git a/subscription/subscription_poll_test.go 
b/subscription/subscription_poll_test.go
new file mode 100644
index 0000000..0fe30ab
--- /dev/null
+++ b/subscription/subscription_poll_test.go
@@ -0,0 +1,347 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package subscription
+
+import (
+       "encoding/binary"
+       "encoding/hex"
+       "errors"
+       "fmt"
+       "reflect"
+       "testing"
+       "time"
+)
+
+// This fixture is transcribed independently from Java 
SubscriptionPollResponse,
+// TabletsPayload and SubscriptionCommitContext.serialize, and from Tablet and
+// MeasurementSchema.serializeTo in TsFile 2.5.1-261023-SNAPSHOT. It does not 
use
+// the Go production encoders. The Tablet contains one INT64 FIELD, with raw
+// timestamp/value pairs (20, 200), (10, 101), (10, 102).
+func javaSubscriptionTabletFrame(t *testing.T) []byte {
+       t.Helper()
+       body, err := hex.DecodeString(
+               "0001" + // TABLETS response.
+               "00000001" + "000000026462" + "00000001" + // One database db, 
one Tablet.
+               "000000066576656e7473" + "00000003" + // Target events, three 
rows.
+               "01" + "00000001" + "01" + // Schemas present, one non-null 
schema.
+               "000000027331" + "020000" + "00000000" + "01" + // s1, 
INT64/PLAIN/UNCOMPRESSED, no properties, FIELD.
+               "01" + "0000000000000014" + "000000000000000a" + 
"000000000000000a" +
+               "00" + "01" + "01" + // No bitmaps, values and column present.
+               "00000000000000c8" + "0000000000000065" + "0000000000000066" +
+               "ffffffff" + // Final offset: one Tablet, rather than three 
rows.
+               "02" + "00000007" + "00000002" + // Context version, node, 
restart count.
+               "00000005746f706963" + "0000000567726f7570" + // topic, group.
+               "000000000000002a" + "0000000000000003" + // Commit ID 42, seek 
generation 3.
+               "0000000172" + "0000000000000064" + "00" + // Region r, 
physical time 100, no writer.
+               "01" + "00000000") // Time selected, no per-table overrides.
+       if err != nil {
+               t.Fatal(err)
+       }
+       return body
+}
+
+func decodedJavaSubscriptionMessage(t *testing.T, body []byte, dialect 
Dialect) SubscriptionMessage {
+       t.Helper()
+       response, err := decodeSubscriptionPollResponse(body)
+       if err != nil {
+               t.Fatalf("decode Java-format frame: %v", err)
+       }
+       message, err := response.message(dialect)
+       if err != nil || message == nil {
+               t.Fatalf("construct record message: message=%v, error=%v", 
message, err)
+       }
+       return message
+}
+
+func TestSubscriptionPollRejectsFileBeforeSiblingDecode(t *testing.T) {
+       for _, kind := range []byte{2, 3, 4} {
+               for _, fileFirst := range []bool{false, true} {
+                       for _, malformedSibling := range []bool{false, true} {
+                               
t.Run(fmt.Sprintf("kind=%d/first=%t/malformed=%t", kind, fileFirst, 
malformedSibling), func(t *testing.T) {
+                                       sibling := 
javaSubscriptionTabletFrame(t)
+                                       if malformedSibling {
+                                               sibling = sibling[:1]
+                                       }
+                                       // A tag alone must reject files 
without attempting file I/O.
+                                       file := []byte{0, kind}
+                                       bodies := [][]byte{sibling, file}
+                                       if fileFirst {
+                                               bodies = [][]byte{file, sibling}
+                                       }
+                                       responses, err := 
decodeSubscriptionPollResponses(bodies)
+                                       if !errors.Is(err, 
ErrUnsupportedTsFile) || responses != nil {
+                                               t.Fatalf("file response 
returned partial data or wrong error: %v, %v", responses, err)
+                                       }
+                               })
+                       }
+               }
+       }
+}
+
+func TestSubscriptionPollRejectsMalformedJavaFramesWithoutPartialResults(t 
*testing.T) {
+       valid := javaSubscriptionTabletFrame(t)
+       cases := map[string][]byte{}
+       cases["trailing-byte"] = append(append([]byte(nil), valid...), 0)
+       // Mandatory fields and the optional projection-map header must be 
complete.
+       // Ending exactly after the context or the global flag is valid in 
older Java clients.
+       for _, length := range []int{1, 5, 9, 29, 41, 60, 100, 105, 106, 120, 
148, 162, 165} {
+               cases[fmt.Sprintf("truncated-%d", length)] = valid[:length]
+       }
+       for _, field := range []struct {
+               name string
+               offset int
+               value uint32
+       }{
+               {"negative-database-count", 2, 0xffffffff},
+               {"invalid-database-name-length", 6, 0xfffffffe},
+               {"negative-tablet-count", 12, 0xffffffff},
+               {"negative-row-count", 26, 0xffffffff},
+               {"oversized-column-count", 31, 0x7fffffff},
+               {"invalid-column-name-length", 36, 0xfffffffe},
+       } {
+               body := append([]byte(nil), valid...)
+               binary.BigEndian.PutUint32(body[field.offset:field.offset+4], 
field.value)
+               cases[field.name] = body
+       }
+       for name, malformed := range cases {
+               t.Run(name, func(t *testing.T) {
+                       responses, err := 
decodeSubscriptionPollResponses([][]byte{valid, malformed})
+                       if err == nil || responses != nil {
+                               t.Fatalf("malformed sibling returned partial 
results: %v, %v", responses, err)
+                       }
+               })
+       }
+}
+
+func TestSubscriptionPollAcceptsLegacyProjectionTails(t *testing.T) {
+       for _, length := range []int{163, 164} { // Context only, or context 
and global flag.
+               t.Run(fmt.Sprintf("frame-length-%d", length), func(t 
*testing.T) {
+                       message := decodedJavaSubscriptionMessage(t, 
javaSubscriptionTabletFrame(t)[:length], TableDialect)
+                       defer message.Release()
+                       batches, err := message.Records()
+                       if err != nil || len(batches) != 1 {
+                               t.Fatalf("legacy record batches: %v, %v", 
batches, err)
+                       }
+                       rows := 0
+                       for row, err := range batches[0].Rows() {
+                               if err != nil || row.Timestamp == nil {
+                                       t.Fatalf("legacy response lost 
projected timestamps: %v, %v", row, err)
+                               }
+                               rows++
+                       }
+                       if rows != 3 {
+                               t.Fatalf("legacy response yielded %d rows, want 
3", rows)
+                       }
+               })
+       }
+}
+
+func TestSubscriptionDecodedRowsKeepStableOrderAndReleaseBorrowedData(t 
*testing.T) {
+       message := decodedJavaSubscriptionMessage(t, 
javaSubscriptionTabletFrame(t), TableDialect)
+       batches, err := message.Records()
+       if err != nil || len(batches) != 1 {
+               t.Fatalf("record batches: %v, %v", batches, err)
+       }
+       batch := batches[0]
+       var timestamps, values []int64
+       for row, err := range batch.Rows() {
+               if err != nil || row.Timestamp == nil || len(row.Values) != 1 {
+                       t.Fatalf("row: %v, %v", row, err)
+               }
+               timestamps = append(timestamps, *row.Timestamp)
+               value, ok := row.Values[0].(int64)
+               if !ok {
+                       t.Fatalf("INT64 value has type %T", row.Values[0])
+               }
+               values = append(values, value)
+       }
+       if !reflect.DeepEqual(timestamps, []int64{10, 10, 20}) || 
!reflect.DeepEqual(values, []int64{101, 102, 200}) {
+               t.Fatalf("unstable user-visible row ordering: timestamps=%v, 
values=%v", timestamps, values)
+       }
+       for index, want := range []int64{20, 10, 10} {
+               got, err := batch.Timestamp(index)
+               if err != nil || got != want {
+                       t.Fatalf("raw timestamp %d was reordered: %d, %v", 
index, got, err)
+               }
+               value, err := batch.Value(index, 0)
+               if err != nil || value != []int64{200, 101, 102}[index] {
+                       t.Fatalf("raw value %d was reordered: %v, %v", index, 
value, err)
+               }
+       }
+       context := message.CommitContext()
+       message.Release()
+       message.Release()
+       if _, err := message.Records(); !errors.Is(err, ErrDataReleased) {
+               t.Fatalf("records after release: %v", err)
+       }
+       if _, err := batch.Timestamp(0); !errors.Is(err, ErrDataReleased) {
+               t.Fatalf("borrowed timestamp after release: %v", err)
+       }
+       if _, err := batch.Value(0, 0); !errors.Is(err, ErrDataReleased) {
+               t.Fatalf("borrowed value after release: %v", err)
+       }
+       yields := 0
+       for _, err := range batch.Rows() {
+               yields++
+               if !errors.Is(err, ErrDataReleased) {
+                       t.Fatalf("iteration after release: %v", err)
+               }
+       }
+       if yields != 1 {
+               t.Fatalf("released iterator yielded %d times", yields)
+       }
+       if message.Kind() != RecordMessage || message.CommitContext() != 
context || context.CommitID != 42 || context.SeekGeneration != 3 {
+               t.Fatal("release discarded the delivery identity required for 
later acknowledgment")
+       }
+}
+
+func TestSubscriptionReleaseInsideRowCallbackStopsIteration(t *testing.T) {
+       message := decodedJavaSubscriptionMessage(t, 
javaSubscriptionTabletFrame(t), TableDialect)
+       batches, err := message.Records()
+       if err != nil || len(batches) != 1 {
+               t.Fatalf("record batches: %v, %v", batches, err)
+       }
+       finished := make(chan error, 1)
+       go func() {
+               rows, released := 0, 0
+               for _, err := range batches[0].Rows() {
+                       if err != nil {
+                               if !errors.Is(err, ErrDataReleased) {
+                                       finished <- err
+                                       return
+                               }
+                               released++
+                               continue
+                       }
+                       rows++
+                       message.Release()
+               }
+               if rows != 1 || released != 1 {
+                       finished <- fmt.Errorf("iteration after callback 
release: rows=%d, errors=%d", rows, released)
+                       return
+               }
+               finished <- nil
+       }()
+       select {
+       case err := <-finished:
+               if err != nil {
+                       t.Fatal(err)
+               }
+       case <-time.After(5 * time.Second):
+               t.Fatal("Release deadlocked inside the row callback")
+       }
+}
+
+func TestSubscriptionContinuationRejectsDifferentDeliveryWithoutLeakingRows(t 
*testing.T) {
+       for _, changedOffset := range []int{133, 141} { // Commit ID and seek 
generation in the Java fixture.
+               t.Run(fmt.Sprintf("context-offset-%d", changedOffset), func(t 
*testing.T) {
+                       firstBody := javaSubscriptionTabletFrame(t)
+                       binary.BigEndian.PutUint32(firstBody[102:106], 1) // 
Request continuation at offset 1.
+                       first, err := decodeSubscriptionPollResponse(firstBody)
+                       if err != nil {
+                               t.Fatal(err)
+                       }
+                       if message, err := first.message(TableDialect); err == 
nil || message != nil {
+                               t.Fatal("incomplete delivery exposed rows 
before continuation")
+                       }
+                       lastBody := javaSubscriptionTabletFrame(t)
+                       binary.BigEndian.PutUint32(lastBody[102:106], 
0xfffffffe) // Two Tablets total.
+                       badBody := append([]byte(nil), lastBody...)
+                       
binary.BigEndian.PutUint64(badBody[changedOffset:changedOffset+8], 999)
+                       bad, err := decodeSubscriptionPollResponse(badBody)
+                       if err != nil {
+                               t.Fatal(err)
+                       }
+                       if err := first.appendTablets(bad); err == nil {
+                               t.Fatal("accepted a continuation from another 
delivery")
+                       }
+                       last, err := decodeSubscriptionPollResponse(lastBody)
+                       if err != nil {
+                               t.Fatal(err)
+                       }
+                       if err := first.appendTablets(last); err != nil {
+                               t.Fatalf("rejected continuation corrupted the 
current delivery: %v", err)
+                       }
+                       message, err := first.message(TableDialect)
+                       if err != nil || message == nil {
+                               t.Fatalf("completed delivery: %v, %v", message, 
err)
+                       }
+                       defer message.Release()
+                       batches, err := message.Records()
+                       if err != nil || len(batches) != 2 {
+                               t.Fatalf("rejected continuation leaked records: 
%v, %v", batches, err)
+                       }
+                       rows := 0
+                       for _, batch := range batches {
+                               for _, err := range batch.Rows() {
+                                       if err != nil {
+                                               t.Fatal(err)
+                                       }
+                                       rows++
+                               }
+                       }
+                       if rows != 6 {
+                               t.Fatalf("completed delivery contains %d rows, 
want 6", rows)
+                       }
+               })
+       }
+}
+
+func TestSubscriptionProjectionChangesVisibleRowsByModel(t *testing.T) {
+       for _, dialect := range []Dialect{TreeDialect, TableDialect} {
+               for _, override := range []bool{false, true} {
+                       t.Run(fmt.Sprintf("%s/override=%t", dialect, override), 
func(t *testing.T) {
+                               body := javaSubscriptionTabletFrame(t)
+                               body[163] = 0 // Global time projection 
excludes timestamps.
+                               if override {
+                                       // One database/table override: 
db.events selects time.
+                                       tail, err := 
hex.DecodeString("0000000100000002646200000001000000066576656e747301")
+                                       if err != nil {
+                                               t.Fatal(err)
+                                       }
+                                       body = append(body[:164], tail...)
+                               }
+                               message := decodedJavaSubscriptionMessage(t, 
body, dialect)
+                               defer message.Release()
+                               batches, err := message.Records()
+                               if err != nil || len(batches) != 1 {
+                                       t.Fatalf("record batches: %v, %v", 
batches, err)
+                               }
+                               rows := 0
+                               for row, err := range batches[0].Rows() {
+                                       if err != nil {
+                                               t.Fatal(err)
+                                       }
+                                       wantTime := dialect == TreeDialect || 
override
+                                       if (row.Timestamp != nil) != wantTime {
+                                               t.Fatalf("projection exposed 
wrong timestamp: %v", row.Timestamp)
+                                       }
+                                       rows++
+                               }
+                               if rows != 3 {
+                                       t.Fatalf("projection changed row count: 
%d", rows)
+                               }
+                               if timestamp, err := batches[0].Timestamp(0); 
err != nil || timestamp != 20 {
+                                       t.Fatalf("projection hid raw processing 
timestamp: %d, %v", timestamp, err)
+                               }
+                       })
+               }
+       }
+}
diff --git a/subscription/watermark_processor_test.go 
b/subscription/watermark_processor_test.go
new file mode 100644
index 0000000..eeb222e
--- /dev/null
+++ b/subscription/watermark_processor_test.go
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package subscription_test
+
+import (
+       "errors"
+       "math"
+       "testing"
+       "time"
+
+       "github.com/apache/iotdb-client-go/subscription"
+)
+
+func newTestWatermarkProcessor(t *testing.T, config 
subscription.WatermarkProcessorConfig) *subscription.WatermarkProcessor {
+       t.Helper()
+       processor, err := subscription.NewWatermarkProcessor(config)
+       if err != nil {
+               t.Fatal(err)
+       }
+       t.Cleanup(processor.Reset)
+       return processor
+}
+
+func TestWatermarkOrdersWholeMessagesAfterEveryRegionAdvances(t *testing.T) {
+       processor := newTestWatermarkProcessor(t, 
subscription.WatermarkProcessorConfig{
+               MaxOutOfOrderness: 10, Timeout: time.Hour, StaleRegionTimeout: 
-1,
+       })
+       high := processorRecord("topic", "a", 1, 
processorTreeBatch([]int64{200}))
+       low := processorRecord("topic", "b", 2, processorTreeBatch([]int64{50}))
+       middle := processorRecord("topic", "a", 3, 
processorTreeBatch([]int64{100, 1}))
+       high.context.CommitID = subscription.InvalidCommitID
+       high.context.Writer = &subscription.WriterPosition{ID: 
subscription.WriterID{RegionID: "a", NodeID: 7}, Progress: 
subscription.WriterProgress{PhysicalTime: 200, LocalSeq: 1}}
+       messages, err := 
processor.Process([]subscription.SubscriptionMessage{high, low, middle})
+       if err != nil || len(messages) != 0 {
+               t.Fatalf("premature delivery: %d messages, error=%v", 
len(messages), err)
+       }
+       buffer := processor.Buffered()
+       if buffer.Count != 3 || len(buffer.Contexts) != 3 {
+               t.Fatalf("renewal snapshot lost buffered deliveries: %+v", 
buffer)
+       }
+       legacy := make(map[int64]bool)
+       writerRenewed := false
+       for _, context := range buffer.Contexts {
+               if context.Writer != nil {
+                       writerRenewed = context.Writer.ID.NodeID == 7 && 
context.Writer.Progress.LocalSeq == 1
+               } else {
+                       legacy[context.CommitID] = true
+               }
+       }
+       if !writerRenewed || !legacy[2] || !legacy[3] {
+               t.Fatalf("renewal snapshot omitted an independent delivery: 
%+v", buffer.Contexts)
+       }
+       messages, err = 
processor.Process([]subscription.SubscriptionMessage{processorWatermark("topic",
 "b", 250)})
+       if err != nil {
+               t.Fatal(err)
+       }
+       requireProcessorCommitIDs(t, messages, 2, 3)
+       batches, err := messages[1].Records()
+       if err != nil || len(batches) != 1 || batches[0].Len() != 2 {
+               t.Fatalf("message was split instead of emitted whole: 
batches=%v error=%v", batches, err)
+       }
+       releaseProcessorMessages(messages)
+       if buffer = processor.Buffered(); buffer.Count != 1 || 
len(buffer.Contexts) != 1 || buffer.Contexts[0].Writer == nil {
+               t.Fatalf("remaining writer delivery is not renewable: %+v", 
buffer)
+       }
+       messages, err = 
processor.Process([]subscription.SubscriptionMessage{processorWatermark("topic",
 "a", 300)})
+       if err != nil || len(messages) != 1 || messages[0] != high {
+               t.Fatalf("high message not released after all sources advanced: 
messages=%v error=%v", messages, err)
+       }
+       releaseProcessorMessages(messages)
+}
+
+func TestWatermarkResetTopicDropsOnlyItsRecordsAndOrderingSource(t *testing.T) 
{
+       processor := newTestWatermarkProcessor(t, 
subscription.WatermarkProcessorConfig{
+               MaxOutOfOrderness: 10, Timeout: time.Hour, StaleRegionTimeout: 
-1,
+       })
+       a := processorRecord("a", "region", 1, processorTreeBatch([]int64{20}))
+       b := processorRecord("b", "region", 2, processorTreeBatch([]int64{100}))
+       messages, err := 
processor.Process([]subscription.SubscriptionMessage{a, b})
+       if err != nil || len(messages) != 0 {
+               t.Fatalf("initial delivery=%v error=%v", messages, err)
+       }
+       processor.ResetTopic("a")
+       if _, err := a.Records(); !errors.Is(err, subscription.ErrDataReleased) 
{
+               t.Fatalf("reset topic retained its borrowed data: %v", err)
+       }
+       if _, err := b.Records(); err != nil {
+               t.Fatalf("other topic's record was released: %v", err)
+       }
+       buffer := processor.Buffered()
+       if buffer.Count != 1 || len(buffer.Contexts) != 1 || 
buffer.Contexts[0].Topic != "b" {
+               t.Fatalf("reset corrupted other topic's renewal: %+v", buffer)
+       }
+       messages, err = 
processor.Process([]subscription.SubscriptionMessage{processorWatermark("b", 
"region", 120)})
+       if err != nil {
+               t.Fatal(err)
+       }
+       requireProcessorCommitIDs(t, messages, 2)
+       releaseProcessorMessages(messages)
+}
+
+func TestWatermarkForceFlushDoesNotWaitForRegionProgress(t *testing.T) {
+       for _, test := range []struct {
+               name string
+               timeout time.Duration
+               limit int64
+       }{
+               {name: "capacity", timeout: time.Hour, limit: 1},
+               {name: "zero_timeout", timeout: 0},
+       } {
+               t.Run(test.name, func(t *testing.T) {
+                       processor := newTestWatermarkProcessor(t, 
subscription.WatermarkProcessorConfig{
+                               MaxOutOfOrderness: 1000, Timeout: test.timeout, 
MaxBufferBytes: test.limit, StaleRegionTimeout: -1,
+                       })
+                       high := processorRecord("topic", "region", 2, 
processorTreeBatch([]int64{20}))
+                       low := processorRecord("topic", "region", 1, 
processorTreeBatch([]int64{10}))
+                       messages, err := 
processor.Process([]subscription.SubscriptionMessage{high, low})
+                       if err != nil {
+                               t.Fatal(err)
+                       }
+                       requireProcessorCommitIDs(t, messages, 1, 2)
+                       releaseProcessorMessages(messages)
+                       if buffer := processor.Buffered(); buffer.Count != 0 || 
len(buffer.Contexts) != 0 {
+                               t.Fatalf("flushed records remain renewable: 
%+v", buffer)
+                       }
+               })
+       }
+}
+
+func TestWatermarkTimestampSubtractionCannotWrapAndDeliverEarly(t *testing.T) {
+       processor := newTestWatermarkProcessor(t, 
subscription.WatermarkProcessorConfig{
+               MaxOutOfOrderness: 10, Timeout: time.Hour, StaleRegionTimeout: 
-1,
+       })
+       _, err := 
processor.Process([]subscription.SubscriptionMessage{processorWatermark("topic",
 "region", math.MinInt64+5)})
+       if err != nil {
+               t.Fatal(err)
+       }
+       record := processorRecord("topic", "region", 1, 
processorTreeBatch([]int64{math.MinInt64+1}))
+       messages, err := 
processor.Process([]subscription.SubscriptionMessage{record})
+       if err != nil || len(messages) != 0 {
+               t.Fatalf("underflow delivered an unready message: messages=%v 
error=%v", messages, err)
+       }
+       messages, err = processor.Flush()
+       if err != nil {
+               t.Fatal(err)
+       }
+       requireProcessorCommitIDs(t, messages, 1)
+       releaseProcessorMessages(messages)
+}
+
+func TestWatermarkReadFailureDoesNotAdvanceOrTakeOwnership(t *testing.T) {
+       processor := newTestWatermarkProcessor(t, 
subscription.WatermarkProcessorConfig{
+               MaxOutOfOrderness: 10, Timeout: time.Hour, StaleRegionTimeout: 
-1,
+       })
+       seed := processorRecord("topic", "a", 1, 
processorTreeBatch([]int64{100}))
+       if messages, err := 
processor.Process([]subscription.SubscriptionMessage{seed}); err != nil || 
len(messages) != 0 {
+               t.Fatalf("initial delivery=%v error=%v", messages, err)
+       }
+       readFailure := errors.New("input timestamp unavailable")
+       brokenBatch := processorTreeBatch([]int64{600})
+       brokenBatch.timestampErr = readFailure
+       broken := processorRecord("topic", "a", 3, brokenBatch)
+       good := processorRecord("topic", "a", 2, 
processorTreeBatch([]int64{500}))
+       messages, err := processor.Process([]subscription.SubscriptionMessage{
+               processorWatermark("topic", "b", 1000), good, broken,
+       })
+       if !errors.Is(err, readFailure) || len(messages) != 0 {
+               t.Fatalf("failed input delivered data or lost error: 
messages=%v error=%v", messages, err)
+       }
+       for _, input := range []*processorInputMessage{good, broken} {
+               if _, err := input.Records(); err != nil {
+                       t.Fatalf("failed round consumed caller-owned input: 
%v", err)
+               }
+               defer input.Release()
+       }
+       next := processorRecord("topic", "a", 4, 
processorTreeBatch([]int64{101}))
+       messages, err = 
processor.Process([]subscription.SubscriptionMessage{next})
+       if err != nil || len(messages) != 0 {
+               t.Fatalf("failed round advanced ordering: messages=%v 
error=%v", messages, err)
+       }
+       messages, err = processor.Flush()
+       if err != nil {
+               t.Fatal(err)
+       }
+       requireProcessorCommitIDs(t, messages, 1, 4)
+       releaseProcessorMessages(messages)
+}

Reply via email to