This is an automated email from the ASF dual-hosted git repository. shuwenwei pushed a commit to branch supports_subscription in repository https://gitbox.apache.org/repos/asf/iotdb-client-go.git
commit cc734d4508a39007d9a5d22428c100d153caa1f7 Author: shuwenwei <[email protected]> AuthorDate: Fri Nov 13 12:43:36 2026 +0800 Add subscription protocol and processor behavior unit tests --- subscription/column_align_processor_test.go | 215 +++++++++++++++++ subscription/processor_fixture_test.go | 263 +++++++++++++++++++++ subscription/subscription_poll_test.go | 347 ++++++++++++++++++++++++++++ subscription/watermark_processor_test.go | 204 ++++++++++++++++ 4 files changed, 1029 insertions(+) diff --git a/subscription/column_align_processor_test.go b/subscription/column_align_processor_test.go new file mode 100644 index 0000000..7a63c4a --- /dev/null +++ b/subscription/column_align_processor_test.go @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package subscription_test + +import ( + "bytes" + "errors" + "testing" + + "github.com/apache/iotdb-client-go/subscription" + "github.com/apache/iotdb-client-go/v2/client" +) + +func newTestColumnAlignProcessor(t *testing.T, dialect subscription.Dialect) *subscription.ColumnAlignProcessor { + t.Helper() + processor, err := subscription.NewColumnAlignProcessor(dialect) + if err != nil { + t.Fatal(err) + } + t.Cleanup(processor.Reset) + return processor +} + +func processAlignment(t *testing.T, processor *subscription.ColumnAlignProcessor, input ...subscription.SubscriptionMessage) []subscription.SubscriptionMessage { + t.Helper() + messages, err := processor.Process(input) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { releaseProcessorMessages(messages) }) + return messages +} + +func TestColumnAlignNeverFillsFromEqualOrFutureTimestamps(t *testing.T) { + processor := newTestColumnAlignProcessor(t, subscription.TreeDialect) + seed := processorRecord("topic", "region", 1, processorTreeBatch([]int64{10}, int64(100))) + processAlignment(t, processor, seed) + input := processorRecord("topic", "region", 2, processorTreeBatch( + []int64{10, 5, 6, 11, 12, 13}, nil, nil, int64(60), nil, int64(120), nil, + )) + messages := processAlignment(t, processor, input) + for row, expected := range []any{nil, nil, int64(60), int64(100), int64(120), int64(120)} { + if value := processorOutputValue(t, messages, row, 0); value != expected { + t.Fatalf("raw row %d value=%v, want %v", row, value, expected) + } + } + if _, err := input.Records(); !errors.Is(err, subscription.ErrDataReleased) { + t.Fatalf("consumed input was not released: %v", err) + } +} + +func TestColumnAlignMatchesColumnNamesAfterSchemaReordering(t *testing.T) { + processor := newTestColumnAlignProcessor(t, subscription.TreeDialect) + seed := processorTreeBatch([]int64{1}) + seed.columns = []subscription.Column{ + {Name: "left", DataType: client.INT64, Category: client.FIELD}, + {Name: "right", DataType: client.INT64, Category: client.FIELD}, + } + seed.values = [][]any{{int64(11), int64(22)}} + processAlignment(t, processor, processorRecord("topic", "region", 1, seed)) + input := processorTreeBatch([]int64{2}) + input.columns = []subscription.Column{ + {Name: "right", DataType: client.INT64, Category: client.FIELD}, + {Name: "left", DataType: client.INT64, Category: client.FIELD}, + {Name: "new", DataType: client.INT64, Category: client.FIELD}, + } + input.values = [][]any{{nil, nil, nil}} + messages := processAlignment(t, processor, processorRecord("topic", "region", 2, input)) + for column, expected := range []any{int64(22), int64(11), nil} { + if value := processorOutputValue(t, messages, 0, column); value != expected { + t.Fatalf("column %d reused another column's value: got=%v want=%v", column, value, expected) + } + } +} + +func TestColumnAlignTableIdentityCannotLeakValuesAcrossDevices(t *testing.T) { + for _, test := range []struct { + name string + topic string + database string + table string + seedTags []any + otherTags []any + }{ + {name: "topic", topic: "other", database: "db", table: "table", seedTags: []any{"a"}, otherTags: []any{"a"}}, + {name: "database", topic: "topic", database: "other", table: "table", seedTags: []any{"a"}, otherTags: []any{"a"}}, + {name: "table", topic: "topic", database: "db", table: "other", seedTags: []any{"a"}, otherTags: []any{"a"}}, + {name: "tag", topic: "topic", database: "db", table: "table", seedTags: []any{"a"}, otherTags: []any{"b"}}, + {name: "null_and_empty", topic: "topic", database: "db", table: "table", seedTags: []any{nil, "x"}, otherTags: []any{"", "x"}}, + {name: "embedded_delimiters", topic: "topic", database: "db", table: "table", seedTags: []any{"a|b", "c"}, otherTags: []any{"a", "b|c"}}, + } { + t.Run(test.name, func(t *testing.T) { + processor := newTestColumnAlignProcessor(t, subscription.TableDialect) + processAlignment(t, processor, processorTableRecord("topic", "db", "table", test.seedTags, 1, int64(101))) + other := processAlignment(t, processor, processorTableRecord(test.topic, test.database, test.table, test.otherTags, 2, nil)) + if value := processorOutputValue(t, other, 0, len(test.otherTags)); value != nil { + t.Fatalf("another device inherited the seed value: %v", value) + } + same := processAlignment(t, processor, processorTableRecord("topic", "db", "table", test.seedTags, 3, nil)) + if value := processorOutputValue(t, same, 0, len(test.seedTags)); value != int64(101) { + t.Fatalf("same device lost its value: %v", value) + } + }) + } +} + +func TestColumnAlignTableIdentityIgnoresTrailingNullTags(t *testing.T) { + for _, test := range []struct { + name string + seedTags []any + nextTags []any + }{ + {name: "trailing_null", seedTags: []any{"device"}, nextTags: []any{"device", nil}}, + {name: "no_tags_and_all_null", seedTags: nil, nextTags: []any{nil, nil}}, + } { + t.Run(test.name, func(t *testing.T) { + processor := newTestColumnAlignProcessor(t, subscription.TableDialect) + processAlignment(t, processor, processorTableRecord("topic", "db", "table", test.seedTags, 1, int64(101))) + messages := processAlignment(t, processor, processorTableRecord("topic", "db", "table", test.nextTags, 2, nil)) + if value := processorOutputValue(t, messages, 0, len(test.nextTags)); value != int64(101) { + t.Fatalf("equivalent device identity did not fill its field: %v", value) + } + }) + } +} + +func TestColumnAlignResetTopicDoesNotDiscardOtherTopicValues(t *testing.T) { + processor := newTestColumnAlignProcessor(t, subscription.TreeDialect) + processAlignment(t, processor, + processorRecord("a", "region", 1, processorTreeBatch([]int64{10}, int64(11))), + processorRecord("b", "region", 2, processorTreeBatch([]int64{10}, int64(22))), + ) + processor.ResetTopic("a") + a := processAlignment(t, processor, processorRecord("a", "region", 3, processorTreeBatch([]int64{20}))) + if value := processorOutputValue(t, a, 0, 0); value != nil { + t.Fatalf("reset topic reused its obsolete cached value: %v", value) + } + b := processAlignment(t, processor, processorRecord("b", "region", 4, processorTreeBatch([]int64{20}))) + if value := processorOutputValue(t, b, 0, 0); value != int64(22) { + t.Fatalf("reset discarded another topic's value: %v", value) + } +} + +func TestColumnAlignCopiedBytesSurviveInputAndOutputRelease(t *testing.T) { + processor := newTestColumnAlignProcessor(t, subscription.TreeDialect) + seedBatch := processorTreeBatch([]int64{1}, []byte{1, 2, 3}) + seedBatch.columns[0].DataType = client.BLOB + seed := processorRecord("topic", "region", 1, seedBatch) + messages := processAlignment(t, processor, seed) + if _, err := seed.Records(); !errors.Is(err, subscription.ErrDataReleased) { + t.Fatalf("input storage was not reclaimed: %v", err) + } + if value, ok := processorOutputValue(t, messages, 0, 0).([]byte); !ok || !bytes.Equal(value, []byte{1, 2, 3}) { + t.Fatalf("output borrowed storage reclaimed by input Release: %v", value) + } + releaseProcessorMessages(messages) + for timestamp := int64(2); timestamp <= 3; timestamp++ { + batch := processorTreeBatch([]int64{timestamp}) + batch.columns[0].DataType = client.BLOB + messages = processAlignment(t, processor, processorRecord("topic", "region", timestamp, batch)) + if value, ok := processorOutputValue(t, messages, 0, 0).([]byte); !ok || !bytes.Equal(value, []byte{1, 2, 3}) { + t.Fatalf("cached bytes failed after earlier Release: %v", value) + } + releaseProcessorMessages(messages) + } +} + +func TestColumnAlignFailedInputCannotPolluteEarlierValues(t *testing.T) { + for _, failure := range []string{"dialect", "read"} { + t.Run(failure, func(t *testing.T) { + processor := newTestColumnAlignProcessor(t, subscription.TreeDialect) + processAlignment(t, processor, processorRecord("topic", "region", 1, processorTreeBatch([]int64{10}, int64(11)))) + good := processorRecord("topic", "region", 2, processorTreeBatch([]int64{20}, int64(99))) + brokenBatch := processorTreeBatch([]int64{21}, int64(123)) + readFailure := errors.New("input value unavailable") + if failure == "dialect" { + brokenBatch.dialect = subscription.TableDialect + } else { + brokenBatch.valueErr = readFailure + } + broken := processorRecord("topic", "region", 3, brokenBatch) + messages, err := processor.Process([]subscription.SubscriptionMessage{good, broken}) + if err == nil || len(messages) != 0 || (failure == "read" && !errors.Is(err, readFailure)) { + t.Fatalf("failed round delivered data or lost the read failure: messages=%v error=%v", messages, err) + } + for _, input := range []*processorInputMessage{good, broken} { + if _, err := input.Records(); err != nil { + t.Fatalf("failed round consumed caller-owned input: %v", err) + } + defer input.Release() + } + messages = processAlignment(t, processor, processorRecord("topic", "region", 4, processorTreeBatch([]int64{30}))) + if value := processorOutputValue(t, messages, 0, 0); value != int64(11) { + t.Fatalf("failed round changed the cache: %v", value) + } + }) + } +} diff --git a/subscription/processor_fixture_test.go b/subscription/processor_fixture_test.go new file mode 100644 index 0000000..aae4ab1 --- /dev/null +++ b/subscription/processor_fixture_test.go @@ -0,0 +1,263 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package subscription_test + +import ( + "fmt" + "iter" + "sort" + "testing" + + "github.com/apache/iotdb-client-go/subscription" + "github.com/apache/iotdb-client-go/v2/client" +) + +// processorInputMessage is a caller-owned implementation of the public message +// contract. Release reclaims its borrowed buffers to exercise ownership transfer. +type processorInputMessage struct { + context subscription.CommitContext + batches []*processorInputBatch + watermark *int64 + released bool +} + +func processorRecord(topic, region string, commitID int64, batches ...*processorInputBatch) *processorInputMessage { + return &processorInputMessage{ + context: subscription.CommitContext{Topic: topic, GroupID: "group", DataNodeID: 1, RegionID: region, CommitID: commitID}, + batches: batches, + } +} + +func processorWatermark(topic, region string, timestamp int64) *processorInputMessage { + message := processorRecord(topic, region, subscription.InvalidCommitID) + message.watermark = ×tamp + return message +} + +func (m *processorInputMessage) Kind() subscription.SubscriptionMessageKind { + if m.watermark != nil { + return subscription.WatermarkMessage + } + return subscription.RecordMessage +} + +func (m *processorInputMessage) CommitContext() subscription.CommitContext { + result := m.context + if result.Writer != nil { + writer := *result.Writer + result.Writer = &writer + } + return result +} + +func (m *processorInputMessage) Records() ([]subscription.RecordBatch, error) { + if m.watermark != nil { + return nil, subscription.ErrIncompatibleMessage + } + if m.released { + return nil, subscription.ErrDataReleased + } + result := make([]subscription.RecordBatch, len(m.batches)) + for index, batch := range m.batches { + result[index] = batch + } + return result, nil +} + +func (m *processorInputMessage) Watermark() (int64, bool) { + if m.watermark == nil { + return 0, false + } + return *m.watermark, true +} + +func (m *processorInputMessage) EstimatedSize() int64 { return 64 } + +func (m *processorInputMessage) Release() { + if m.released { + return + } + m.released = true + for _, batch := range m.batches { + for _, row := range batch.values { + for _, value := range row { + if binary, ok := value.([]byte); ok { + clear(binary) + } + } + } + batch.released = true + batch.values = nil + batch.timestamps = nil + } + m.batches = nil +} + +// processorInputBatch stores rows, independently of the processor's column +// storage. Optional read errors represent failures in a caller-provided reader. +type processorInputBatch struct { + dialect subscription.Dialect + database string + target string + columns []subscription.Column + timestamps []int64 + values [][]any + timestampErr error + valueErr error + released bool +} + +func processorTreeBatch(timestamps []int64, values ...any) *processorInputBatch { + rows := make([][]any, len(timestamps)) + for row := range rows { + var value any + if row < len(values) { + value = values[row] + } + rows[row] = []any{value} + } + return &processorInputBatch{ + dialect: subscription.TreeDialect, + target: "root.db.device", + columns: []subscription.Column{{Name: "value", DataType: client.INT64, Category: client.FIELD}}, + timestamps: timestamps, + values: rows, + } +} + +func processorTableRecord(topic, database, table string, tags []any, timestamp int64, value any) *processorInputMessage { + columns := make([]subscription.Column, 0, len(tags)+1) + for index := range tags { + columns = append(columns, subscription.Column{Name: fmt.Sprintf("tag%d", index), DataType: client.STRING, Category: client.TAG}) + } + columns = append(columns, subscription.Column{Name: "value", DataType: client.INT64, Category: client.FIELD}) + row := append([]any(nil), tags...) + row = append(row, value) + return processorRecord(topic, "region", timestamp, &processorInputBatch{ + dialect: subscription.TableDialect, + database: database, + target: table, + columns: columns, + timestamps: []int64{timestamp}, + values: [][]any{row}, + }) +} + +func (b *processorInputBatch) Dialect() subscription.Dialect { return b.dialect } +func (b *processorInputBatch) Database() string { return b.database } +func (b *processorInputBatch) Target() string { return b.target } +func (b *processorInputBatch) Columns() []subscription.Column { return b.columns } +func (b *processorInputBatch) TimeSelected() bool { return true } +func (b *processorInputBatch) Len() int { return len(b.timestamps) } + +func (b *processorInputBatch) Timestamp(row int) (int64, error) { + if b.released { + return 0, subscription.ErrDataReleased + } + if b.timestampErr != nil { + return 0, b.timestampErr + } + if row < 0 || row >= len(b.timestamps) { + return 0, fmt.Errorf("input row %d out of range", row) + } + return b.timestamps[row], nil +} + +func (b *processorInputBatch) Value(row, column int) (any, error) { + if b.released { + return nil, subscription.ErrDataReleased + } + if b.valueErr != nil { + return nil, b.valueErr + } + if row < 0 || row >= len(b.values) || column < 0 || column >= len(b.columns) { + return nil, fmt.Errorf("input cell (%d,%d) out of range", row, column) + } + return b.values[row][column], nil +} + +func (b *processorInputBatch) Rows() iter.Seq2[subscription.Row, error] { + return func(yield func(subscription.Row, error) bool) { + if b.released { + yield(subscription.Row{}, subscription.ErrDataReleased) + return + } + order := make([]int, b.Len()) + for index := range order { + order[index] = index + } + sort.SliceStable(order, func(i, j int) bool { return b.timestamps[order[i]] < b.timestamps[order[j]] }) + for _, index := range order { + timestamp, err := b.Timestamp(index) + if err != nil { + yield(subscription.Row{}, err) + return + } + values := make([]any, len(b.columns)) + for column := range values { + values[column], err = b.Value(index, column) + if err != nil { + yield(subscription.Row{}, err) + return + } + } + if !yield(subscription.Row{Timestamp: ×tamp, Values: values}, nil) { + return + } + } + } +} + +func releaseProcessorMessages(messages []subscription.SubscriptionMessage) { + for _, message := range messages { + message.Release() + } +} + +func requireProcessorCommitIDs(t *testing.T, messages []subscription.SubscriptionMessage, expected ...int64) { + t.Helper() + if len(messages) != len(expected) { + t.Fatalf("delivered %d messages, want %d", len(messages), len(expected)) + } + for index, message := range messages { + if id := message.CommitContext().CommitID; id != expected[index] { + t.Fatalf("message %d commit ID = %d, want %d", index, id, expected[index]) + } + if _, err := message.Records(); err != nil { + t.Fatalf("delivered message %d is unreadable: %v", index, err) + } + } +} + +func processorOutputValue(t *testing.T, messages []subscription.SubscriptionMessage, row, column int) any { + t.Helper() + if len(messages) != 1 { + t.Fatalf("delivered %d messages, want one", len(messages)) + } + batches, err := messages[0].Records() + if err != nil || len(batches) != 1 { + t.Fatalf("output batches = %d, error = %v", len(batches), err) + } + value, err := batches[0].Value(row, column) + if err != nil { + t.Fatal(err) + } + return value +} diff --git a/subscription/subscription_poll_test.go b/subscription/subscription_poll_test.go new file mode 100644 index 0000000..0fe30ab --- /dev/null +++ b/subscription/subscription_poll_test.go @@ -0,0 +1,347 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package subscription + +import ( + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "reflect" + "testing" + "time" +) + +// This fixture is transcribed independently from Java SubscriptionPollResponse, +// TabletsPayload and SubscriptionCommitContext.serialize, and from Tablet and +// MeasurementSchema.serializeTo in TsFile 2.5.1-261023-SNAPSHOT. It does not use +// the Go production encoders. The Tablet contains one INT64 FIELD, with raw +// timestamp/value pairs (20, 200), (10, 101), (10, 102). +func javaSubscriptionTabletFrame(t *testing.T) []byte { + t.Helper() + body, err := hex.DecodeString( + "0001" + // TABLETS response. + "00000001" + "000000026462" + "00000001" + // One database db, one Tablet. + "000000066576656e7473" + "00000003" + // Target events, three rows. + "01" + "00000001" + "01" + // Schemas present, one non-null schema. + "000000027331" + "020000" + "00000000" + "01" + // s1, INT64/PLAIN/UNCOMPRESSED, no properties, FIELD. + "01" + "0000000000000014" + "000000000000000a" + "000000000000000a" + + "00" + "01" + "01" + // No bitmaps, values and column present. + "00000000000000c8" + "0000000000000065" + "0000000000000066" + + "ffffffff" + // Final offset: one Tablet, rather than three rows. + "02" + "00000007" + "00000002" + // Context version, node, restart count. + "00000005746f706963" + "0000000567726f7570" + // topic, group. + "000000000000002a" + "0000000000000003" + // Commit ID 42, seek generation 3. + "0000000172" + "0000000000000064" + "00" + // Region r, physical time 100, no writer. + "01" + "00000000") // Time selected, no per-table overrides. + if err != nil { + t.Fatal(err) + } + return body +} + +func decodedJavaSubscriptionMessage(t *testing.T, body []byte, dialect Dialect) SubscriptionMessage { + t.Helper() + response, err := decodeSubscriptionPollResponse(body) + if err != nil { + t.Fatalf("decode Java-format frame: %v", err) + } + message, err := response.message(dialect) + if err != nil || message == nil { + t.Fatalf("construct record message: message=%v, error=%v", message, err) + } + return message +} + +func TestSubscriptionPollRejectsFileBeforeSiblingDecode(t *testing.T) { + for _, kind := range []byte{2, 3, 4} { + for _, fileFirst := range []bool{false, true} { + for _, malformedSibling := range []bool{false, true} { + t.Run(fmt.Sprintf("kind=%d/first=%t/malformed=%t", kind, fileFirst, malformedSibling), func(t *testing.T) { + sibling := javaSubscriptionTabletFrame(t) + if malformedSibling { + sibling = sibling[:1] + } + // A tag alone must reject files without attempting file I/O. + file := []byte{0, kind} + bodies := [][]byte{sibling, file} + if fileFirst { + bodies = [][]byte{file, sibling} + } + responses, err := decodeSubscriptionPollResponses(bodies) + if !errors.Is(err, ErrUnsupportedTsFile) || responses != nil { + t.Fatalf("file response returned partial data or wrong error: %v, %v", responses, err) + } + }) + } + } + } +} + +func TestSubscriptionPollRejectsMalformedJavaFramesWithoutPartialResults(t *testing.T) { + valid := javaSubscriptionTabletFrame(t) + cases := map[string][]byte{} + cases["trailing-byte"] = append(append([]byte(nil), valid...), 0) + // Mandatory fields and the optional projection-map header must be complete. + // Ending exactly after the context or the global flag is valid in older Java clients. + for _, length := range []int{1, 5, 9, 29, 41, 60, 100, 105, 106, 120, 148, 162, 165} { + cases[fmt.Sprintf("truncated-%d", length)] = valid[:length] + } + for _, field := range []struct { + name string + offset int + value uint32 + }{ + {"negative-database-count", 2, 0xffffffff}, + {"invalid-database-name-length", 6, 0xfffffffe}, + {"negative-tablet-count", 12, 0xffffffff}, + {"negative-row-count", 26, 0xffffffff}, + {"oversized-column-count", 31, 0x7fffffff}, + {"invalid-column-name-length", 36, 0xfffffffe}, + } { + body := append([]byte(nil), valid...) + binary.BigEndian.PutUint32(body[field.offset:field.offset+4], field.value) + cases[field.name] = body + } + for name, malformed := range cases { + t.Run(name, func(t *testing.T) { + responses, err := decodeSubscriptionPollResponses([][]byte{valid, malformed}) + if err == nil || responses != nil { + t.Fatalf("malformed sibling returned partial results: %v, %v", responses, err) + } + }) + } +} + +func TestSubscriptionPollAcceptsLegacyProjectionTails(t *testing.T) { + for _, length := range []int{163, 164} { // Context only, or context and global flag. + t.Run(fmt.Sprintf("frame-length-%d", length), func(t *testing.T) { + message := decodedJavaSubscriptionMessage(t, javaSubscriptionTabletFrame(t)[:length], TableDialect) + defer message.Release() + batches, err := message.Records() + if err != nil || len(batches) != 1 { + t.Fatalf("legacy record batches: %v, %v", batches, err) + } + rows := 0 + for row, err := range batches[0].Rows() { + if err != nil || row.Timestamp == nil { + t.Fatalf("legacy response lost projected timestamps: %v, %v", row, err) + } + rows++ + } + if rows != 3 { + t.Fatalf("legacy response yielded %d rows, want 3", rows) + } + }) + } +} + +func TestSubscriptionDecodedRowsKeepStableOrderAndReleaseBorrowedData(t *testing.T) { + message := decodedJavaSubscriptionMessage(t, javaSubscriptionTabletFrame(t), TableDialect) + batches, err := message.Records() + if err != nil || len(batches) != 1 { + t.Fatalf("record batches: %v, %v", batches, err) + } + batch := batches[0] + var timestamps, values []int64 + for row, err := range batch.Rows() { + if err != nil || row.Timestamp == nil || len(row.Values) != 1 { + t.Fatalf("row: %v, %v", row, err) + } + timestamps = append(timestamps, *row.Timestamp) + value, ok := row.Values[0].(int64) + if !ok { + t.Fatalf("INT64 value has type %T", row.Values[0]) + } + values = append(values, value) + } + if !reflect.DeepEqual(timestamps, []int64{10, 10, 20}) || !reflect.DeepEqual(values, []int64{101, 102, 200}) { + t.Fatalf("unstable user-visible row ordering: timestamps=%v, values=%v", timestamps, values) + } + for index, want := range []int64{20, 10, 10} { + got, err := batch.Timestamp(index) + if err != nil || got != want { + t.Fatalf("raw timestamp %d was reordered: %d, %v", index, got, err) + } + value, err := batch.Value(index, 0) + if err != nil || value != []int64{200, 101, 102}[index] { + t.Fatalf("raw value %d was reordered: %v, %v", index, value, err) + } + } + context := message.CommitContext() + message.Release() + message.Release() + if _, err := message.Records(); !errors.Is(err, ErrDataReleased) { + t.Fatalf("records after release: %v", err) + } + if _, err := batch.Timestamp(0); !errors.Is(err, ErrDataReleased) { + t.Fatalf("borrowed timestamp after release: %v", err) + } + if _, err := batch.Value(0, 0); !errors.Is(err, ErrDataReleased) { + t.Fatalf("borrowed value after release: %v", err) + } + yields := 0 + for _, err := range batch.Rows() { + yields++ + if !errors.Is(err, ErrDataReleased) { + t.Fatalf("iteration after release: %v", err) + } + } + if yields != 1 { + t.Fatalf("released iterator yielded %d times", yields) + } + if message.Kind() != RecordMessage || message.CommitContext() != context || context.CommitID != 42 || context.SeekGeneration != 3 { + t.Fatal("release discarded the delivery identity required for later acknowledgment") + } +} + +func TestSubscriptionReleaseInsideRowCallbackStopsIteration(t *testing.T) { + message := decodedJavaSubscriptionMessage(t, javaSubscriptionTabletFrame(t), TableDialect) + batches, err := message.Records() + if err != nil || len(batches) != 1 { + t.Fatalf("record batches: %v, %v", batches, err) + } + finished := make(chan error, 1) + go func() { + rows, released := 0, 0 + for _, err := range batches[0].Rows() { + if err != nil { + if !errors.Is(err, ErrDataReleased) { + finished <- err + return + } + released++ + continue + } + rows++ + message.Release() + } + if rows != 1 || released != 1 { + finished <- fmt.Errorf("iteration after callback release: rows=%d, errors=%d", rows, released) + return + } + finished <- nil + }() + select { + case err := <-finished: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("Release deadlocked inside the row callback") + } +} + +func TestSubscriptionContinuationRejectsDifferentDeliveryWithoutLeakingRows(t *testing.T) { + for _, changedOffset := range []int{133, 141} { // Commit ID and seek generation in the Java fixture. + t.Run(fmt.Sprintf("context-offset-%d", changedOffset), func(t *testing.T) { + firstBody := javaSubscriptionTabletFrame(t) + binary.BigEndian.PutUint32(firstBody[102:106], 1) // Request continuation at offset 1. + first, err := decodeSubscriptionPollResponse(firstBody) + if err != nil { + t.Fatal(err) + } + if message, err := first.message(TableDialect); err == nil || message != nil { + t.Fatal("incomplete delivery exposed rows before continuation") + } + lastBody := javaSubscriptionTabletFrame(t) + binary.BigEndian.PutUint32(lastBody[102:106], 0xfffffffe) // Two Tablets total. + badBody := append([]byte(nil), lastBody...) + binary.BigEndian.PutUint64(badBody[changedOffset:changedOffset+8], 999) + bad, err := decodeSubscriptionPollResponse(badBody) + if err != nil { + t.Fatal(err) + } + if err := first.appendTablets(bad); err == nil { + t.Fatal("accepted a continuation from another delivery") + } + last, err := decodeSubscriptionPollResponse(lastBody) + if err != nil { + t.Fatal(err) + } + if err := first.appendTablets(last); err != nil { + t.Fatalf("rejected continuation corrupted the current delivery: %v", err) + } + message, err := first.message(TableDialect) + if err != nil || message == nil { + t.Fatalf("completed delivery: %v, %v", message, err) + } + defer message.Release() + batches, err := message.Records() + if err != nil || len(batches) != 2 { + t.Fatalf("rejected continuation leaked records: %v, %v", batches, err) + } + rows := 0 + for _, batch := range batches { + for _, err := range batch.Rows() { + if err != nil { + t.Fatal(err) + } + rows++ + } + } + if rows != 6 { + t.Fatalf("completed delivery contains %d rows, want 6", rows) + } + }) + } +} + +func TestSubscriptionProjectionChangesVisibleRowsByModel(t *testing.T) { + for _, dialect := range []Dialect{TreeDialect, TableDialect} { + for _, override := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/override=%t", dialect, override), func(t *testing.T) { + body := javaSubscriptionTabletFrame(t) + body[163] = 0 // Global time projection excludes timestamps. + if override { + // One database/table override: db.events selects time. + tail, err := hex.DecodeString("0000000100000002646200000001000000066576656e747301") + if err != nil { + t.Fatal(err) + } + body = append(body[:164], tail...) + } + message := decodedJavaSubscriptionMessage(t, body, dialect) + defer message.Release() + batches, err := message.Records() + if err != nil || len(batches) != 1 { + t.Fatalf("record batches: %v, %v", batches, err) + } + rows := 0 + for row, err := range batches[0].Rows() { + if err != nil { + t.Fatal(err) + } + wantTime := dialect == TreeDialect || override + if (row.Timestamp != nil) != wantTime { + t.Fatalf("projection exposed wrong timestamp: %v", row.Timestamp) + } + rows++ + } + if rows != 3 { + t.Fatalf("projection changed row count: %d", rows) + } + if timestamp, err := batches[0].Timestamp(0); err != nil || timestamp != 20 { + t.Fatalf("projection hid raw processing timestamp: %d, %v", timestamp, err) + } + }) + } + } +} diff --git a/subscription/watermark_processor_test.go b/subscription/watermark_processor_test.go new file mode 100644 index 0000000..eeb222e --- /dev/null +++ b/subscription/watermark_processor_test.go @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package subscription_test + +import ( + "errors" + "math" + "testing" + "time" + + "github.com/apache/iotdb-client-go/subscription" +) + +func newTestWatermarkProcessor(t *testing.T, config subscription.WatermarkProcessorConfig) *subscription.WatermarkProcessor { + t.Helper() + processor, err := subscription.NewWatermarkProcessor(config) + if err != nil { + t.Fatal(err) + } + t.Cleanup(processor.Reset) + return processor +} + +func TestWatermarkOrdersWholeMessagesAfterEveryRegionAdvances(t *testing.T) { + processor := newTestWatermarkProcessor(t, subscription.WatermarkProcessorConfig{ + MaxOutOfOrderness: 10, Timeout: time.Hour, StaleRegionTimeout: -1, + }) + high := processorRecord("topic", "a", 1, processorTreeBatch([]int64{200})) + low := processorRecord("topic", "b", 2, processorTreeBatch([]int64{50})) + middle := processorRecord("topic", "a", 3, processorTreeBatch([]int64{100, 1})) + high.context.CommitID = subscription.InvalidCommitID + high.context.Writer = &subscription.WriterPosition{ID: subscription.WriterID{RegionID: "a", NodeID: 7}, Progress: subscription.WriterProgress{PhysicalTime: 200, LocalSeq: 1}} + messages, err := processor.Process([]subscription.SubscriptionMessage{high, low, middle}) + if err != nil || len(messages) != 0 { + t.Fatalf("premature delivery: %d messages, error=%v", len(messages), err) + } + buffer := processor.Buffered() + if buffer.Count != 3 || len(buffer.Contexts) != 3 { + t.Fatalf("renewal snapshot lost buffered deliveries: %+v", buffer) + } + legacy := make(map[int64]bool) + writerRenewed := false + for _, context := range buffer.Contexts { + if context.Writer != nil { + writerRenewed = context.Writer.ID.NodeID == 7 && context.Writer.Progress.LocalSeq == 1 + } else { + legacy[context.CommitID] = true + } + } + if !writerRenewed || !legacy[2] || !legacy[3] { + t.Fatalf("renewal snapshot omitted an independent delivery: %+v", buffer.Contexts) + } + messages, err = processor.Process([]subscription.SubscriptionMessage{processorWatermark("topic", "b", 250)}) + if err != nil { + t.Fatal(err) + } + requireProcessorCommitIDs(t, messages, 2, 3) + batches, err := messages[1].Records() + if err != nil || len(batches) != 1 || batches[0].Len() != 2 { + t.Fatalf("message was split instead of emitted whole: batches=%v error=%v", batches, err) + } + releaseProcessorMessages(messages) + if buffer = processor.Buffered(); buffer.Count != 1 || len(buffer.Contexts) != 1 || buffer.Contexts[0].Writer == nil { + t.Fatalf("remaining writer delivery is not renewable: %+v", buffer) + } + messages, err = processor.Process([]subscription.SubscriptionMessage{processorWatermark("topic", "a", 300)}) + if err != nil || len(messages) != 1 || messages[0] != high { + t.Fatalf("high message not released after all sources advanced: messages=%v error=%v", messages, err) + } + releaseProcessorMessages(messages) +} + +func TestWatermarkResetTopicDropsOnlyItsRecordsAndOrderingSource(t *testing.T) { + processor := newTestWatermarkProcessor(t, subscription.WatermarkProcessorConfig{ + MaxOutOfOrderness: 10, Timeout: time.Hour, StaleRegionTimeout: -1, + }) + a := processorRecord("a", "region", 1, processorTreeBatch([]int64{20})) + b := processorRecord("b", "region", 2, processorTreeBatch([]int64{100})) + messages, err := processor.Process([]subscription.SubscriptionMessage{a, b}) + if err != nil || len(messages) != 0 { + t.Fatalf("initial delivery=%v error=%v", messages, err) + } + processor.ResetTopic("a") + if _, err := a.Records(); !errors.Is(err, subscription.ErrDataReleased) { + t.Fatalf("reset topic retained its borrowed data: %v", err) + } + if _, err := b.Records(); err != nil { + t.Fatalf("other topic's record was released: %v", err) + } + buffer := processor.Buffered() + if buffer.Count != 1 || len(buffer.Contexts) != 1 || buffer.Contexts[0].Topic != "b" { + t.Fatalf("reset corrupted other topic's renewal: %+v", buffer) + } + messages, err = processor.Process([]subscription.SubscriptionMessage{processorWatermark("b", "region", 120)}) + if err != nil { + t.Fatal(err) + } + requireProcessorCommitIDs(t, messages, 2) + releaseProcessorMessages(messages) +} + +func TestWatermarkForceFlushDoesNotWaitForRegionProgress(t *testing.T) { + for _, test := range []struct { + name string + timeout time.Duration + limit int64 + }{ + {name: "capacity", timeout: time.Hour, limit: 1}, + {name: "zero_timeout", timeout: 0}, + } { + t.Run(test.name, func(t *testing.T) { + processor := newTestWatermarkProcessor(t, subscription.WatermarkProcessorConfig{ + MaxOutOfOrderness: 1000, Timeout: test.timeout, MaxBufferBytes: test.limit, StaleRegionTimeout: -1, + }) + high := processorRecord("topic", "region", 2, processorTreeBatch([]int64{20})) + low := processorRecord("topic", "region", 1, processorTreeBatch([]int64{10})) + messages, err := processor.Process([]subscription.SubscriptionMessage{high, low}) + if err != nil { + t.Fatal(err) + } + requireProcessorCommitIDs(t, messages, 1, 2) + releaseProcessorMessages(messages) + if buffer := processor.Buffered(); buffer.Count != 0 || len(buffer.Contexts) != 0 { + t.Fatalf("flushed records remain renewable: %+v", buffer) + } + }) + } +} + +func TestWatermarkTimestampSubtractionCannotWrapAndDeliverEarly(t *testing.T) { + processor := newTestWatermarkProcessor(t, subscription.WatermarkProcessorConfig{ + MaxOutOfOrderness: 10, Timeout: time.Hour, StaleRegionTimeout: -1, + }) + _, err := processor.Process([]subscription.SubscriptionMessage{processorWatermark("topic", "region", math.MinInt64+5)}) + if err != nil { + t.Fatal(err) + } + record := processorRecord("topic", "region", 1, processorTreeBatch([]int64{math.MinInt64+1})) + messages, err := processor.Process([]subscription.SubscriptionMessage{record}) + if err != nil || len(messages) != 0 { + t.Fatalf("underflow delivered an unready message: messages=%v error=%v", messages, err) + } + messages, err = processor.Flush() + if err != nil { + t.Fatal(err) + } + requireProcessorCommitIDs(t, messages, 1) + releaseProcessorMessages(messages) +} + +func TestWatermarkReadFailureDoesNotAdvanceOrTakeOwnership(t *testing.T) { + processor := newTestWatermarkProcessor(t, subscription.WatermarkProcessorConfig{ + MaxOutOfOrderness: 10, Timeout: time.Hour, StaleRegionTimeout: -1, + }) + seed := processorRecord("topic", "a", 1, processorTreeBatch([]int64{100})) + if messages, err := processor.Process([]subscription.SubscriptionMessage{seed}); err != nil || len(messages) != 0 { + t.Fatalf("initial delivery=%v error=%v", messages, err) + } + readFailure := errors.New("input timestamp unavailable") + brokenBatch := processorTreeBatch([]int64{600}) + brokenBatch.timestampErr = readFailure + broken := processorRecord("topic", "a", 3, brokenBatch) + good := processorRecord("topic", "a", 2, processorTreeBatch([]int64{500})) + messages, err := processor.Process([]subscription.SubscriptionMessage{ + processorWatermark("topic", "b", 1000), good, broken, + }) + if !errors.Is(err, readFailure) || len(messages) != 0 { + t.Fatalf("failed input delivered data or lost error: messages=%v error=%v", messages, err) + } + for _, input := range []*processorInputMessage{good, broken} { + if _, err := input.Records(); err != nil { + t.Fatalf("failed round consumed caller-owned input: %v", err) + } + defer input.Release() + } + next := processorRecord("topic", "a", 4, processorTreeBatch([]int64{101})) + messages, err = processor.Process([]subscription.SubscriptionMessage{next}) + if err != nil || len(messages) != 0 { + t.Fatalf("failed round advanced ordering: messages=%v error=%v", messages, err) + } + messages, err = processor.Flush() + if err != nil { + t.Fatal(err) + } + requireProcessorCommitIDs(t, messages, 1, 4) + releaseProcessorMessages(messages) +}
