This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-13218-7fd23f83f7836321bdde1397b785d3d3cb5d0ea9 in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit a3e1a731a3ca6d7136e26d9d46ca685d9675c4da Author: corgy-w <[email protected]> AuthorDate: Wed Nov 11 14:38:34 2026 +0000 [Fix][Connector-V2] Keep Qdrant payload numeric types and ranges (#13218) --- .../seatunnel/qdrant/sink/QdrantBatchWriter.java | 5 +- .../qdrant/sink/QdrantBatchWriterTest.java | 64 ++++++++++++++++++++++ .../e2e/connector/v2/qdrant/QdrantIT.java | 52 ++++++++++++++++++ .../resources/fake_to_qdrant_numeric_payload.conf | 43 +++++++++++++++ 4 files changed, 162 insertions(+), 2 deletions(-) diff --git a/seatunnel-connectors-v2/connector-qdrant/src/main/java/org/apache/seatunnel/connectors/seatunnel/qdrant/sink/QdrantBatchWriter.java b/seatunnel-connectors-v2/connector-qdrant/src/main/java/org/apache/seatunnel/connectors/seatunnel/qdrant/sink/QdrantBatchWriter.java index ed217af840..48edceca0d 100644 --- a/seatunnel-connectors-v2/connector-qdrant/src/main/java/org/apache/seatunnel/connectors/seatunnel/qdrant/sink/QdrantBatchWriter.java +++ b/seatunnel-connectors-v2/connector-qdrant/src/main/java/org/apache/seatunnel/connectors/seatunnel/qdrant/sink/QdrantBatchWriter.java @@ -158,11 +158,12 @@ public class QdrantBatchWriter { switch (sqlType) { case SMALLINT: case INT: - case BIGINT: return ValueFactory.value(Integer.parseInt(value.toString())); + case BIGINT: + return ValueFactory.value(Long.parseLong(value.toString())); case FLOAT: case DOUBLE: - return ValueFactory.value(Long.parseLong(value.toString())); + return ValueFactory.value(Double.parseDouble(value.toString())); case STRING: case DATE: return ValueFactory.value(value.toString()); diff --git a/seatunnel-connectors-v2/connector-qdrant/src/test/java/org/apache/seatunnel/connectors/seatunnel/qdrant/sink/QdrantBatchWriterTest.java b/seatunnel-connectors-v2/connector-qdrant/src/test/java/org/apache/seatunnel/connectors/seatunnel/qdrant/sink/QdrantBatchWriterTest.java new file mode 100644 index 0000000000..417d7e06ab --- /dev/null +++ b/seatunnel-connectors-v2/connector-qdrant/src/test/java/org/apache/seatunnel/connectors/seatunnel/qdrant/sink/QdrantBatchWriterTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.qdrant.sink; + +import org.apache.seatunnel.api.table.type.BasicType; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import io.qdrant.client.grpc.JsonWithInt; + +/** + * Verifies that payload values keep the shape their SeaTunnel type declares. + * + * <p>The sink documents {@code BIGINT -> payload integer} and {@code FLOAT/DOUBLE -> payload + * double}, so a numeric payload must be stored with the matching Qdrant value type and without + * losing range or precision. + */ +public class QdrantBatchWriterTest { + + @Test + public void testDoublePayloadKeepsFractionalPart() { + JsonWithInt.Value value = QdrantBatchWriter.buildPayload(BasicType.DOUBLE_TYPE, 1.5d); + + Assertions.assertEquals(1.5d, value.getDoubleValue(), 0.0d); + } + + @Test + public void testFloatPayloadKeepsFractionalPart() { + JsonWithInt.Value value = QdrantBatchWriter.buildPayload(BasicType.FLOAT_TYPE, 2.5f); + + Assertions.assertEquals(2.5d, value.getDoubleValue(), 0.0d); + } + + @Test + public void testBigintPayloadKeepsFullRange() { + JsonWithInt.Value value = + QdrantBatchWriter.buildPayload(BasicType.LONG_TYPE, 5_000_000_000L); + + Assertions.assertEquals(5_000_000_000L, value.getIntegerValue()); + } + + @Test + public void testIntPayloadStaysInteger() { + JsonWithInt.Value value = QdrantBatchWriter.buildPayload(BasicType.INT_TYPE, 42); + + Assertions.assertEquals(42L, value.getIntegerValue()); + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-qdrant-e2e/src/test/java/org/apache/seatunnel/e2e/connector/v2/qdrant/QdrantIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-qdrant-e2e/src/test/java/org/apache/seatunnel/e2e/connector/v2/qdrant/QdrantIT.java index 879038a247..b54d658617 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-qdrant-e2e/src/test/java/org/apache/seatunnel/e2e/connector/v2/qdrant/QdrantIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-qdrant-e2e/src/test/java/org/apache/seatunnel/e2e/connector/v2/qdrant/QdrantIT.java @@ -36,6 +36,7 @@ import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; import io.qdrant.client.QdrantClient; import io.qdrant.client.QdrantGrpcClient; import io.qdrant.client.grpc.Collections; +import io.qdrant.client.grpc.JsonWithInt; import io.qdrant.client.grpc.Points; import lombok.extern.slf4j.Slf4j; @@ -64,6 +65,7 @@ public class QdrantIT extends TestSuiteBase implements TestResource { private static final String AUTH_API_KEY = "seatunnel-test-key"; private static final String SOURCE_COLLECTION = "source_collection"; private static final String SINK_COLLECTION = "sink_collection"; + private static final String NUMERIC_PAYLOAD_COLLECTION = "numeric_payload_collection"; /** * Fixed Qdrant at v1.15.0 for stability; upgrading to v1.17.0+ requires ensuring the SeaTunnel * Qdrant connector is compatible with the latest breaking changes. @@ -118,6 +120,16 @@ public class QdrantIT extends TestSuiteBase implements TestResource { .build())) .get(); + client.createCollectionAsync( + NUMERIC_PAYLOAD_COLLECTION, + ImmutableMap.of( + "my_vector", + Collections.VectorParams.newBuilder() + .setSize(4) + .setDistance(Collections.Distance.Cosine) + .build())) + .get(); + client.createCollectionAsync( SINK_COLLECTION, ImmutableMap.of( @@ -171,4 +183,44 @@ public class QdrantIT extends TestSuiteBase implements TestResource { Assertions.assertEquals(0, execResult.getExitCode()); Assertions.assertEquals(10, authenticatedClient.countAsync(SINK_COLLECTION).get()); } + + /** + * Numeric payloads must keep the shape their declared SeaTunnel type maps to: BIGINT becomes a + * Qdrant integer and FLOAT/DOUBLE become Qdrant doubles. Before the fix the sink parsed every + * numeric payload with the wrong primitive parser, so any fractional double failed the job with + * a NumberFormatException and a BIGINT above the int range failed as well. + */ + @TestTemplate + public void testQdrantNumericPayload(TestContainer container) + throws IOException, InterruptedException, ExecutionException { + Container.ExecResult execResult = + container.executeJob("/fake_to_qdrant_numeric_payload.conf"); + Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr()); + Assertions.assertEquals(5, qdrantClient.countAsync(NUMERIC_PAYLOAD_COLLECTION).get()); + + Points.ScrollResponse response = + qdrantClient + .scrollAsync( + Points.ScrollPoints.newBuilder() + .setCollectionName(NUMERIC_PAYLOAD_COLLECTION) + .setLimit(10) + .build()) + .get(); + List<Points.RetrievedPoint> points = response.getResultList(); + Assertions.assertEquals(5, points.size()); + for (Points.RetrievedPoint point : points) { + Assertions.assertEquals( + JsonWithInt.Value.KindCase.INTEGER_VALUE, + point.getPayloadOrThrow("id").getKindCase(), + "BIGINT payload must be stored as a Qdrant integer"); + Assertions.assertEquals( + JsonWithInt.Value.KindCase.DOUBLE_VALUE, + point.getPayloadOrThrow("score").getKindCase(), + "DOUBLE payload must be stored as a Qdrant double"); + Assertions.assertEquals( + JsonWithInt.Value.KindCase.DOUBLE_VALUE, + point.getPayloadOrThrow("ratio").getKindCase(), + "FLOAT payload must be stored as a Qdrant double"); + } + } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-qdrant-e2e/src/test/resources/fake_to_qdrant_numeric_payload.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-qdrant-e2e/src/test/resources/fake_to_qdrant_numeric_payload.conf new file mode 100644 index 0000000000..582d245b54 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-qdrant-e2e/src/test/resources/fake_to_qdrant_numeric_payload.conf @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + FakeSource { + row.num = 5 + vector.dimension = 4 + schema = { + fields { + id = bigint + score = double + ratio = float + my_vector = float_vector + } + } + } +} + +sink { + Qdrant { + collection_name = "numeric_payload_collection" + host = "qdrante2e" + } +}
