This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-13190-7f6fe441c0e8d920799f663ff3011a32ba3320f1 in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit 35657a514ddd5437bf8e6e2f809fae1984c731a0 Author: Sepuri Sai Krishna <[email protected]> AuthorDate: Tue Nov 17 15:53:50 2026 +0000 [Fix][Connector-V2][Databend] Create and write a TIME column as text (#13190) --- docs/en/connectors/sink/Databend.md | 2 +- docs/zh/connectors/sink/Databend.md | 2 +- .../databend/catalog/DatabendCatalog.java | 13 ++++- .../databend/sink/DatabendSinkWriter.java | 10 +++- .../databend/catalog/DatabendCatalogTest.java | 23 ++++++++- .../databend/sink/DatabendSinkWriterTest.java | 28 +++++++++++ .../e2e/connector/databend/DatabendIT.java | 31 ++++++++++++ .../resources/databend/fake_time_to_databend.conf | 55 ++++++++++++++++++++++ 8 files changed, 158 insertions(+), 6 deletions(-) diff --git a/docs/en/connectors/sink/Databend.md b/docs/en/connectors/sink/Databend.md index 22ecf15093..f78516f3b9 100644 --- a/docs/en/connectors/sink/Databend.md +++ b/docs/en/connectors/sink/Databend.md @@ -96,7 +96,7 @@ Option descriptions: | STRING | STRING | | BYTES | VARBINARY | | DATE | DATE | -| TIME | TIME | +| TIME | STRING | | TIMESTAMP | TIMESTAMP | ## Task Examples diff --git a/docs/zh/connectors/sink/Databend.md b/docs/zh/connectors/sink/Databend.md index f391ec2317..1407d7cf65 100644 --- a/docs/zh/connectors/sink/Databend.md +++ b/docs/zh/connectors/sink/Databend.md @@ -95,7 +95,7 @@ Databend sink 内部通过 stage attachment 实现数据的批量导入。 | STRING | STRING | | BYTES | VARBINARY | | DATE | DATE | -| TIME | TIME | +| TIME | STRING | | TIMESTAMP | TIMESTAMP | ## 任务示例 diff --git a/seatunnel-connectors-v2/connector-databend/src/main/java/org/apache/seatunnel/connectors/seatunnel/databend/catalog/DatabendCatalog.java b/seatunnel-connectors-v2/connector-databend/src/main/java/org/apache/seatunnel/connectors/seatunnel/databend/catalog/DatabendCatalog.java index 4758d41681..630c7d3f5f 100644 --- a/seatunnel-connectors-v2/connector-databend/src/main/java/org/apache/seatunnel/connectors/seatunnel/databend/catalog/DatabendCatalog.java +++ b/seatunnel-connectors-v2/connector-databend/src/main/java/org/apache/seatunnel/connectors/seatunnel/databend/catalog/DatabendCatalog.java @@ -420,7 +420,14 @@ public class DatabendCatalog implements Catalog { return sb.toString(); } - private String toDatabendTypeString(SeaTunnelDataType<?> dataType) { + /** + * Converts a SeaTunnel type to the Databend column type used in CREATE TABLE. Package private + * so the mapping can be asserted directly in unit tests. + * + * @param dataType the SeaTunnel type of the column + * @return the Databend column type + */ + static String toDatabendTypeString(SeaTunnelDataType<?> dataType) { switch (dataType.getSqlType()) { case BOOLEAN: return "BOOLEAN"; @@ -447,7 +454,9 @@ public class DatabendCatalog implements Catalog { case DATE: return "DATE"; case TIME: - return "TIME"; + // Databend has no TIME type, so a CREATE TABLE naming TIME fails to parse. A time + // of day is stored as text, as DatabendTypeConverter does for schema changes. + return "STRING"; case TIMESTAMP: LocalTimeType timeType = (LocalTimeType) dataType; return "TIMESTAMP"; diff --git a/seatunnel-connectors-v2/connector-databend/src/main/java/org/apache/seatunnel/connectors/seatunnel/databend/sink/DatabendSinkWriter.java b/seatunnel-connectors-v2/connector-databend/src/main/java/org/apache/seatunnel/connectors/seatunnel/databend/sink/DatabendSinkWriter.java index 9b245b28af..4356049e2e 100644 --- a/seatunnel-connectors-v2/connector-databend/src/main/java/org/apache/seatunnel/connectors/seatunnel/databend/sink/DatabendSinkWriter.java +++ b/seatunnel-connectors-v2/connector-databend/src/main/java/org/apache/seatunnel/connectors/seatunnel/databend/sink/DatabendSinkWriter.java @@ -48,6 +48,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.time.LocalDateTime; +import java.time.LocalTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; @@ -686,7 +687,14 @@ public class DatabendSinkWriter i + 1, field, field.getClass().getSimpleName()); - preparedStatement.setObject(i + 1, field); + if (field instanceof LocalTime) { + // A TIME column is created as STRING, since Databend has no TIME type, and + // databend-jdbc's setObject rejects a LocalTime. Write the time of day as text. + preparedStatement.setString( + i + 1, ((LocalTime) field).format(DateTimeFormatter.ISO_LOCAL_TIME)); + } else { + preparedStatement.setObject(i + 1, field); + } } } preparedStatement.addBatch(); diff --git a/seatunnel-connectors-v2/connector-databend/src/test/java/org/apache/seatunnel/connectors/seatunnel/databend/catalog/DatabendCatalogTest.java b/seatunnel-connectors-v2/connector-databend/src/test/java/org/apache/seatunnel/connectors/seatunnel/databend/catalog/DatabendCatalogTest.java index a70e8d2a60..798def8dd7 100644 --- a/seatunnel-connectors-v2/connector-databend/src/test/java/org/apache/seatunnel/connectors/seatunnel/databend/catalog/DatabendCatalogTest.java +++ b/seatunnel-connectors-v2/connector-databend/src/test/java/org/apache/seatunnel/connectors/seatunnel/databend/catalog/DatabendCatalogTest.java @@ -17,10 +17,13 @@ package org.apache.seatunnel.connectors.seatunnel.databend.catalog; +import org.apache.seatunnel.api.table.catalog.Column; +import org.apache.seatunnel.api.table.catalog.PhysicalColumn; import org.apache.seatunnel.api.table.type.BasicType; import org.apache.seatunnel.api.table.type.DecimalType; import org.apache.seatunnel.api.table.type.LocalTimeType; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.connectors.seatunnel.databend.util.DatabendTypeConverter; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -31,7 +34,8 @@ import java.util.Locale; /** * Covers the Databend to SeaTunnel type mapping the catalog applies to every column of every table * the source reads. The type names used here are the ones the JDBC driver reports as TYPE_NAME: - * without a Nullable(...) wrapper, and with the arguments of a parameterised type. + * without a Nullable(...) wrapper, and with the arguments of a parameterised type. Also covers the + * column type the catalog names when it creates a table. */ public class DatabendCatalogTest { @@ -113,6 +117,23 @@ public class DatabendCatalogTest { assertMaps("array(int32)", BasicType.STRING_TYPE); } + @Test + public void aTimeColumnIsCreatedAsString() { + // Databend has no TIME type, so CREATE TABLE has to name another one. It is the type the + // schema change path already uses for a TIME column. + Column timeColumn = + PhysicalColumn.builder() + .name("start_time") + .dataType(LocalTimeType.LOCAL_TIME_TYPE) + .nullable(true) + .build(); + Assertions.assertEquals( + "STRING", DatabendCatalog.toDatabendTypeString(LocalTimeType.LOCAL_TIME_TYPE)); + Assertions.assertEquals( + DatabendTypeConverter.convertToDatabendType(timeColumn).getColumnType(), + DatabendCatalog.toDatabendTypeString(LocalTimeType.LOCAL_TIME_TYPE)); + } + private static void assertMaps(String databendTypeName, SeaTunnelDataType<?> expected) { Assertions.assertEquals( expected, diff --git a/seatunnel-connectors-v2/connector-databend/src/test/java/org/apache/seatunnel/connectors/seatunnel/databend/sink/DatabendSinkWriterTest.java b/seatunnel-connectors-v2/connector-databend/src/test/java/org/apache/seatunnel/connectors/seatunnel/databend/sink/DatabendSinkWriterTest.java index b3ab305244..f3dd631143 100644 --- a/seatunnel-connectors-v2/connector-databend/src/test/java/org/apache/seatunnel/connectors/seatunnel/databend/sink/DatabendSinkWriterTest.java +++ b/seatunnel-connectors-v2/connector-databend/src/test/java/org/apache/seatunnel/connectors/seatunnel/databend/sink/DatabendSinkWriterTest.java @@ -34,10 +34,13 @@ import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.Statement; +import java.time.LocalTime; import java.util.Collections; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.startsWith; import static org.mockito.Mockito.doCallRealMethod; @@ -312,6 +315,31 @@ public class DatabendSinkWriterTest { verify(statement, never()).execute(startsWith("MERGE INTO")); } + @Test + public void testProcessRowWritesTimeAsText() throws Exception { + // databend-jdbc's setObject throws "Unsupported object type" for a LocalTime, so a TIME + // value has to reach the driver as text, the type the column is created with. + DatabendSinkWriter sinkWriter = mock(DatabendSinkWriter.class); + PreparedStatement preparedStatement = mock(PreparedStatement.class); + setPrivateField(sinkWriter, "preparedStatement", preparedStatement); + SeaTunnelRow row = + new SeaTunnelRow( + new Object[] { + 1, LocalTime.of(8, 0), LocalTime.of(10, 15, 30, 123_000_000) + }); + + Method processRow = + DatabendSinkWriter.class.getDeclaredMethod("processRow", SeaTunnelRow.class); + processRow.setAccessible(true); + processRow.invoke(sinkWriter, row); + + verify(preparedStatement).setObject(1, 1); + verify(preparedStatement).setString(2, "08:00:00"); + verify(preparedStatement).setString(3, "10:15:30.123"); + verify(preparedStatement, never()).setObject(anyInt(), any(LocalTime.class)); + verify(preparedStatement).addBatch(); + } + private DatabendSinkWriter mockCdcWriter(Statement statement) throws Exception { Connection connection = mock(Connection.class); when(connection.createStatement()).thenReturn(statement); diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-databend-e2e/src/test/java/org/apache/seatunnel/e2e/connector/databend/DatabendIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-databend-e2e/src/test/java/org/apache/seatunnel/e2e/connector/databend/DatabendIT.java index 3b2db2e652..cbc1638b0e 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-databend-e2e/src/test/java/org/apache/seatunnel/e2e/connector/databend/DatabendIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-databend-e2e/src/test/java/org/apache/seatunnel/e2e/connector/databend/DatabendIT.java @@ -244,6 +244,37 @@ public class DatabendIT extends TestSuiteBase implements TestResource { } } + /** + * Writes a TIME column to a table the sink creates through the catalog. Databend has no TIME + * type, so the table only exists, and the rows only arrive, when the catalog names a text type + * for the column and the writer sends the time of day as text. + */ + @TestTemplate + public void testFakeTimeToDatabend(TestContainer container) throws Exception { + String table = DATABASE + ".time_sink_table"; + try (Connection conn = getConnection(); + Statement statement = conn.createStatement()) { + statement.execute("DROP TABLE IF EXISTS " + table); + } + + Container.ExecResult execResult = + container.executeJob("/databend/fake_time_to_databend.conf"); + Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr()); + + try (Connection conn = getConnection(); + Statement statement = conn.createStatement()) { + List<String> values = new ArrayList<>(); + try (ResultSet resultSet = + statement.executeQuery("SELECT start_time FROM " + table + " ORDER BY id")) { + while (resultSet.next()) { + values.add(resultSet.getString(1)); + } + } + Assertions.assertEquals(Arrays.asList("08:00:00", "10:15:30.123"), values); + statement.execute("DROP TABLE IF EXISTS " + table); + } + } + private synchronized Connection getConnection() throws SQLException { if (this.connection == null || this.connection.isClosed()) { LOG.info("Creating new database connection"); diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-databend-e2e/src/test/resources/databend/fake_time_to_databend.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-databend-e2e/src/test/resources/databend/fake_time_to_databend.conf new file mode 100644 index 0000000000..22398ee612 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-databend-e2e/src/test/resources/databend/fake_time_to_databend.conf @@ -0,0 +1,55 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + execution.parallelism = 1 + job.mode = "BATCH" +} + +source { + FakeSource { + schema { + fields { + id = int + start_time = time + } + } + rows = [ + { + kind = INSERT + fields = [1, "08:00:00"] + }, + { + kind = INSERT + fields = [2, "10:15:30.123"] + } + ] + } +} + +sink { + Databend { + url = "jdbc:databend://databend:8000/default" + username = "root" + password = "" + database = "default" + table = "time_sink_table" + batch_size = 1 + schema_save_mode = "CREATE_SCHEMA_WHEN_NOT_EXIST" + data_save_mode = "APPEND_DATA" + } +}
