This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d75e10c4b6e [fix](trino-connector) Refactor some code of
trino-connector jni reader (#37302)
d75e10c4b6e is described below
commit d75e10c4b6ec445ec2066314188aae150f255b86
Author: Tiewei Fang <[email protected]>
AuthorDate: Fri Jul 19 14:37:51 2024 +0800
[fix](trino-connector) Refactor some code of trino-connector jni reader
(#37302)
Previously, TrinoConnectorJniScanner converted Trino types to Hive
types, and then to Doris types.
In fact, the Hive type can be directly obtained from `columns_types`.
In this way, the step of converting Trino types to Hive types can be
deleted.
---
.../format/table/trino_connector_jni_reader.cpp | 5 +-
.../trinoconnector/TrinoConnectorJniScanner.java | 16 +--
.../TrinoTypeToHiveTypeTranslator.java | 115 ---------------------
3 files changed, 12 insertions(+), 124 deletions(-)
diff --git a/be/src/vec/exec/format/table/trino_connector_jni_reader.cpp
b/be/src/vec/exec/format/table/trino_connector_jni_reader.cpp
index b72da757629..eaced8572b9 100644
--- a/be/src/vec/exec/format/table/trino_connector_jni_reader.cpp
+++ b/be/src/vec/exec/format/table/trino_connector_jni_reader.cpp
@@ -43,9 +43,11 @@ TrinoConnectorJniReader::TrinoConnectorJniReader(
RuntimeProfile* profile, const TFileRangeDesc& range)
: JniReader(file_slot_descs, state, profile) {
std::vector<std::string> column_names;
+ std::vector<std::string> column_types;
for (const auto& desc : _file_slot_descs) {
std::string field = desc->col_name();
column_names.emplace_back(field);
+ column_types.emplace_back(JniConnector::get_jni_type(desc->type()));
}
std::map<String, String> params = {
{"catalog_name",
range.table_format_params.trino_connector_params.catalog_name},
@@ -63,7 +65,8 @@ TrinoConnectorJniReader::TrinoConnectorJniReader(
range.table_format_params.trino_connector_params.trino_connector_predicate},
{"trino_connector_trascation_handle",
range.table_format_params.trino_connector_params.trino_connector_trascation_handle},
- {"required_fields", join(column_names, ",")}};
+ {"required_fields", join(column_names, ",")},
+ {"columns_types", join(column_types, "#")}};
// Used to create trino connector options
for (const auto& kv :
diff --git
a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorJniScanner.java
b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorJniScanner.java
index af20668081d..6b15abfa413 100644
---
a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorJniScanner.java
+++
b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorJniScanner.java
@@ -129,11 +129,16 @@ public class TrinoConnectorJniScanner extends JniScanner {
public TrinoConnectorJniScanner(int batchSize, Map<String, String> params)
{
- catalogNameString = params.get("catalog_name");
- super.batchSize = batchSize;
- super.fields = params.get("required_fields").split(",");
+ String[] requiredFields = params.get("required_fields").split(",");
+ String[] requiredTypes = params.get("columns_types").split("#");
+ ColumnType[] columnTypes = new ColumnType[requiredTypes.length];
+ for (int i = 0; i < requiredTypes.length; i++) {
+ columnTypes[i] = ColumnType.parseType(requiredFields[i],
requiredTypes[i]);
+ }
+ initTableInfo(columnTypes, requiredFields, batchSize);
appendDataTimeNs = new long[fields.length];
+ catalogNameString = params.get("catalog_name");
connectorSplitString = params.get("trino_connector_split");
connectorTableHandleString =
params.get("trino_connector_table_handle");
connectorColumnHandleString =
params.get("trino_connector_column_handles");
@@ -336,7 +341,6 @@ public class TrinoConnectorJniScanner extends JniScanner {
}
private void parseRequiredTypes() {
- ColumnType[] columnTypes = new ColumnType[fields.length];
appendDataTimeNs = new long[fields.length];
trinoTypeList = Lists.newArrayList();
for (int i = 0; i < fields.length; i++) {
@@ -345,12 +349,8 @@ public class TrinoConnectorJniScanner extends JniScanner {
throw new RuntimeException(String.format("Cannot find field %s
in schema %s",
fields[i], trinoConnectorAllFieldNames));
}
-
trinoTypeList.add(columnMetadataList.get(index).getType());
- String hiveType =
TrinoTypeToHiveTypeTranslator.fromTrinoTypeToHiveType(trinoTypeList.get(i));
- columnTypes[i] = ColumnType.parseType(fields[i], hiveType);
}
- super.types = columnTypes;
}
private Session createSession(TrinoConnectorServicesProvider
trinoConnectorServicesProvider) {
diff --git
a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoTypeToHiveTypeTranslator.java
b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoTypeToHiveTypeTranslator.java
deleted file mode 100644
index 4948e5067a3..00000000000
---
a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoTypeToHiveTypeTranslator.java
+++ /dev/null
@@ -1,115 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.trinoconnector;
-
-import io.trino.spi.type.ArrayType;
-import io.trino.spi.type.BigintType;
-import io.trino.spi.type.BooleanType;
-import io.trino.spi.type.CharType;
-import io.trino.spi.type.DateType;
-import io.trino.spi.type.DecimalType;
-import io.trino.spi.type.DoubleType;
-import io.trino.spi.type.IntegerType;
-import io.trino.spi.type.MapType;
-import io.trino.spi.type.RealType;
-import io.trino.spi.type.RowType;
-import io.trino.spi.type.SmallintType;
-import io.trino.spi.type.TimeType;
-import io.trino.spi.type.TimestampType;
-import io.trino.spi.type.TimestampWithTimeZoneType;
-import io.trino.spi.type.TinyintType;
-import io.trino.spi.type.Type;
-import io.trino.spi.type.VarbinaryType;
-import io.trino.spi.type.VarcharType;
-
-public final class TrinoTypeToHiveTypeTranslator {
- private TrinoTypeToHiveTypeTranslator() {
- }
-
- // trino do not support LargeInt
- public static String fromTrinoTypeToHiveType(Type type) {
- if (type instanceof BooleanType) {
- return "boolean";
- } else if (type instanceof TinyintType) {
- return "tinyint";
- } else if (type instanceof SmallintType) {
- return "smallint";
- } else if (type instanceof IntegerType) {
- return "int";
- } else if (type instanceof BigintType) {
- return "bigint";
- } else if (type instanceof RealType) {
- return "float";
- } else if (type instanceof DoubleType) {
- return "double";
- } else if (type instanceof CharType) {
- return type.toString();
- } else if (type instanceof VarcharType) {
- return "string";
- } else if (type instanceof VarbinaryType) {
- return "string";
- } else if (type instanceof DecimalType) {
- StringBuilder sb = new StringBuilder("decimal");
- sb.append("(");
- sb.append(((DecimalType) type).getPrecision());
- sb.append(", ");
- sb.append(((DecimalType) type).getScale());
- sb.append(")");
- return sb.toString();
- } else if (type instanceof TimeType) {
- return "string";
- } else if (type instanceof DateType) {
- return "date";
- } else if (type instanceof TimestampType) {
- return "timestamp";
- } else if (type instanceof TimestampWithTimeZoneType) {
- return "timestamp";
- } else if (type instanceof ArrayType) {
- StringBuilder sb = new StringBuilder();
- sb.append("array<")
- .append(fromTrinoTypeToHiveType(((ArrayType)
type).getElementType()))
- .append(">");
- return sb.toString();
- } else if (type instanceof MapType) {
- StringBuilder sb = new StringBuilder();
- sb.append("map<")
- .append(fromTrinoTypeToHiveType(((MapType)
type).getKeyType()))
- .append(",")
- .append(fromTrinoTypeToHiveType(((MapType)
type).getValueType()));
- sb.append(">");
- return sb.toString();
- } else if (type instanceof RowType) {
- StringBuilder sb = new StringBuilder();
- sb.append("struct<");
- for (int i = 0; i < type.getTypeParameters().size(); ++i) {
- if (i != 0) {
- sb.append(",");
- }
- Type field = type.getTypeParameters().get(i);
- sb.append(field.getDisplayName())
- .append(":")
- .append(fromTrinoTypeToHiveType(field));
- }
- sb.append(">");
- return sb.toString();
- } else {
- throw new IllegalArgumentException("Cannot transform unknown type:
" + type);
- }
- }
-}
-
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
