This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new a99acfc7a40 [improvement](be-java-extensions) Drop hive-catalog-shade
from be-java-extensions (#65733)
a99acfc7a40 is described below
commit a99acfc7a406a29a08213f2cfa5bed80055181be
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Fri Jul 17 11:47:29 2026 +0800
[improvement](be-java-extensions) Drop hive-catalog-shade from
be-java-extensions (#65733)
Remove the two consumers of the ~122MB
org.apache.doris:hive-catalog-shade fat jar under fe/be-java-extensions.
1. Delete the dead avro-scanner module. Its AvroJNIScanner is never
routed to (no FE code references the class name it would be loaded by),
it is absent from the shared preload-extensions jar, and its only
regression test (test_tvf_avro.groovy) is entirely commented out.
Removes the module, its be-java-extensions/pom.xml and build.sh
registrations, and the dead test plus its .out data.
2. Slim java-udf: replace the hive-catalog-shade dependency with a new
minimal hive-udf-shade module that shades hive-exec:3.1.3:core down to
only the Hive UDF contract classes a user UDF extends -- ql.exec.UDF and
its load-time closure (DefaultUDFMethodResolver/UDFMethodResolver),
ql.exec.Description, and ql.metadata.HiveException. The shaded jar is 20
KB / 10 classes; java-udf's fat jar goes from 12807 hive classes to 10
(all metastore/serde2/objectinspector/relocated-thrift dropped).
The fe/pom.xml version pin and the fe-core / fe-connector-hms /
fe-common consumers still depend on hive-catalog-shade and are
intentionally left untouched (out of scope).
---
build.sh | 4 +-
fe/be-java-extensions/avro-scanner/pom.xml | 116 -----------
.../org/apache/doris/avro/AvroColumnValue.java | 204 -------------------
.../org/apache/doris/avro/AvroFileContext.java | 61 ------
.../java/org/apache/doris/avro/AvroJNIScanner.java | 225 ---------------------
.../java/org/apache/doris/avro/AvroProperties.java | 45 -----
.../java/org/apache/doris/avro/AvroReader.java | 111 ----------
.../java/org/apache/doris/avro/AvroTypeUtils.java | 126 ------------
.../java/org/apache/doris/avro/HDFSFileReader.java | 80 --------
.../java/org/apache/doris/avro/S3FileReader.java | 103 ----------
.../main/java/org/apache/doris/avro/S3Utils.java | 102 ----------
.../avro-scanner/src/main/resources/package.xml | 41 ----
.../org/apache/doris/avro/AvroTypeUtilsTest.java | 105 ----------
fe/be-java-extensions/hive-udf-shade/pom.xml | 114 +++++++++++
fe/be-java-extensions/java-udf/pom.xml | 3 +-
fe/be-java-extensions/pom.xml | 2 +-
.../data/external_table_p0/tvf/test_tvf_avro.out | 97 ---------
.../external_table_p0/tvf/test_tvf_avro.groovy | 211 -------------------
18 files changed, 118 insertions(+), 1632 deletions(-)
diff --git a/build.sh b/build.sh
index 0b777aabff5..b10d7898923 100755
--- a/build.sh
+++ b/build.sh
@@ -99,7 +99,7 @@ Usage: $0 <options>
$0 --be --fe build Backend, Frontend, and Java
UDF library
$0 --be --coverage build Backend with coverage enabled
$0 --be --output PATH build Backend, the result will be
output to PATH(relative paths are available)
- $0 --be-extension-ignore avro-scanner build be-java-extensions, choose
which modules to ignore. Multiple modules separated by commas, like
--be-extension-ignore avro-scanner,hadoop-hudi-scanner
+ $0 --be-extension-ignore paimon-scanner build be-java-extensions, choose
which modules to ignore. Multiple modules separated by commas, like
--be-extension-ignore paimon-scanner,hadoop-hudi-scanner
USE_AVX2=0 $0 --be build Backend and not using AVX2
instruction.
USE_AVX2=0 STRIP_DEBUG_INFO=ON $0 build all and not using AVX2
instruction, and strip the debug info for Backend
@@ -748,7 +748,6 @@ if [[ "${BUILD_BE_JAVA_EXTENSIONS}" -eq 1 ]]; then
modules+=("be-java-extensions/paimon-scanner")
modules+=("be-java-extensions/trino-connector-scanner")
modules+=("be-java-extensions/max-compute-connector")
- modules+=("be-java-extensions/avro-scanner")
# lakesoul-scanner has been deprecated
# modules+=("be-java-extensions/lakesoul-scanner")
modules+=("be-java-extensions/preload-extensions")
@@ -1169,7 +1168,6 @@ EOF
extensions_modules+=("paimon-scanner")
extensions_modules+=("trino-connector-scanner")
extensions_modules+=("max-compute-connector")
- extensions_modules+=("avro-scanner")
# lakesoul-scanner has been deprecated
# extensions_modules+=("lakesoul-scanner")
extensions_modules+=("preload-extensions")
diff --git a/fe/be-java-extensions/avro-scanner/pom.xml
b/fe/be-java-extensions/avro-scanner/pom.xml
deleted file mode 100644
index 4f2d706ba36..00000000000
--- a/fe/be-java-extensions/avro-scanner/pom.xml
+++ /dev/null
@@ -1,116 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0";
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
- <parent>
- <artifactId>be-java-extensions</artifactId>
- <groupId>org.apache.doris</groupId>
- <version>${revision}</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>avro-scanner</artifactId>
-
- <properties>
- <maven.compiler.source>8</maven.compiler.source>
- <maven.compiler.target>8</maven.compiler.target>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.doris</groupId>
- <artifactId>java-common</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libthrift</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-mapred</artifactId>
- <version>${avro.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-tools</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.doris</groupId>
- <artifactId>hive-catalog-shade</artifactId>
- <scope>provided</scope>
- </dependency>
- </dependencies>
-
-
- <build>
- <finalName>avro-scanner</finalName>
- <directory>${project.basedir}/target/</directory>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <descriptors>
- <descriptor>src/main/resources/package.xml</descriptor>
- </descriptors>
- <archive>
- <manifest>
- <mainClass></mainClass>
- </manifest>
- </archive>
- </configuration>
- <executions>
- <execution>
- <id>make-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git
a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroColumnValue.java
b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroColumnValue.java
deleted file mode 100644
index 70070ee2cb8..00000000000
---
a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroColumnValue.java
+++ /dev/null
@@ -1,204 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.avro;
-
-import org.apache.doris.common.jni.vec.ColumnValue;
-
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericData.Fixed;
-import org.apache.avro.generic.GenericFixed;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StructField;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.nio.ByteBuffer;
-import java.time.LocalDate;
-import java.time.LocalDateTime;
-import java.util.List;
-import java.util.Map.Entry;
-import java.util.Objects;
-
-public class AvroColumnValue implements ColumnValue {
-
- private final Object fieldData;
- private final ObjectInspector fieldInspector;
-
- public AvroColumnValue(ObjectInspector fieldInspector, Object fieldData) {
- this.fieldInspector = fieldInspector;
- this.fieldData = fieldData;
- }
-
- private Object inspectObject() {
- if (fieldData instanceof ByteBuffer) {
- return ((PrimitiveObjectInspector)
fieldInspector).getPrimitiveJavaObject(((ByteBuffer) fieldData).array());
- } else if (fieldData instanceof Fixed) {
- return ((PrimitiveObjectInspector)
fieldInspector).getPrimitiveJavaObject(
- ((GenericFixed) fieldData).bytes());
- }
- return ((PrimitiveObjectInspector)
fieldInspector).getPrimitiveJavaObject(fieldData);
- }
-
- @Override
- public boolean canGetStringAsBytes() {
- return false;
- }
-
- @Override
- public boolean isNull() {
- return false;
- }
-
- @Override
- public boolean getBoolean() {
- return (boolean) inspectObject();
- }
-
- @Override
- public byte getByte() {
- return (byte) inspectObject();
- }
-
- @Override
- public short getShort() {
- return (short) inspectObject();
- }
-
- @Override
- public int getInt() {
- return (int) inspectObject();
- }
-
- @Override
- public float getFloat() {
- return (float) inspectObject();
- }
-
- @Override
- public long getLong() {
- return (long) inspectObject();
- }
-
- @Override
- public double getDouble() {
- return (double) inspectObject();
- }
-
- @Override
- public BigInteger getBigInteger() {
- return null;
- }
-
- @Override
- public BigDecimal getDecimal() {
- return (BigDecimal) inspectObject();
- }
-
- @Override
- public String getString() {
- return inspectObject().toString();
- }
-
- @Override
- public byte[] getStringAsBytes() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public LocalDate getDate() {
- // avro has no date type
- return null;
- }
-
- @Override
- public LocalDateTime getDateTime() {
- // avro has no dateTime type
- return null;
- }
-
- @Override
- public LocalDateTime getTimeStampTz() {
- return null;
- }
-
- @Override
- public byte[] getBytes() {
- return (byte[]) inspectObject();
- }
-
- @Override
- public void unpackArray(List<ColumnValue> values) {
- ListObjectInspector inspector = (ListObjectInspector) fieldInspector;
- List<?> items = inspector.getList(fieldData);
- ObjectInspector itemInspector =
inspector.getListElementObjectInspector();
- for (Object item : items) {
- AvroColumnValue avroColumnValue = null;
- if (item != null) {
- avroColumnValue = new AvroColumnValue(itemInspector, item);
- }
- values.add(avroColumnValue);
- }
- }
-
- @Override
- public void unpackMap(List<ColumnValue> keys, List<ColumnValue> values) {
- MapObjectInspector inspector = (MapObjectInspector) fieldInspector;
- ObjectInspector keyObjectInspector =
inspector.getMapKeyObjectInspector();
- ObjectInspector valueObjectInspector =
inspector.getMapValueObjectInspector();
- for (Entry<?, ?> kv : inspector.getMap(fieldData).entrySet()) {
- AvroColumnValue avroKey = null;
- AvroColumnValue avroValue = null;
- if (kv.getKey() != null) {
- avroKey = new AvroColumnValue(keyObjectInspector, kv.getKey());
- }
- if (kv.getValue() != null) {
- avroValue = new AvroColumnValue(valueObjectInspector,
kv.getValue());
- }
- keys.add(avroKey);
- values.add(avroValue);
- }
- }
-
- @Override
- public void unpackStruct(List<Integer> structFieldIndex, List<ColumnValue>
values) {
- StructObjectInspector inspector = (StructObjectInspector)
fieldInspector;
- List<? extends StructField> fields = inspector.getAllStructFieldRefs();
- for (Integer idx : structFieldIndex) {
- AvroColumnValue cv = null;
- if (idx != null) {
- StructField sf = fields.get(idx);
- Object o;
- if (fieldData instanceof GenericData.Record) {
- GenericRecord record = (GenericRecord)
inspector.getStructFieldData(fieldData, sf);
- o = record.get(idx);
- } else {
- o = inspector.getStructFieldData(fieldData, sf);
- }
- if (Objects.nonNull(o)) {
- cv = new AvroColumnValue(sf.getFieldObjectInspector(), o);
- }
- }
- values.add(cv);
- }
- }
-}
diff --git
a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroFileContext.java
b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroFileContext.java
deleted file mode 100644
index 3e3264e6291..00000000000
---
a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroFileContext.java
+++ /dev/null
@@ -1,61 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.avro;
-
-import org.apache.avro.Schema;
-
-import java.util.Set;
-
-public class AvroFileContext {
- private Schema schema;
- private Set<String> requiredFields;
- private Long splitStartOffset;
- private Long splitSize;
-
- public void setSchema(Schema schema) {
- this.schema = schema;
- }
-
- public Schema getSchema() {
- return schema;
- }
-
- public void setRequiredFields(Set<String> requiredFields) {
- this.requiredFields = requiredFields;
- }
-
- public void setSplitStartOffset(Long splitStartOffset) {
- this.splitStartOffset = splitStartOffset;
- }
-
- public void setSplitSize(Long splitSize) {
- this.splitSize = splitSize;
- }
-
- public Long getSplitStartOffset() {
- return this.splitStartOffset;
- }
-
- public Long getSplitSize() {
- return this.splitSize;
- }
-
- public Set<String> getRequiredFields() {
- return requiredFields;
- }
-}
diff --git
a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroJNIScanner.java
b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroJNIScanner.java
deleted file mode 100644
index 2ef0bf1d45a..00000000000
---
a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroJNIScanner.java
+++ /dev/null
@@ -1,225 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.avro;
-
-import org.apache.doris.common.jni.JniScanner;
-import org.apache.doris.common.jni.vec.ColumnType;
-import org.apache.doris.common.jni.vec.TableSchema;
-import org.apache.doris.thrift.TFileType;
-
-import org.apache.avro.Schema;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.avro.mapred.AvroWrapper;
-import org.apache.avro.mapred.Pair;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.common.JavaUtils;
-import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
-import org.apache.hadoop.hive.serde2.Deserializer;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StructField;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.Logger;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Properties;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-public class AvroJNIScanner extends JniScanner {
-
- private static final Logger LOG =
LogManager.getLogger(AvroJNIScanner.class);
- private final TFileType fileType;
- private final String uri;
- private final Map<String, String> requiredParams;
- private final Integer fetchSize;
- private final ClassLoader classLoader;
- private int[] requiredColumnIds;
- private String[] columnTypes;
- private String[] requiredFields;
- private Set<String> requiredFieldSet;
- private ColumnType[] requiredTypes;
- private AvroReader avroReader;
- private final boolean isGetTableSchema;
- private StructObjectInspector rowInspector;
- private Deserializer deserializer;
- private StructField[] structFields;
- private ObjectInspector[] fieldInspectors;
- private String serde;
- private AvroFileContext avroFileContext;
- private AvroWrapper<Pair<Integer, Long>> inputPair;
- private NullWritable ignore;
- private Long splitStartOffset;
- private Long splitSize;
- private Long splitFileSize;
-
- /**
- * Call by JNI for get table data or get table schema
- *
- * @param fetchSize The size of data fetched each time
- * @param requiredParams required params
- */
- public AvroJNIScanner(int fetchSize, Map<String, String> requiredParams) {
- this.classLoader = this.getClass().getClassLoader();
- this.requiredParams = requiredParams;
- this.fetchSize = fetchSize;
- this.isGetTableSchema =
Boolean.parseBoolean(requiredParams.get(AvroProperties.IS_GET_TABLE_SCHEMA));
- this.fileType =
TFileType.findByValue(Integer.parseInt(requiredParams.get(AvroProperties.FILE_TYPE)));
- this.uri = requiredParams.get(AvroProperties.URI);
- if (!isGetTableSchema) {
- this.columnTypes = requiredParams.get(AvroProperties.COLUMNS_TYPES)
- .split(AvroProperties.COLUMNS_TYPE_DELIMITER);
- this.requiredFields =
requiredParams.get(AvroProperties.REQUIRED_FIELDS)
- .split(AvroProperties.FIELDS_DELIMITER);
- this.requiredFieldSet = new
HashSet<>(Arrays.asList(requiredFields));
- this.requiredTypes = new ColumnType[requiredFields.length];
- this.serde = requiredParams.get(AvroProperties.HIVE_SERDE);
- this.structFields = new StructField[requiredFields.length];
- this.fieldInspectors = new ObjectInspector[requiredFields.length];
- this.inputPair = new AvroWrapper<>(null);
- this.ignore = NullWritable.get();
- this.splitStartOffset =
Long.parseLong(requiredParams.get(AvroProperties.SPLIT_START_OFFSET));
- this.splitSize =
Long.parseLong(requiredParams.get(AvroProperties.SPLIT_SIZE));
- this.splitFileSize =
Long.parseLong(requiredParams.get(AvroProperties.SPLIT_FILE_SIZE));
- }
- }
-
- private void initFieldInspector() throws Exception {
- requiredColumnIds = new int[requiredFields.length];
- for (int i = 0; i < requiredFields.length; i++) {
- ColumnType columnType = ColumnType.parseType(requiredFields[i],
columnTypes[i]);
- requiredTypes[i] = columnType;
- requiredColumnIds[i] = i;
- }
-
- Properties properties = createProperties();
- deserializer = getDeserializer(new Configuration(), properties,
this.serde);
- rowInspector = (StructObjectInspector)
deserializer.getObjectInspector();
-
- for (int i = 0; i < requiredFields.length; i++) {
- StructField field =
rowInspector.getStructFieldRef(requiredFields[i]);
- structFields[i] = field;
- fieldInspectors[i] = field.getFieldObjectInspector();
- }
- }
-
- public Properties createProperties() {
- Properties properties = new Properties();
- properties.setProperty(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR,
-
Arrays.stream(this.requiredColumnIds).mapToObj(String::valueOf).collect(Collectors.joining(",")));
-
properties.setProperty(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR,
String.join(",", requiredFields));
- properties.setProperty(AvroProperties.COLUMNS, String.join(",",
requiredFields));
- properties.setProperty(AvroProperties.COLUMNS2TYPES, String.join(",",
columnTypes));
- properties.setProperty(serdeConstants.SERIALIZATION_LIB, this.serde);
- return properties;
- }
-
- private Deserializer getDeserializer(Configuration configuration,
Properties properties, String name)
- throws Exception {
- Class<? extends Deserializer> deserializerClass = Class.forName(name,
true, JavaUtils.getClassLoader())
- .asSubclass(Deserializer.class);
- Deserializer deserializer =
deserializerClass.getConstructor().newInstance();
- deserializer.initialize(configuration, properties);
- return deserializer;
- }
-
- @Override
- public void open() throws IOException {
- Thread.currentThread().setContextClassLoader(classLoader);
- switch (fileType) {
- case FILE_HDFS:
- this.avroReader = new HDFSFileReader(uri);
- break;
- case FILE_S3:
- String accessKey =
requiredParams.get(AvroProperties.S3_ACCESS_KEY);
- String secretKey =
requiredParams.get(AvroProperties.S3_SECRET_KEY);
- String endpoint =
requiredParams.get(AvroProperties.S3_ENDPOINT);
- String region = requiredParams.get(AvroProperties.S3_REGION);
- this.avroReader = new S3FileReader(accessKey, secretKey,
endpoint, region, uri);
- break;
- default:
- LOG.warn("Unsupported " + fileType.name() + " file type.");
- throw new IOException("Unsupported " + fileType.name() + "
file type.");
- }
- if (!isGetTableSchema) {
- initDataReader();
- }
- this.avroReader.open(avroFileContext, isGetTableSchema);
- }
-
- private void initDataReader() {
- try {
- initAvroFileContext();
- initFieldInspector();
- initTableInfo(requiredTypes, requiredFields, fetchSize);
- } catch (Exception e) {
- LOG.warn("Failed to init avro scanner. ", e);
- throw new RuntimeException(e);
- }
- }
-
- private void initAvroFileContext() {
- avroFileContext = new AvroFileContext();
- avroFileContext.setRequiredFields(requiredFieldSet);
- avroFileContext.setSplitStartOffset(splitStartOffset);
- avroFileContext.setSplitSize(splitSize);
- }
-
- @Override
- public void close() throws IOException {
- if (Objects.nonNull(avroReader)) {
- avroReader.close();
- }
- }
-
- @Override
- protected int getNext() throws IOException {
- int numRows = 0;
- long startTime = System.nanoTime();
- for (; numRows < getBatchSize(); numRows++) {
- if (!avroReader.hasNext(inputPair, ignore)) {
- break;
- }
- GenericRecord rowRecord = (GenericRecord) avroReader.getNext();
- for (int i = 0; i < requiredFields.length; i++) {
- Object fieldData = rowRecord.get(requiredFields[i]);
- if (fieldData == null) {
- appendData(i, null);
- } else {
- AvroColumnValue fieldValue = new
AvroColumnValue(fieldInspectors[i], fieldData);
- appendData(i, fieldValue);
- }
- }
- }
- appendDataTime += System.nanoTime() - startTime;
- return numRows;
- }
-
- @Override
- protected TableSchema parseTableSchema() throws
UnsupportedOperationException {
- Schema schema = avroReader.getSchema();
- return AvroTypeUtils.parseTableSchema(schema);
- }
-
-}
diff --git
a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroProperties.java
b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroProperties.java
deleted file mode 100644
index 416b7d25897..00000000000
---
a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroProperties.java
+++ /dev/null
@@ -1,45 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.avro;
-
-public class AvroProperties {
-
- protected static final String COLUMNS_TYPE_DELIMITER = "#";
- protected static final String FIELDS_DELIMITER = ",";
-
- protected static final String IS_GET_TABLE_SCHEMA = "is_get_table_schema";
- protected static final String COLUMNS_TYPES = "columns_types";
- protected static final String REQUIRED_FIELDS = "required_fields";
- protected static final String FILE_TYPE = "file_type";
- protected static final String URI = "uri";
- protected static final String S3_ACCESS_KEY = "s3.access_key";
- protected static final String S3_SECRET_KEY = "s3.secret_key";
- protected static final String S3_ENDPOINT = "s3.endpoint";
- protected static final String S3_REGION = "s3.region";
- protected static final String HIVE_SERDE = "hive.serde";
- protected static final String COLUMNS = "columns";
- protected static final String COLUMNS2TYPES = "columns.types";
- protected static final String FS_S3A_ACCESS_KEY = "fs.s3a.access.key";
- protected static final String FS_S3A_SECRET_KEY = "fs.s3a.secret.key";
- protected static final String FS_S3A_ENDPOINT = "fs.s3a.endpoint";
- protected static final String FS_S3A_REGION = "fs.s3a.region";
- protected static final String SPLIT_START_OFFSET = "split_start_offset";
- protected static final String SPLIT_SIZE = "split_size";
- protected static final String SPLIT_FILE_SIZE = "split_file_size";
-
-}
diff --git
a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroReader.java
b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroReader.java
deleted file mode 100644
index ac8d4fd621d..00000000000
---
a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroReader.java
+++ /dev/null
@@ -1,111 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.avro;
-
-import com.google.gson.Gson;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonObject;
-import org.apache.avro.Schema;
-import org.apache.avro.Schema.Parser;
-import org.apache.avro.file.DataFileStream;
-import org.apache.avro.generic.GenericDatumReader;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.avro.mapred.AvroJob;
-import org.apache.avro.mapred.AvroRecordReader;
-import org.apache.avro.mapred.AvroWrapper;
-import org.apache.avro.mapred.Pair;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapred.FileSplit;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.Logger;
-
-import java.io.BufferedInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Set;
-
-public abstract class AvroReader {
-
- private static final Logger LOG = LogManager.getLogger(AvroReader.class);
- protected AvroRecordReader<Pair<Integer, Long>> dataReader;
- protected DataFileStream<GenericRecord> schemaReader;
- protected Path path;
- protected FileSystem fileSystem;
-
- public abstract void open(AvroFileContext avroFileContext, boolean
tableSchema) throws IOException;
-
- public abstract Schema getSchema();
-
- public abstract boolean hasNext(AvroWrapper<Pair<Integer, Long>>
inputPair, NullWritable ignore) throws IOException;
-
- public abstract Object getNext();
-
- public abstract void close() throws IOException;
-
- protected void openSchemaReader() throws IOException {
- InputStream inputStream = new
BufferedInputStream(fileSystem.open(path));
- schemaReader = new DataFileStream<>(inputStream, new
GenericDatumReader<>());
- if (LOG.isDebugEnabled()) {
- LOG.debug("success open avro schema reader.");
- }
- }
-
- protected void openDataReader(AvroFileContext avroFileContext) throws
IOException {
- JobConf job = new JobConf();
- projectionSchema(job, avroFileContext);
- FileSplit fileSplit =
- new FileSplit(path, avroFileContext.getSplitStartOffset(),
avroFileContext.getSplitSize(), job);
- dataReader = new AvroRecordReader<>(job, fileSplit);
- if (LOG.isDebugEnabled()) {
- LOG.debug("success open avro data reader.");
- }
- }
-
- protected void projectionSchema(JobConf job, AvroFileContext
avroFileContext) {
- Schema projectionSchema;
- Set<String> filedNames = avroFileContext.getRequiredFields();
- Schema avroSchema = avroFileContext.getSchema();
- // The number of fields that need to be queried is the same as that of
the avro file,
- // so no projection is required.
- if (filedNames.size() != avroSchema.getFields().size()) {
- JsonObject schemaJson = new Gson().fromJson(avroSchema.toString(),
JsonObject.class);
- JsonArray schemaFields = schemaJson.get("fields").getAsJsonArray();
- JsonObject copySchemaJson = schemaJson.deepCopy();
- JsonArray copySchemaFields =
copySchemaJson.get("fields").getAsJsonArray();
- for (int i = 0; i < schemaFields.size(); i++) {
- JsonObject object = schemaFields.get(i).getAsJsonObject();
- String name = object.get("name").getAsString();
- if (filedNames.contains(name)) {
- continue;
- }
- copySchemaFields.remove(schemaFields.get(i));
- }
- projectionSchema = new Parser().parse(copySchemaJson.toString());
- } else {
- projectionSchema = avroSchema;
- }
- AvroJob.setInputSchema(job, projectionSchema);
- if (LOG.isDebugEnabled()) {
- LOG.debug("projection avro schema is:" +
projectionSchema.toString());
- }
- }
-
-}
diff --git
a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroTypeUtils.java
b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroTypeUtils.java
deleted file mode 100644
index 32ead116dd1..00000000000
---
a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroTypeUtils.java
+++ /dev/null
@@ -1,126 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.avro;
-
-import org.apache.doris.common.jni.vec.TableSchema;
-import org.apache.doris.common.jni.vec.TableSchema.SchemaColumn;
-import org.apache.doris.thrift.TPrimitiveType;
-
-import com.google.common.base.Preconditions;
-import org.apache.avro.Schema;
-import org.apache.avro.Schema.Field;
-import org.apache.commons.compress.utils.Lists;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.Logger;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.stream.Collectors;
-
-public class AvroTypeUtils {
-
- private static final Logger LOG =
LogManager.getLogger(AvroTypeUtils.class);
-
- protected static TableSchema parseTableSchema(Schema schema) throws
UnsupportedOperationException {
- List<Field> schemaFields = schema.getFields();
- List<SchemaColumn> schemaColumns = new ArrayList<>();
- for (Field schemaField : schemaFields) {
- Schema avroSchema = schemaField.schema();
- String columnName = schemaField.name();
-
- SchemaColumn schemaColumn = new SchemaColumn();
- TPrimitiveType tPrimitiveType = typeFromAvro(avroSchema,
schemaColumn);
- schemaColumn.setName(columnName);
- schemaColumn.setType(tPrimitiveType);
- schemaColumns.add(schemaColumn);
- }
- return new TableSchema(schemaColumns);
- }
-
- private static TPrimitiveType typeFromAvro(Schema avroSchema, SchemaColumn
schemaColumn)
- throws UnsupportedOperationException {
- Schema.Type type = avroSchema.getType();
- switch (type) {
- case ENUM:
- case STRING:
- return TPrimitiveType.STRING;
- case INT:
- return TPrimitiveType.INT;
- case BOOLEAN:
- return TPrimitiveType.BOOLEAN;
- case LONG:
- return TPrimitiveType.BIGINT;
- case FLOAT:
- return TPrimitiveType.FLOAT;
- case FIXED:
- case BYTES:
- return TPrimitiveType.BINARY;
- case DOUBLE:
- return TPrimitiveType.DOUBLE;
- case ARRAY:
- SchemaColumn arrayChildColumn = new SchemaColumn();
-
schemaColumn.addChildColumns(Collections.singletonList(arrayChildColumn));
-
arrayChildColumn.setType(typeFromAvro(avroSchema.getElementType(),
arrayChildColumn));
- return TPrimitiveType.ARRAY;
- case MAP:
- // The default type of AVRO MAP structure key is STRING
- SchemaColumn keyChildColumn = new SchemaColumn();
- keyChildColumn.setType(TPrimitiveType.STRING);
- SchemaColumn valueChildColumn = new SchemaColumn();
-
valueChildColumn.setType(typeFromAvro(avroSchema.getValueType(),
valueChildColumn));
-
- schemaColumn.addChildColumns(Arrays.asList(keyChildColumn,
valueChildColumn));
- return TPrimitiveType.MAP;
- case RECORD:
- List<Field> fields = avroSchema.getFields();
- List<SchemaColumn> childSchemaColumn = Lists.newArrayList();
- for (Field field : fields) {
- SchemaColumn structChildColumn = new SchemaColumn();
- structChildColumn.setName(field.name());
- structChildColumn.setType(typeFromAvro(field.schema(),
structChildColumn));
- childSchemaColumn.add(structChildColumn);
- }
- schemaColumn.addChildColumns(childSchemaColumn);
- return TPrimitiveType.STRUCT;
- case UNION:
- List<Schema> nonNullableMembers =
filterNullableUnion(avroSchema);
- Preconditions.checkArgument(!nonNullableMembers.isEmpty(),
- avroSchema.getName() + "Union child type not all
nullAble type");
- List<SchemaColumn> childSchemaColumns = Lists.newArrayList();
- for (Schema nullableMember : nonNullableMembers) {
- SchemaColumn childColumn = new SchemaColumn();
- childColumn.setName(nullableMember.getName());
- childColumn.setType(typeFromAvro(nullableMember,
childColumn));
- childSchemaColumns.add(childColumn);
- }
- schemaColumn.addChildColumns(childSchemaColumns);
- return TPrimitiveType.STRUCT;
- default:
- throw new UnsupportedOperationException(
- "avro format: " + avroSchema.getName() +
type.getName() + " is not supported.");
- }
- }
-
- private static List<Schema> filterNullableUnion(Schema schema) {
- Preconditions.checkArgument(schema.isUnion(), "Schema must be union");
- return schema.getTypes().stream().filter(s ->
!s.isNullable()).collect(Collectors.toList());
- }
-
-}
diff --git
a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/HDFSFileReader.java
b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/HDFSFileReader.java
deleted file mode 100644
index be9f355912a..00000000000
---
a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/HDFSFileReader.java
+++ /dev/null
@@ -1,80 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.avro;
-
-import org.apache.avro.Schema;
-import org.apache.avro.mapred.AvroWrapper;
-import org.apache.avro.mapred.Pair;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.Logger;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.Objects;
-
-public class HDFSFileReader extends AvroReader {
- private static final Logger LOG =
LogManager.getLogger(HDFSFileReader.class);
- private final String url;
- private AvroWrapper<Pair<Integer, Long>> inputPair;
-
- public HDFSFileReader(String url) {
- this.url = url;
- this.path = new Path(url);
- }
-
- @Override
- public void open(AvroFileContext avroFileContext, boolean tableSchema)
throws IOException {
- fileSystem = FileSystem.get(URI.create(url), new Configuration());
- openSchemaReader();
- if (!tableSchema) {
- avroFileContext.setSchema(schemaReader.getSchema());
- openDataReader(avroFileContext);
- }
- }
-
- @Override
- public Schema getSchema() {
- return schemaReader.getSchema();
- }
-
- @Override
- public boolean hasNext(AvroWrapper<Pair<Integer, Long>> inputPair,
NullWritable ignore) throws IOException {
- this.inputPair = inputPair;
- return dataReader.next(this.inputPair, ignore);
- }
-
- @Override
- public Object getNext() {
- return inputPair.datum();
- }
-
- @Override
- public void close() throws IOException {
- if (Objects.nonNull(schemaReader)) {
- schemaReader.close();
- }
- if (Objects.nonNull(dataReader)) {
- dataReader.close();
- }
- fileSystem.close();
- }
-}
diff --git
a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3FileReader.java
b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3FileReader.java
deleted file mode 100644
index ae31eb815d1..00000000000
---
a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3FileReader.java
+++ /dev/null
@@ -1,103 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.avro;
-
-import org.apache.avro.Schema;
-import org.apache.avro.mapred.AvroWrapper;
-import org.apache.avro.mapred.Pair;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.Logger;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.Objects;
-
-public class S3FileReader extends AvroReader {
-
- private static final Logger LOG = LogManager.getLogger(S3FileReader.class);
- private final String bucketName;
- private final String key;
- private AvroWrapper<Pair<Integer, Long>> inputPair;
- private final String endpoint;
- private final String region;
- private final String accessKey;
- private final String secretKey;
- private final String s3aUri;
-
- public S3FileReader(String accessKey, String secretKey, String endpoint,
String region, String uri)
- throws IOException {
- this.endpoint = endpoint;
- this.region = region;
- S3Utils.parseURI(uri);
- this.bucketName = S3Utils.getBucket();
- this.key = S3Utils.getKey();
- this.accessKey = accessKey;
- this.secretKey = secretKey;
- this.s3aUri = "s3a://" + bucketName + "/" + key;
- }
-
- @Override
- public void open(AvroFileContext avroFileContext, boolean tableSchema)
throws IOException {
- Configuration conf = new Configuration();
- if (!StringUtils.isEmpty(accessKey) &&
!StringUtils.isEmpty(secretKey)) {
- conf.set(AvroProperties.FS_S3A_ACCESS_KEY, accessKey);
- conf.set(AvroProperties.FS_S3A_SECRET_KEY, secretKey);
- }
- conf.set(AvroProperties.FS_S3A_ENDPOINT, endpoint);
- conf.set(AvroProperties.FS_S3A_REGION, region);
- path = new Path(s3aUri);
- fileSystem = FileSystem.get(URI.create(s3aUri), conf);
- openSchemaReader();
- if (!tableSchema) {
- avroFileContext.setSchema(schemaReader.getSchema());
- openDataReader(avroFileContext);
- }
- }
-
- @Override
- public Schema getSchema() {
- return schemaReader.getSchema();
- }
-
- @Override
- public boolean hasNext(AvroWrapper<Pair<Integer, Long>> inputPair,
NullWritable ignore) throws IOException {
- this.inputPair = inputPair;
- return dataReader.next(this.inputPair, ignore);
- }
-
- @Override
- public Object getNext() {
- return inputPair.datum();
- }
-
- @Override
- public void close() throws IOException {
- if (Objects.nonNull(schemaReader)) {
- schemaReader.close();
- }
- if (Objects.nonNull(dataReader)) {
- dataReader.close();
- }
- fileSystem.close();
- }
-}
diff --git
a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3Utils.java
b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3Utils.java
deleted file mode 100644
index 45845af3c03..00000000000
---
a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3Utils.java
+++ /dev/null
@@ -1,102 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.avro;
-
-import org.apache.commons.lang3.StringUtils;
-
-import java.io.IOException;
-
-public class S3Utils {
- private static final String SCHEMA_S3 = "s3";
- private static final String SCHEMA_HTTP = "http";
- private static final String SCHEMA_HTTPS = "https";
- private static final String SCHEME_DELIM = "://";
- private static final String PATH_DELIM = "/";
- private static final String QUERY_DELIM = "\\?";
- private static final String FRAGMENT_DELIM = "#";
- private static String bucket;
- private static String key;
-
- /**
- * eg:
- * s3: s3://bucket1/path/to/file.txt
- * http: http://10.10.10.1:9000/bucket1/to/file.txt
- * https: https://10.10.10.1:9000/bucket1/to/file.txt
- * <p>
- * schema: s3,http,https
- * bucket: bucket1
- * key: path/to/file.txt
- */
- public static void parseURI(String uri) throws IOException {
- if (StringUtils.isEmpty(uri)) {
- throw new IOException("s3 uri is empty.");
- }
- String[] schemeSplit = uri.split(SCHEME_DELIM);
- String rest;
- if (schemeSplit.length == 2) {
- if (schemeSplit[0].equalsIgnoreCase(SCHEMA_S3)) {
- // has scheme, eg: s3://bucket1/path/to/file.txt
- rest = schemeSplit[1];
- String[] authoritySplit = rest.split(PATH_DELIM, 2);
- if (authoritySplit.length < 1) {
- throw new IOException("Invalid S3 URI. uri=" + uri);
- }
- bucket = authoritySplit[0];
- // support s3://bucket1
- key = authoritySplit.length == 1 ? "/" : authoritySplit[1];
- } else if (schemeSplit[0].equalsIgnoreCase(SCHEMA_HTTP) ||
schemeSplit[0].equalsIgnoreCase(SCHEMA_HTTPS)) {
- // has scheme, eg: http(s)://host/bucket1/path/to/file.txt
- rest = schemeSplit[1];
- String[] authoritySplit = rest.split(PATH_DELIM, 3);
- if (authoritySplit.length != 3) {
- throw new IOException("Invalid S3 HTTP URI: uri=" + uri);
- }
- // authority_split[1] is host
- bucket = authoritySplit[1];
- key = authoritySplit[2];
- } else {
- throw new IOException("Invalid S3 HTTP URI: uri=" + uri);
- }
-
- } else if (schemeSplit.length == 1) {
- // no scheme, eg: path/to/file.txt
- bucket = ""; // unknown
- key = uri;
- } else {
- throw new IOException("Invalid S3 URI. uri=" + uri);
- }
-
- key = key.trim();
- if (StringUtils.isEmpty(key)) {
- throw new IOException("Invalid S3 URI. uri=" + uri);
- }
- // Strip query and fragment if they exist
- String[] querySplit = key.split(QUERY_DELIM);
- String[] fragmentSplit = querySplit[0].split(FRAGMENT_DELIM);
- key = fragmentSplit[0];
- }
-
- public static String getBucket() {
- return bucket;
- }
-
- public static String getKey() {
- return key;
- }
-
-}
diff --git a/fe/be-java-extensions/avro-scanner/src/main/resources/package.xml
b/fe/be-java-extensions/avro-scanner/src/main/resources/package.xml
deleted file mode 100644
index 4bbb2610603..00000000000
--- a/fe/be-java-extensions/avro-scanner/src/main/resources/package.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.0.0";
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
- xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.0.0
http://maven.apache.org/xsd/assembly-2.0.0.xsd";>
- <id>jar-with-dependencies</id>
- <formats>
- <format>jar</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <dependencySets>
- <dependencySet>
- <outputDirectory>/</outputDirectory>
- <useProjectArtifact>true</useProjectArtifact>
- <unpack>true</unpack>
- <scope>runtime</scope>
- <unpackOptions>
- <excludes>
- <exclude>**/Log4j2Plugins.dat</exclude>
- </excludes>
- </unpackOptions>
- </dependencySet>
- </dependencySets>
-</assembly>
diff --git
a/fe/be-java-extensions/avro-scanner/src/test/java/org/apache/doris/avro/AvroTypeUtilsTest.java
b/fe/be-java-extensions/avro-scanner/src/test/java/org/apache/doris/avro/AvroTypeUtilsTest.java
deleted file mode 100644
index 04f5fd217bf..00000000000
---
a/fe/be-java-extensions/avro-scanner/src/test/java/org/apache/doris/avro/AvroTypeUtilsTest.java
+++ /dev/null
@@ -1,105 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.avro;
-
-import org.apache.doris.common.jni.vec.TableSchema;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.avro.Schema;
-import org.apache.avro.SchemaBuilder;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-
-public class AvroTypeUtilsTest {
- private Schema allTypesRecordSchema;
- private final ObjectMapper objectMapper = new ObjectMapper();
- private String result;
-
- @Before
- public void setUp() {
- result =
"[{\"name\":\"aBoolean\",\"type\":2,\"childColumns\":null},{\"name\":\"aInt\",\"type\":5,"
- +
"\"childColumns\":null},{\"name\":\"aLong\",\"type\":6,\"childColumns\":null},{\"name\":\""
- +
"aFloat\",\"type\":7,\"childColumns\":null},{\"name\":\"aDouble\",\"type\":8,\"childColumns\""
- +
":null},{\"name\":\"aString\",\"type\":23,\"childColumns\":null},{\"name\":\"aBytes\",\"type\""
- +
":11,\"childColumns\":null},{\"name\":\"aFixed\",\"type\":11,\"childColumns\":null},{\"name\""
- +
":\"anArray\",\"type\":20,\"childColumns\":[{\"name\":null,\"type\":5,\"childColumns\":null}]}"
- +
",{\"name\":\"aMap\",\"type\":21,\"childColumns\":[{\"name\":null,\"type\":23,\"childColumns\""
- +
":null},{\"name\":null,\"type\":5,\"childColumns\":null}]},{\"name\":\"anEnum\",\"type\":23"
- +
",\"childColumns\":null},{\"name\":\"aRecord\",\"type\":22,\"childColumns\":[{\"name\":\"a\","
- +
"\"type\":5,\"childColumns\":null},{\"name\":\"b\",\"type\":8,\"childColumns\":null},{\"name\":"
- +
"\"c\",\"type\":23,\"childColumns\":null}]},{\"name\":\"aUnion\",\"type\":22,\"childColumns\":"
- +
"[{\"name\":\"string\",\"type\":23,\"childColumns\":null}]}]\n";
-
- Schema simpleEnumSchema =
SchemaBuilder.enumeration("myEnumType").symbols("A", "B", "C");
- Schema simpleRecordSchema = SchemaBuilder.record("simpleRecord")
- .fields()
- .name("a")
- .type().intType().noDefault()
- .name("b")
- .type().doubleType().noDefault()
- .name("c")
- .type().stringType().noDefault()
- .endRecord();
-
- allTypesRecordSchema = SchemaBuilder.builder()
- .record("all")
- .fields()
- .name("aBoolean")
- .type().booleanType().noDefault()
- .name("aInt")
- .type().intType().noDefault()
- .name("aLong")
- .type().longType().noDefault()
- .name("aFloat")
- .type().floatType().noDefault()
- .name("aDouble")
- .type().doubleType().noDefault()
- .name("aString")
- .type().stringType().noDefault()
- .name("aBytes")
- .type().bytesType().noDefault()
- .name("aFixed")
- .type().fixed("myFixedType").size(16).noDefault()
- .name("anArray")
- .type().array().items().intType().noDefault()
- .name("aMap")
- .type().map().values().intType().noDefault()
- .name("anEnum")
- .type(simpleEnumSchema).noDefault()
- .name("aRecord")
- .type(simpleRecordSchema).noDefault()
- .name("aUnion")
- .type().optional().stringType()
- .endRecord();
- }
-
- @Test
- public void testParseTableSchema() throws IOException {
- TableSchema tableSchema =
AvroTypeUtils.parseTableSchema(allTypesRecordSchema);
- String tableSchemaTableSchema = tableSchema.getTableSchema();
- JsonNode tableSchemaTree =
objectMapper.readTree(tableSchemaTableSchema);
-
- JsonNode resultSchemaTree = objectMapper.readTree(result);
- Assert.assertEquals(resultSchemaTree, tableSchemaTree);
- }
-
-}
diff --git a/fe/be-java-extensions/hive-udf-shade/pom.xml
b/fe/be-java-extensions/hive-udf-shade/pom.xml
new file mode 100644
index 00000000000..9d638667a67
--- /dev/null
+++ b/fe/be-java-extensions/hive-udf-shade/pom.xml
@@ -0,0 +1,114 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <parent>
+ <artifactId>be-java-extensions</artifactId>
+ <groupId>org.apache.doris</groupId>
+ <version>${revision}</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>hive-udf-shade</artifactId>
+ <packaging>jar</packaging>
+ <name>Doris BE Java Extensions - Hive UDF Contract Shade</name>
+ <description>
+ Minimal shaded jar carrying ONLY the Hive UDF contract classes
+ (org.apache.hadoop.hive.ql.exec.UDF plus its load-time closure) that a
+ user Hive UDF extends. Replaces the ~122MB shaded Hive catalog fat jar
+ that java-udf previously bundled just to provide this handful of
classes.
+ </description>
+
+ <dependencies>
+ <!--
+ Source of the UDF contract classes. Marked optional so java-udf,
which
+ depends on this shaded module, does NOT transitively re-pull the full
+ hive-exec:core jar: the ~handful of classes it needs are already
baked
+ into hive-udf-shade.jar by the shade filter below. The wildcard
+ exclusion prunes hive-exec's heavy transitive tree (calcite/orc/...)
+ from resolution entirely, since we bundle nothing but hive-exec
itself.
+ -->
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <classifier>core</classifier>
+ <version>${hive.version}</version>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <finalName>hive-udf-shade</finalName>
+ <directory>${project.basedir}/target/</directory>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>3.2.4</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+
<createDependencyReducedPom>false</createDependencyReducedPom>
+ <!-- Bundle only hive-exec; the project itself has no
sources. -->
+ <artifactSet>
+ <includes>
+ <include>org.apache.hive:hive-exec</include>
+ </includes>
+ </artifactSet>
+ <!--
+ Keep ONLY the Hive UDF contract + its load-time closure:
+ - ql/exec/UDF* UDF base class,
UDFMethodResolver,
+ UDFArgument*Exception,
UDFClassLoader
+ - ql/exec/DefaultUDFMethodResolver instantiated by
the UDF ctor
+ - ql/exec/Description the @Description
annotation
+ - ql/metadata/HiveException* declared by some user
UDFs
+ FunctionRegistry is referenced only inside the
never-called
+ DefaultUDFMethodResolver.getEvalMethod(), so it is
intentionally
+ excluded. Everything else in hive-exec is dropped.
+ -->
+ <filters>
+ <filter>
+ <artifact>org.apache.hive:hive-exec</artifact>
+ <includes>
+
<include>org/apache/hadoop/hive/ql/exec/UDF*.class</include>
+
<include>org/apache/hadoop/hive/ql/exec/DefaultUDFMethodResolver*.class</include>
+
<include>org/apache/hadoop/hive/ql/exec/Description*.class</include>
+
<include>org/apache/hadoop/hive/ql/metadata/HiveException*.class</include>
+ </includes>
+ </filter>
+ </filters>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/fe/be-java-extensions/java-udf/pom.xml
b/fe/be-java-extensions/java-udf/pom.xml
index 8b1e57490dc..37e920f976d 100644
--- a/fe/be-java-extensions/java-udf/pom.xml
+++ b/fe/be-java-extensions/java-udf/pom.xml
@@ -49,7 +49,8 @@ under the License.
</dependency>
<dependency>
<groupId>org.apache.doris</groupId>
- <artifactId>hive-catalog-shade</artifactId>
+ <artifactId>hive-udf-shade</artifactId>
+ <version>${project.version}</version>
<scope>compile</scope>
</dependency>
</dependencies>
diff --git a/fe/be-java-extensions/pom.xml b/fe/be-java-extensions/pom.xml
index 4d832a0ceea..075b59cc8d6 100644
--- a/fe/be-java-extensions/pom.xml
+++ b/fe/be-java-extensions/pom.xml
@@ -24,11 +24,11 @@ under the License.
<module>iceberg-metadata-scanner</module>
<module>hadoop-hudi-scanner</module>
<module>java-common</module>
+ <module>hive-udf-shade</module>
<module>java-udf</module>
<module>jdbc-scanner</module>
<module>paimon-scanner</module>
<module>max-compute-connector</module>
- <module>avro-scanner</module>
<!-- lakesoul-scanner module has been deprecated -->
<!-- <module>lakesoul-scanner</module> -->
<module>preload-extensions</module>
diff --git a/regression-test/data/external_table_p0/tvf/test_tvf_avro.out
b/regression-test/data/external_table_p0/tvf/test_tvf_avro.out
deleted file mode 100644
index 174d60ce949..00000000000
--- a/regression-test/data/external_table_p0/tvf/test_tvf_avro.out
+++ /dev/null
@@ -1,97 +0,0 @@
--- This file is automatically generated. You should know what you did if you
want to edit this
--- !1 --
-aBoolean BOOLEAN Yes false \N NONE
-aInt INT Yes false \N NONE
-aLong BIGINT Yes false \N NONE
-aFloat FLOAT Yes false \N NONE
-aDouble DOUBLE Yes false \N NONE
-aString TEXT Yes false \N NONE
-anArray ARRAY<INT> Yes false \N NONE
-aMap MAP<TEXT,INT> Yes false \N NONE
-anEnum TEXT Yes false \N NONE
-aRecord STRUCT<a:INT,b:DOUBLE,c:TEXT> Yes false \N NONE
-aUnion STRUCT<string:TEXT> Yes false \N NONE
-mapArrayLong MAP<TEXT,ARRAY<BIGINT>> Yes false \N NONE
-arrayMapBoolean ARRAY<MAP<TEXT,BOOLEAN>> Yes false \N
NONE
-
--- !2 --
-2
-
--- !1 --
-false 100 9999 2.11 9.1102 string test [5, 6, 7]
{"k1":1, "k2":2} B {"a":5, "b":3.14159265358979, "c":"Simple
Record String Field"} \N {"k11":[77, 11, 33], "k22":[10, 20]}
[{"Key11":1}, {"Key22":0}]
-true 42 3400 3.14 9.81 a test string [1, 2, 3, 4]
{"key1":1, "key2":2} A {"a":5, "b":3.14159265358979, "c":"Simple
Record String Field"} \N {"k1":[99, 88, 77], "k2":[10, 20]}
[{"arrayMapKey1":0}, {"arrayMapKey2":1}]
-
--- !2 --
-[1, 2, 3, 4]
-[5, 6, 7]
-
--- !3 --
-{"k1":1, "k2":2}
-{"key1":1, "key2":2}
-
--- !4 --
-A
-B
-
--- !5 --
-{"a":5, "b":3.14159265358979, "c":"Simple Record String Field"}
-{"a":5, "b":3.14159265358979, "c":"Simple Record String Field"}
-
--- !6 --
-\N
-\N
-
--- !7 --
-{"k1":[99, 88, 77], "k2":[10, 20]}
-{"k11":[77, 11, 33], "k22":[10, 20]}
-
--- !8 --
-[{"Key11":1}, {"Key22":0}]
-[{"arrayMapKey1":0}, {"arrayMapKey2":1}]
-
--- !10 --
-[{"Key11":1}, {"Key22":0}] false {"k1":1, "k2":2} 9999 \N
-[{"arrayMapKey1":0}, {"arrayMapKey2":1}] true {"key1":1, "key2":2}
3400 \N
-
--- !11 --
-a test string 9.81 A
-string test 9.1102 B
-
--- !12 --
-{"a":5, "b":3.14159265358979, "c":"Simple Record String Field"}
{"k1":1, "k2":2} 2.11
-{"a":5, "b":3.14159265358979, "c":"Simple Record String Field"}
{"key1":1, "key2":2} 3.14
-
--- !3 --
-aBoolean BOOLEAN Yes false \N NONE
-aInt INT Yes false \N NONE
-aLong BIGINT Yes false \N NONE
-aFloat FLOAT Yes false \N NONE
-aDouble DOUBLE Yes false \N NONE
-aString TEXT Yes false \N NONE
-anArray ARRAY<INT> Yes false \N NONE
-aMap MAP<TEXT,INT> Yes false \N NONE
-anEnum TEXT Yes false \N NONE
-aRecord STRUCT<a:INT,b:DOUBLE,c:TEXT> Yes false \N NONE
-aUnion STRUCT<string:TEXT> Yes false \N NONE
-mapArrayLong MAP<TEXT,ARRAY<BIGINT>> Yes false \N NONE
-arrayMapBoolean ARRAY<MAP<TEXT,BOOLEAN>> Yes false \N
NONE
-
--- !9 --
-false 100 9999 2.11 9.1102 string test [5, 6, 7]
{"k1":1, "k2":2} B {"a":5, "b":3.14159265358979, "c":"Simple
Record String Field"} \N {"k11":[77, 11, 33], "k22":[10, 20]}
[{"Key11":1}, {"Key22":0}]
-true 42 3400 3.14 9.81 a test string [1, 2, 3, 4]
{"key1":1, "key2":2} A {"a":5, "b":3.14159265358979, "c":"Simple
Record String Field"} \N {"k1":[99, 88, 77], "k2":[10, 20]}
[{"arrayMapKey1":0}, {"arrayMapKey2":1}]
-
--- !4 --
-2
-
--- !13 --
-[{"Key11":1}, {"Key22":0}] false {"k1":1, "k2":2} 9999 \N
-[{"arrayMapKey1":0}, {"arrayMapKey2":1}] true {"key1":1, "key2":2}
3400 \N
-
--- !14 --
-a test string 9.81 A
-string test 9.1102 B
-
--- !15 --
-{"a":5, "b":3.14159265358979, "c":"Simple Record String Field"}
{"k1":1, "k2":2} 2.11
-{"a":5, "b":3.14159265358979, "c":"Simple Record String Field"}
{"key1":1, "key2":2} 3.14
-
diff --git a/regression-test/suites/external_table_p0/tvf/test_tvf_avro.groovy
b/regression-test/suites/external_table_p0/tvf/test_tvf_avro.groovy
deleted file mode 100644
index ec05c3a7067..00000000000
--- a/regression-test/suites/external_table_p0/tvf/test_tvf_avro.groovy
+++ /dev/null
@@ -1,211 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-suite("test_tvf_avro", "p0,external") {
-
- // def all_type_file = "all_type.avro";
- // def format = "avro"
-
- // // s3 config
- // String ak = getS3AK()
- // String sk = getS3SK()
- // String s3_endpoint = getS3Endpoint()
- // String region = getS3Region()
- // String bucket = context.config.otherConfigs.get("s3BucketName");
- // def s3Uri =
"https://${bucket}.${s3_endpoint}/regression/datalake/pipeline_data/tvf/${all_type_file}";;
-
- // // hdfs config
- // String hdfs_port = context.config.otherConfigs.get("hive2HdfsPort")
- // String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
- // def hdfsUserName = "doris"
- // def defaultFS = "hdfs://${externalEnvIp}:${hdfs_port}"
- // def hdfsUri = "${defaultFS}" +
"/user/doris/preinstalled_data/avro/avro_all_types/${all_type_file}"
-
- // // TVF s3()
- // qt_1 """
- // desc function s3(
- // "uri" = "${s3Uri}",
- // "ACCESS_KEY" = "${ak}",
- // "SECRET_KEY" = "${sk}",
- // "REGION" = "${region}",
- // "FORMAT" = "${format}");
- // """
-
- // qt_2 """
- // select count(*) from s3(
- // "uri" ="${s3Uri}",
- // "ACCESS_KEY" = "${ak}",
- // "SECRET_KEY" = "${sk}",
- // "REGION" = "${region}",
- // "provider" = "${getS3Provider()}",
- // "FORMAT" = "${format}");
- // """
-
- // order_qt_1 """
- // select * from s3(
- // "uri" = "${s3Uri}",
- // "ACCESS_KEY" = "${ak}",
- // "SECRET_KEY" = "${sk}",
- // "REGION" = "${region}",
- // "provider" = "${getS3Provider()}",
- // "FORMAT" = "${format}");
- // """
-
- // order_qt_2 """
- // select anArray from s3(
- // "uri" = "${s3Uri}",
- // "ACCESS_KEY" = "${ak}",
- // "SECRET_KEY" = "${sk}",
- // "REGION" = "${region}",
- // "provider" = "${getS3Provider()}",
- // "FORMAT" = "${format}");
- // """
-
- // order_qt_3 """
- // select aMap from s3(
- // "uri" = "${s3Uri}",
- // "ACCESS_KEY" = "${ak}",
- // "SECRET_KEY" = "${sk}",
- // "REGION" = "${region}",
- // "provider" = "${getS3Provider()}",
- // "FORMAT" = "${format}");
- // """
-
- // order_qt_4 """
- // select anEnum from s3(
- // "uri" = "${s3Uri}",
- // "ACCESS_KEY" = "${ak}",
- // "SECRET_KEY" = "${sk}",
- // "REGION" = "${region}",
- // "provider" = "${getS3Provider()}",
- // "FORMAT" = "${format}");
- // """
-
- // order_qt_5 """
- // select aRecord from s3(
- // "uri" = "${s3Uri}",
- // "ACCESS_KEY" = "${ak}",
- // "SECRET_KEY" = "${sk}",
- // "REGION" = "${region}",
- // "provider" = "${getS3Provider()}",
- // "FORMAT" = "${format}");
- // """
-
- // order_qt_6 """
- // select aUnion from s3(
- // "uri" = "${s3Uri}",
- // "ACCESS_KEY" = "${ak}",
- // "SECRET_KEY" = "${sk}",
- // "REGION" = "${region}",
- // "provider" = "${getS3Provider()}",
- // "FORMAT" = "${format}");
- // """
-
- // order_qt_7 """
- // select mapArrayLong from s3(
- // "uri" ="${s3Uri}",
- // "ACCESS_KEY" = "${ak}",
- // "SECRET_KEY" = "${sk}",
- // "REGION" = "${region}",
- // "provider" = "${getS3Provider()}",
- // "FORMAT" = "${format}");
- // """
-
- // order_qt_8 """
- // select arrayMapBoolean from s3(
- // "uri" = "${s3Uri}",
- // "ACCESS_KEY" = "${ak}",
- // "SECRET_KEY" = "${sk}",
- // "REGION" = "${region}",
- // "provider" = "${getS3Provider()}",
- // "FORMAT" = "${format}");
- // """
-
- // order_qt_10 """
- // select arrayMapBoolean,aBoolean,aMap,aLong,aUnion from s3(
- // "uri" = "${s3Uri}",
- // "ACCESS_KEY" = "${ak}",
- // "SECRET_KEY" = "${sk}",
- // "REGION" = "${region}",
- // "provider" = "${getS3Provider()}",
- // "FORMAT" = "${format}");
- // """
-
- // order_qt_11 """
- // select aString,aDouble,anEnum from s3(
- // "uri" = "${s3Uri}",
- // "ACCESS_KEY" = "${ak}",
- // "SECRET_KEY" = "${sk}",
- // "REGION" = "${region}",
- // "provider" = "${getS3Provider()}",
- // "FORMAT" = "${format}");
- // """
-
- // order_qt_12 """
- // select aRecord,aMap,aFloat from s3(
- // "uri" = "${s3Uri}",
- // "ACCESS_KEY" = "${ak}",
- // "SECRET_KEY" = "${sk}",
- // "REGION" = "${region}",
- // "provider" = "${getS3Provider()}",
- // "FORMAT" = "${format}");
- // """
-
- // // TVF hdfs()
- // String enabled = context.config.otherConfigs.get("enableHiveTest")
- // if (enabled != null && enabled.equalsIgnoreCase("true")) {
- // try {
- // qt_3 """
- // desc function HDFS(
- // "uri" = "${hdfsUri}",
- // "fs.defaultFS" = "${defaultFS}",
- // "hadoop.username" = "${hdfsUserName}",
- // "FORMAT" = "${format}"); """
-
- // order_qt_9 """ select * from HDFS(
- // "uri" = "${hdfsUri}",
- // "fs.defaultFS" = "${defaultFS}",
- // "hadoop.username" = "${hdfsUserName}",
- // "format" = "${format}")"""
-
- // qt_4 """ select count(*) from HDFS(
- // "uri" = "${hdfsUri}",
- // "fs.defaultFS" = "${defaultFS}",
- // "hadoop.username" = "${hdfsUserName}",
- // "format" = "${format}"); """
-
- // order_qt_13 """ select
arrayMapBoolean,aBoolean,aMap,aLong,aUnion from HDFS(
- // "uri" = "${hdfsUri}",
- // "fs.defaultFS" = "${defaultFS}",
- // "hadoop.username" = "${hdfsUserName}",
- // "format" = "${format}")"""
-
- // order_qt_14 """ select aString,aDouble,anEnum from HDFS(
- // "uri" = "${hdfsUri}",
- // "fs.defaultFS" = "${defaultFS}",
- // "hadoop.username" = "${hdfsUserName}",
- // "format" = "${format}")"""
-
- // order_qt_15 """ select aRecord,aMap,aFloat from HDFS(
- // "uri" = "${hdfsUri}",
- // "fs.defaultFS" = "${defaultFS}",
- // "hadoop.username" = "${hdfsUserName}",
- // "format" = "${format}")"""
- // } finally {
- // }
- // }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
