This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch tpch500
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/tpch500 by this push:
new 2989931fe72 [tmp] add external table nullable config
2989931fe72 is described below
commit 2989931fe72fb04ebbdf2aae231cbac0938251b4
Author: morningman <[email protected]>
AuthorDate: Mon Dec 18 14:52:26 2023 +0800
[tmp] add external table nullable config
---
be/src/vec/core/decimal_comparison.h | 2 +-
be/src/vec/exec/format/parquet/vparquet_column_reader.cpp | 12 ++++++------
.../src/main/java/org/apache/doris/common/Config.java | 3 +++
.../org/apache/doris/catalog/external/HMSExternalTable.java | 3 ++-
4 files changed, 12 insertions(+), 8 deletions(-)
diff --git a/be/src/vec/core/decimal_comparison.h
b/be/src/vec/core/decimal_comparison.h
index 16877583695..a24b85bc7fd 100644
--- a/be/src/vec/core/decimal_comparison.h
+++ b/be/src/vec/core/decimal_comparison.h
@@ -241,7 +241,7 @@ private:
}
template <bool scale_left, bool scale_right>
- static Int8 apply(A a, B b, CompareInt scale [[maybe_unused]]) {
+ static UInt8 apply(A a, B b, CompareInt scale [[maybe_unused]]) {
CompareInt x = a;
CompareInt y = b;
diff --git a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
index d6f7d746bcd..79bd8eadeb5 100644
--- a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
@@ -285,7 +285,7 @@ Status ScalarColumnReader::_read_values(size_t num_values,
ColumnPtr& doris_colu
}
} else {
if (_chunk_reader->max_def_level() > 0) {
- return Status::Corruption("Not nullable column has null values in
parquet file");
+ // return Status::Corruption("Not nullable column has null values
in parquet file");
}
data_column = doris_column->assume_mutable();
}
@@ -371,7 +371,7 @@ Status ScalarColumnReader::_read_nested_column(ColumnPtr&
doris_column, DataType
map_data_column = &(nullable_column->get_null_map_data());
} else {
if (_field_schema->is_nullable) {
- return Status::Corruption("Not nullable column has null values in
parquet file");
+ // return Status::Corruption("Not nullable column has null values
in parquet file");
}
data_column = doris_column->assume_mutable();
}
@@ -605,7 +605,7 @@ Status ArrayColumnReader::read_column_data(ColumnPtr&
doris_column, DataTypePtr&
data_column = nullable_column->get_nested_column_ptr();
} else {
if (_field_schema->is_nullable) {
- return Status::Corruption("Not nullable column has null values in
parquet file");
+ // return Status::Corruption("Not nullable column has null values
in parquet file");
}
data_column = doris_column->assume_mutable();
}
@@ -651,7 +651,7 @@ Status MapColumnReader::read_column_data(ColumnPtr&
doris_column, DataTypePtr& t
data_column = nullable_column->get_nested_column_ptr();
} else {
if (_field_schema->is_nullable) {
- return Status::Corruption("Not nullable column has null values in
parquet file");
+ // return Status::Corruption("Not nullable column has null values
in parquet file");
}
data_column = doris_column->assume_mutable();
}
@@ -714,7 +714,7 @@ Status StructColumnReader::read_column_data(ColumnPtr&
doris_column, DataTypePtr
data_column = nullable_column->get_nested_column_ptr();
} else {
if (_field_schema->is_nullable) {
- return Status::Corruption("Not nullable column has null values in
parquet file");
+ // return Status::Corruption("Not nullable column has null values
in parquet file");
}
data_column = doris_column->assume_mutable();
}
@@ -759,4 +759,4 @@ Status StructColumnReader::read_column_data(ColumnPtr&
doris_column, DataTypePtr
return Status::OK();
}
-}; // namespace doris::vectorized
\ No newline at end of file
+}; // namespace doris::vectorized
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 9421364b482..41f8bfa3cbb 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2324,4 +2324,7 @@ public class Config extends ConfigBase {
@ConfField(description = {"是否开启通过http接口获取log文件的功能",
"Whether to enable the function of getting log files through http
interface"})
public static boolean enable_get_log_file_api = false;
+
+ @ConfField
+ public static boolean external_table_column_nullable = true;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
index b284b4d60ec..3607de2af16 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
@@ -25,6 +25,7 @@ import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
import org.apache.doris.datasource.HMSExternalCatalog;
import org.apache.doris.datasource.hive.HiveMetaStoreCache;
import org.apache.doris.datasource.hive.PooledHiveMetaStoreClient;
@@ -433,7 +434,7 @@ public class HMSExternalTable extends ExternalTable {
for (FieldSchema field : schema) {
tmpSchema.add(new
Column(field.getName().toLowerCase(Locale.ROOT),
HiveMetaStoreClientHelper.hiveTypeToDorisType(field.getType()), true, null,
- true, field.getComment(), true, -1));
+ Config.external_table_column_nullable,
field.getComment(), true, -1));
}
columns = tmpSchema;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
