This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f38e11ec4ed [fix](paimon)fix type convert for paimon (#28774)
f38e11ec4ed is described below
commit f38e11ec4ed0bddd91bc262a46ef902ecc124e05
Author: wuwenchi <[email protected]>
AuthorDate: Fri Dec 22 13:18:25 2023 +0800
[fix](paimon)fix type convert for paimon (#28774)
fix type convert for paimon
---
.../org/apache/doris/paimon/PaimonColumnValue.java | 10 ++++-----
.../org/apache/doris/paimon/PaimonTypeUtils.java | 11 ++++++++-
.../paimon/test_paimon_catalog.out | 26 +++++++++++++---------
.../paimon/test_paimon_catalog.groovy | 4 +++-
4 files changed, 33 insertions(+), 18 deletions(-)
diff --git
a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java
b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java
index c55b80b6e86..b1b7440649e 100644
---
a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java
+++
b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java
@@ -24,16 +24,17 @@ import org.apache.paimon.data.DataGetters;
import org.apache.paimon.data.InternalArray;
import org.apache.paimon.data.InternalMap;
import org.apache.paimon.data.InternalRow;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.math.BigDecimal;
import java.math.BigInteger;
-import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
-import java.time.ZoneOffset;
import java.util.List;
public class PaimonColumnValue implements ColumnValue {
+ private static final Logger LOG =
LoggerFactory.getLogger(PaimonColumnValue.class);
private int idx;
private DataGetters record;
private ColumnType dorisType;
@@ -118,13 +119,12 @@ public class PaimonColumnValue implements ColumnValue {
@Override
public LocalDate getDate() {
- return LocalDate.ofEpochDay(record.getLong(idx));
+ return LocalDate.ofEpochDay(record.getInt(idx));
}
@Override
public LocalDateTime getDateTime() {
- return Instant.ofEpochMilli(record.getTimestamp(idx, 3)
- .getMillisecond()).atZone(ZoneOffset.ofHours(0)).toLocalDateTime();
+ return record.getTimestamp(idx,
dorisType.getPrecision()).toLocalDateTime();
}
@Override
diff --git
a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java
b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java
index 575809c5df9..c0bb94dd8e4 100644
---
a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java
+++
b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java
@@ -98,7 +98,16 @@ public class PaimonTypeUtils {
@Override
public PaimonColumnType visit(DecimalType decimalType) {
- return new PaimonColumnType(Type.DECIMAL128,
decimalType.getPrecision(), decimalType.getScale());
+ int precision = decimalType.getPrecision();
+ Type type;
+ if (precision <= ColumnType.MAX_DECIMAL32_PRECISION) {
+ type = Type.DECIMAL32;
+ } else if (precision <= ColumnType.MAX_DECIMAL64_PRECISION) {
+ type = Type.DECIMAL64;
+ } else {
+ type = Type.DECIMAL128;
+ }
+ return new PaimonColumnType(type, decimalType.getPrecision(),
decimalType.getScale());
}
@Override
diff --git
a/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out
b/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out
index ab02456f352..a2e4f2752c0 100644
--- a/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out
+++ b/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out
@@ -1,7 +1,11 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
-- !all --
1 2 3 4 5 6 7 8 9.1 10.1
11.10 2020-02-02 13str 14varchar a true aaaa
2023-08-13T09:32:38.530
-10 20 30 40 50 60 70 80 90.1 100.1
0.00 2020-03-02 130str 140varchar b false bbbb
2023-08-14T08:32:52.821
+10 20 30 40 50 60 70 80 90.1 100.1
110.10 2020-03-02 130str 140varchar b false bbbb
2023-08-14T08:32:52.821
+
+-- !all_with_parquet --
+2 2 3 4 5 6 7 8 9.1 10.1
11.10 2020-02-02 13str 14varchar a true aaaa
2023-12-21T10:02:32.747
+10 20 30 40 50 60 70 80 90.1 100.1
110.10 2020-03-02 130str 140varchar b false bbbb
2023-12-21T10:02:37.527
-- !c1 --
1 2 3 4 5 6 7 8 9.1 10.1
11.10 2020-02-02 13str 14varchar a true aaaa
2023-08-13T09:32:38.530
@@ -85,9 +89,9 @@ true \N \N \N
9999
-- !c27 --
-\N [1, null, null] [1, null, null, null] [4, null, null] [null, null]
[null, 12, 13] [null, 15] [null, null] [18, 19, 20] [21.11, 22, 22]
[null] [null] [null, null, "2023-10-24"] ["13str", "arr_string"]
["14varchar", "arr_varcha"] [null, null] [null, null] [null, null,
null] [null, null] {1:null, 0:null} {1:null, 0:null}
{1:null, 2:null} {1:6, 0:null} {1:7} {0:null, 1:20} {1:null}
{1:null} {0:21.11, 1:22.22} {1:23.333} {0:24.44, 1:null}
{1:null, 4:"2023-02-03"} {0:"byte_string"} {0:"aa", [...]
-1 [1, 0] [1, 2, 3] [4, 5, 6] [7, 8, 9] [10, 11, 12,
13] [14, 15] [16, 17] [18, 19, 20] [21.11, 22, 22]
[23.333] [24.44] ["2020-02-02", "2023-10-24"] ["13str", "arr_string"]
["14varchar", "arr_varcha"] ["a", "b"] [1, 0, 1] ["aaaa", "bbb"]
["2023-10-24 16:10:40.529", "2023-10-24 16:10:40.529"] {1:0, 0:1} {0:2,
1:3} {1:4, 0:5} {1:6, 0:9} {1:7} {0:19, 1:20} {1:21}
{1:22} {0:21.11, 1:22.22} {1:23.333} {0:24.44, 1:0.00}
{1:"2021-02-03", 4:"2023-02-03"} {0:"byte_string"} {0:"aa",
1:"bb"} {1:"o" [...]
-2 [1, null, 0] [1, null, 2, 3] [4, 5, 6] [null, null] [null,
12, 13] [14, 15] [null, null] [18, 19, 20] [21.11, 22, 22]
[23.333] [null, 0.00] ["2020-02-02", null, "2023-10-24"]
["13str", "arr_string"] ["14varchar", "arr_varcha"] ["a", "b"] [1, 0,
1] ["aaaa", "bbb", null, null] ["2023-10-24 18:19:51.670", null,
"2023-10-24 18:19:51.670"] {1:null, 0:null} {1:null, 0:null}
{1:4, 2:null} {1:6, 0:9} {1:7} {0:19, 1:20} {1:21} {1:22}
{0:21.11, 1:22.22} {1:23.333} {0:25.55, 1:0.00} {1:null, 4:"2
[...]
+\N [1, null, null] [1, null, null, null] [4, null, null] [null, null]
[null, 12, 13] [null, 15] [null, null] [18, 19, 20] [21.11, 22, 22]
[null] [null] [null, null, "2023-10-24"] ["13str", "arr_string"]
["14varchar", "arr_varcha"] [null, null] [null, null] [null, null,
null] [null, null] {1:null, 0:null} {1:null, 0:null}
{1:null, 2:null} {1:6, 0:null} {1:7} {0:null, 1:20} {1:null}
{1:null} {0:21.11, 1:22.22} {1:23.333} {0:24.44, 1:null}
{1:null, 4:"2023-02-03"} {0:"byte_string"} {0:"aa", [...]
+1 [1, 0] [1, 2, 3] [4, 5, 6] [7, 8, 9] [10, 11, 12,
13] [14, 15] [16, 17] [18, 19, 20] [21.11, 22, 22]
[23.333] [24.44] ["2020-02-02", "2023-10-24"] ["13str", "arr_string"]
["14varchar", "arr_varcha"] ["a", "b"] [1, 0, 1] ["aaaa", "bbb"]
["2023-10-24 16:10:40.529", "2023-10-24 16:10:40.529"] {1:0, 0:1} {0:2,
1:3} {1:4, 0:5} {1:6, 0:9} {1:7} {0:19, 1:20} {1:21}
{1:22} {0:21.11, 1:22.22} {1:23.333} {0:24.44, 1:25.55}
{1:"2021-02-03", 4:"2023-02-03"} {0:"byte_string"} {0:"aa",
1:"bb"} {1:"o [...]
+2 [1, null, 0] [1, null, 2, 3] [4, 5, 6] [null, null] [null,
12, 13] [14, 15] [null, null] [18, 19, 20] [21.11, 22, 22]
[23.333] [null, 24.44] ["2020-02-02", null, "2023-10-24"]
["13str", "arr_string"] ["14varchar", "arr_varcha"] ["a", "b"] [1, 0,
1] ["aaaa", "bbb", null, null] ["2023-10-24 18:19:51.670", null,
"2023-10-24 18:19:51.670"] {1:null, 0:null} {1:null, 0:null}
{1:4, 2:null} {1:6, 0:9} {1:7} {0:19, 1:20} {1:21} {1:22}
{0:21.11, 1:22.22} {1:23.333} {0:24.44, 1:25.55} {1:null, 4:
[...]
-- !c28 --
0
@@ -141,7 +145,7 @@ true \N \N \N
-- !c38 --
24.44
-0.00
+24.44
\N
-- !c39 --
@@ -231,7 +235,7 @@ true
-- !c56 --
24.44
-25.55
+24.44
24.44
-- !c57 --
@@ -321,7 +325,7 @@ bbb
-- !c74 --
24.44
-25.55
+24.44
24.44
-- !c75 --
@@ -411,7 +415,7 @@ bbb
-- !c92 --
24.44
-25.55
+24.44
24.44
-- !c93 --
@@ -450,7 +454,7 @@ bbb
\N
-- !c100 --
-1 [[0, 1, 1], [0, 1, 1], [1, 1, 1]] [["3", "7", "a"], ["0", "e",
"2"], ["b", "4", "5"]] [["77", "83", "1c"], ["af", "f0", "0b"], ["be",
"d2", "7b"]]
[["4a3aee3c345936f7a6eeb5307fc79d5f9ee3ae3a5e58edcfd4d40ec3d27353bf7b15a2eb6b6011e9ddf2c2cd1d6d50d6cc20",
"00cb283c1a2bb2f3f244f89eafb79effd8aea55d5e68ce9d707250a3b47b65c0ea2b591b7145a56c801b9e6bda853e2f0581",
"ae8fbe21e20f32a486757b5254faea09906bf451e096cc7d33b5a6fb56995c1601e7469a674f5e8475a2b86d2a69e02f9438"],
["6827f8e65869d476a9e2fdfa03 [...]
-2 [[0, 0, 0], [1, 0, 0], [0, 1, 1]] [["a", "5", "f"], ["7", "2",
"3"], ["1", "b", "f"]] [["11", "a8", "e3"], ["f4", "ee", "c3"], ["0f",
"c6", "05"]]
[["b723869515b24e9fbb54503f8a7584f083479998766213b784a9c530cbc0376bd5035054c657437251b85fa3dd41a0483776",
"ee780d81d3e9faa36aad06522a09cf9b18e00614c991d2c079243dcc7190f3dd6559e75e2c1992270272d9a9c01e950c7bd4",
"6e825a52cdae65786801caae53182956c80f88a48fa258a90d2c93302023c78b83f1dce758615a74731c9eef993a8c1dc4ad"],
["2ae97869372970f7ad7d0007a1 [...]
-3 [[1, 0, 0], [1, 0, 1], [1, 0, 1]] [["c", "6", "4"], ["1", "e",
"7"], ["7", "1", "d"]] [["50", "a9", "b0"], ["02", "bf", "3a"], ["0a",
"1d", "9a"]]
[["2df05ba6ce8661f0fb9bc88386a1ba67188e3e99a4142a0703e1cd8bdf041fbc20131e50bea2a9891498c638ebac842d3d46",
"f6fecf3a4263ed8d4c5b63e4b3d9f4084b83835d9fba2046bf48d0a8068f2044c48271a1e9726741a09badea72c37cf18de3",
"d23af3266db4eda12673f5c451d36343ba1cea00fbfeeff2165de40e834778eb96a1199cb523dd394b4f08824f6af2a7d894"],
["dc00e9b27b9540e170caf93805 [...]
+1 [[0, 1, 1], [0, 1, 1], [1, 1, 1]] [["3", "7", "a"], ["0", "e",
"2"], ["b", "4", "5"]] [["77", "83", "1c"], ["af", "f0", "0b"], ["be",
"d2", "7b"]]
[["4a3aee3c345936f7a6eeb5307fc79d5f9ee3ae3a5e58edcfd4d40ec3d27353bf7b15a2eb6b6011e9ddf2c2cd1d6d50d6cc20",
"00cb283c1a2bb2f3f244f89eafb79effd8aea55d5e68ce9d707250a3b47b65c0ea2b591b7145a56c801b9e6bda853e2f0581",
"ae8fbe21e20f32a486757b5254faea09906bf451e096cc7d33b5a6fb56995c1601e7469a674f5e8475a2b86d2a69e02f9438"],
["6827f8e65869d476a9e2fdfa03 [...]
+2 [[0, 0, 0], [1, 0, 0], [0, 1, 1]] [["a", "5", "f"], ["7", "2",
"3"], ["1", "b", "f"]] [["11", "a8", "e3"], ["f4", "ee", "c3"], ["0f",
"c6", "05"]]
[["b723869515b24e9fbb54503f8a7584f083479998766213b784a9c530cbc0376bd5035054c657437251b85fa3dd41a0483776",
"ee780d81d3e9faa36aad06522a09cf9b18e00614c991d2c079243dcc7190f3dd6559e75e2c1992270272d9a9c01e950c7bd4",
"6e825a52cdae65786801caae53182956c80f88a48fa258a90d2c93302023c78b83f1dce758615a74731c9eef993a8c1dc4ad"],
["2ae97869372970f7ad7d0007a1 [...]
+3 [[1, 0, 0], [1, 0, 1], [1, 0, 1]] [["c", "6", "4"], ["1", "e",
"7"], ["7", "1", "d"]] [["50", "a9", "b0"], ["02", "bf", "3a"], ["0a",
"1d", "9a"]]
[["2df05ba6ce8661f0fb9bc88386a1ba67188e3e99a4142a0703e1cd8bdf041fbc20131e50bea2a9891498c638ebac842d3d46",
"f6fecf3a4263ed8d4c5b63e4b3d9f4084b83835d9fba2046bf48d0a8068f2044c48271a1e9726741a09badea72c37cf18de3",
"d23af3266db4eda12673f5c451d36343ba1cea00fbfeeff2165de40e834778eb96a1199cb523dd394b4f08824f6af2a7d894"],
["dc00e9b27b9540e170caf93805 [...]
diff --git
a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
index 03603fda0de..ee74abd7ed8 100644
--- a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
+++ b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
@@ -54,7 +54,8 @@ suite("test_paimon_catalog",
"p0,external,doris,external_docker,external_docker_
String enabled = context.config.otherConfigs.get("enablePaimonTest")
if (enabled != null && enabled.equalsIgnoreCase("true")) {
- def all = """select * from all_table;"""
+ def all = """select * from all_table order by c1;"""
+ def all_with_parquet = """select * from all_table_with_parquet
order by c1;"""
def c1 = """select * from all_table where c1=1;"""
def c2 = """select * from all_table where c2=2;"""
def c3 = """select * from all_table where c3=3;"""
@@ -173,6 +174,7 @@ suite("test_paimon_catalog",
"p0,external,doris,external_docker,external_docker_
sql """use `${catalog_name}`.`db1`"""
qt_all all
+ qt_all_with_parquet all_with_parquet
qt_c1 c1
qt_c2 c2
qt_c3 c3
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
