This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 821c0d13809 branch-2.1: [improvement](paimon)Using table serialization
on the jni side (#43475)
821c0d13809 is described below
commit 821c0d1380962dff99f34369d8c4fae1abed0e77
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Nov 12 14:43:32 2024 +0800
branch-2.1: [improvement](paimon)Using table serialization on the jni side
(#43475)
Cherry-picked from #43167
Co-authored-by: wuwenchi <[email protected]>
Co-authored-by: wuwenchi <[email protected]>
---
be/src/vec/exec/format/table/paimon_jni_reader.cpp | 3 ++
.../org/apache/doris/paimon/PaimonJniScanner.java | 32 ++++++++++++++++------
.../datasource/paimon/source/PaimonScanNode.java | 1 +
gensrc/thrift/PlanNodes.thrift | 17 ++++++------
4 files changed, 36 insertions(+), 17 deletions(-)
diff --git a/be/src/vec/exec/format/table/paimon_jni_reader.cpp
b/be/src/vec/exec/format/table/paimon_jni_reader.cpp
index fa73454f4b4..daf1be1cf0c 100644
--- a/be/src/vec/exec/format/table/paimon_jni_reader.cpp
+++ b/be/src/vec/exec/format/table/paimon_jni_reader.cpp
@@ -61,6 +61,9 @@ PaimonJniReader::PaimonJniReader(const
std::vector<SlotDescriptor*>& file_slot_d
std::to_string(range.table_format_params.paimon_params.last_update_time);
params["required_fields"] = join(column_names, ",");
params["columns_types"] = join(column_types, "#");
+ if (range.table_format_params.paimon_params.__isset.paimon_table) {
+ params["paimon_table"] =
range.table_format_params.paimon_params.paimon_table;
+ }
// Used to create paimon option
for (auto& kv : range.table_format_params.paimon_params.paimon_options) {
diff --git
a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
index 7bd9fa631c8..e85d465f663 100644
---
a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
+++
b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
@@ -42,13 +42,19 @@ import java.util.stream.Collectors;
public class PaimonJniScanner extends JniScanner {
private static final Logger LOG =
LoggerFactory.getLogger(PaimonJniScanner.class);
+ @Deprecated
private static final String PAIMON_OPTION_PREFIX = "paimon.";
+ @Deprecated
private static final String HADOOP_OPTION_PREFIX = "hadoop.";
private final Map<String, String> params;
+ @Deprecated
private final Map<String, String> paimonOptionParams;
+ @Deprecated
private final Map<String, String> hadoopOptionParams;
+ @Deprecated
private final String dbName;
+ @Deprecated
private final String tblName;
private final String paimonSplit;
private final String paimonPredicate;
@@ -58,9 +64,13 @@ public class PaimonJniScanner extends JniScanner {
private List<String> paimonAllFieldNames;
private List<DataType> paimonDataTypeList;
+ @Deprecated
private long ctlId;
+ @Deprecated
private long dbId;
+ @Deprecated
private long tblId;
+ @Deprecated
private long lastUpdateTime;
private RecordReader.RecordIterator<InternalRow> recordIterator = null;
private final ClassLoader classLoader;
@@ -214,16 +224,20 @@ public class PaimonJniScanner extends JniScanner {
}
private void initTable() {
- PaimonTableCacheKey key = new PaimonTableCacheKey(ctlId, dbId, tblId,
- paimonOptionParams, hadoopOptionParams, dbName, tblName);
- TableExt tableExt = PaimonTableCache.getTable(key);
- if (tableExt.getCreateTime() < lastUpdateTime) {
- LOG.warn("invalidate cache table:{}, localTime:{}, remoteTime:{}",
key, tableExt.getCreateTime(),
- lastUpdateTime);
- PaimonTableCache.invalidateTableCache(key);
- tableExt = PaimonTableCache.getTable(key);
+ if (params.containsKey("paimon_table")) {
+ table = PaimonUtils.deserialize(params.get("paimon_table"));
+ } else {
+ PaimonTableCacheKey key = new PaimonTableCacheKey(ctlId, dbId,
tblId,
+ paimonOptionParams, hadoopOptionParams, dbName, tblName);
+ TableExt tableExt = PaimonTableCache.getTable(key);
+ if (tableExt.getCreateTime() < lastUpdateTime) {
+ LOG.warn("invalidate cache table:{}, localTime:{},
remoteTime:{}", key, tableExt.getCreateTime(),
+ lastUpdateTime);
+ PaimonTableCache.invalidateTableCache(key);
+ tableExt = PaimonTableCache.getTable(key);
+ }
+ this.table = tableExt.getTable();
}
- this.table = tableExt.getTable();
paimonAllFieldNames = PaimonUtils.getFieldNames(this.table.rowType());
if (LOG.isDebugEnabled()) {
LOG.debug("paimonAllFieldNames:{}", paimonAllFieldNames);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
index cd477cc9b29..29e3d0529f2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
@@ -164,6 +164,7 @@ public class PaimonScanNode extends FileQueryScanNode {
fileDesc.setDbId(((PaimonExternalTable)
source.getTargetTable()).getDbId());
fileDesc.setTblId(source.getTargetTable().getId());
fileDesc.setLastUpdateTime(source.getTargetTable().getUpdateTime());
+ fileDesc.setPaimonTable(encodeObjectToString(source.getPaimonTable()));
// The hadoop conf should be same with
PaimonExternalCatalog.createCatalog()#getConfiguration()
fileDesc.setHadoopConf(source.getCatalog().getCatalogProperty().getHadoopProperties());
Optional<DeletionFile> optDeletionFile = paimonSplit.getDeletionFile();
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index 72d0951062c..8ba89387d97 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -319,17 +319,18 @@ struct TPaimonDeletionFileDesc {
struct TPaimonFileDesc {
1: optional string paimon_split
2: optional string paimon_column_names
- 3: optional string db_name
- 4: optional string table_name
+ 3: optional string db_name // deprecated
+ 4: optional string table_name // deprecated
5: optional string paimon_predicate
- 6: optional map<string, string> paimon_options
- 7: optional i64 ctl_id
- 8: optional i64 db_id
- 9: optional i64 tbl_id
- 10: optional i64 last_update_time
+ 6: optional map<string, string> paimon_options // deprecated
+ 7: optional i64 ctl_id // deprecated
+ 8: optional i64 db_id // deprecated
+ 9: optional i64 tbl_id // deprecated
+ 10: optional i64 last_update_time // deprecated
11: optional string file_format
12: optional TPaimonDeletionFileDesc deletion_file;
- 13: optional map<string, string> hadoop_conf
+ 13: optional map<string, string> hadoop_conf // deprecated
+ 14: optional string paimon_table
}
struct TMaxComputeFileDesc {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
