This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 10086d5dc2a [fix](jni-reader) Add an interface for JNI Reader so that
Profile can collect statistics about JniReader. (#37262)
10086d5dc2a is described below
commit 10086d5dc2a21275a228955fadeb8af87b1d8ece
Author: Tiewei Fang <[email protected]>
AuthorDate: Tue Jul 9 17:03:45 2024 +0800
[fix](jni-reader) Add an interface for JNI Reader so that Profile can
collect statistics about JniReader. (#37262)
---
be/src/vec/exec/format/jni_reader.h | 6 ++++++
be/src/vec/exec/format/table/hudi_jni_reader.cpp | 6 +-----
be/src/vec/exec/format/table/lakesoul_jni_reader.cpp | 6 +-----
be/src/vec/exec/format/table/max_compute_jni_reader.cpp | 6 +-----
be/src/vec/exec/format/table/paimon_jni_reader.cpp | 6 +-----
.../vec/exec/format/table/trino_connector_jni_reader.cpp | 6 +-----
.../doris/trinoconnector/TrinoConnectorJniScanner.java | 14 ++++++++++++++
7 files changed, 25 insertions(+), 25 deletions(-)
diff --git a/be/src/vec/exec/format/jni_reader.h
b/be/src/vec/exec/format/jni_reader.h
index 714bdb96b19..3f156f598a0 100644
--- a/be/src/vec/exec/format/jni_reader.h
+++ b/be/src/vec/exec/format/jni_reader.h
@@ -58,6 +58,12 @@ public:
}
protected:
+ void _collect_profile_before_close() override {
+ if (_jni_connector) {
+ _jni_connector->collect_profile_before_close();
+ }
+ }
+
const std::vector<SlotDescriptor*>& _file_slot_descs;
RuntimeState* _state = nullptr;
RuntimeProfile* _profile = nullptr;
diff --git a/be/src/vec/exec/format/table/hudi_jni_reader.cpp
b/be/src/vec/exec/format/table/hudi_jni_reader.cpp
index cffa2ce9ac4..33ba92b540a 100644
--- a/be/src/vec/exec/format/table/hudi_jni_reader.cpp
+++ b/be/src/vec/exec/format/table/hudi_jni_reader.cpp
@@ -78,11 +78,7 @@ HudiJniReader::HudiJniReader(const TFileScanRangeParams&
scan_params,
}
Status HudiJniReader::get_next_block(Block* block, size_t* read_rows, bool*
eof) {
- RETURN_IF_ERROR(_jni_connector->get_next_block(block, read_rows, eof));
- if (*eof) {
- RETURN_IF_ERROR(_jni_connector->close());
- }
- return Status::OK();
+ return _jni_connector->get_next_block(block, read_rows, eof);
}
Status HudiJniReader::get_columns(std::unordered_map<std::string,
TypeDescriptor>* name_to_type,
diff --git a/be/src/vec/exec/format/table/lakesoul_jni_reader.cpp
b/be/src/vec/exec/format/table/lakesoul_jni_reader.cpp
index c2c33ca75ff..c285b72df25 100644
--- a/be/src/vec/exec/format/table/lakesoul_jni_reader.cpp
+++ b/be/src/vec/exec/format/table/lakesoul_jni_reader.cpp
@@ -62,11 +62,7 @@ LakeSoulJniReader::LakeSoulJniReader(const
TLakeSoulFileDesc& lakesoul_params,
}
Status LakeSoulJniReader::get_next_block(Block* block, size_t* read_rows,
bool* eof) {
- RETURN_IF_ERROR(_jni_connector->get_next_block(block, read_rows, eof));
- if (*eof) {
- RETURN_IF_ERROR(_jni_connector->close());
- }
- return Status::OK();
+ return _jni_connector->get_next_block(block, read_rows, eof);
}
Status LakeSoulJniReader::get_columns(std::unordered_map<std::string,
TypeDescriptor>* name_to_type,
diff --git a/be/src/vec/exec/format/table/max_compute_jni_reader.cpp
b/be/src/vec/exec/format/table/max_compute_jni_reader.cpp
index d520bd9b295..f3fd093ca0a 100644
--- a/be/src/vec/exec/format/table/max_compute_jni_reader.cpp
+++ b/be/src/vec/exec/format/table/max_compute_jni_reader.cpp
@@ -81,11 +81,7 @@ MaxComputeJniReader::MaxComputeJniReader(const
MaxComputeTableDescriptor* mc_des
}
Status MaxComputeJniReader::get_next_block(Block* block, size_t* read_rows,
bool* eof) {
- RETURN_IF_ERROR(_jni_connector->get_next_block(block, read_rows, eof));
- if (*eof) {
- RETURN_IF_ERROR(_jni_connector->close());
- }
- return Status::OK();
+ return _jni_connector->get_next_block(block, read_rows, eof);
}
Status MaxComputeJniReader::get_columns(
diff --git a/be/src/vec/exec/format/table/paimon_jni_reader.cpp
b/be/src/vec/exec/format/table/paimon_jni_reader.cpp
index ef690c15b68..b53848f8a1a 100644
--- a/be/src/vec/exec/format/table/paimon_jni_reader.cpp
+++ b/be/src/vec/exec/format/table/paimon_jni_reader.cpp
@@ -70,11 +70,7 @@ PaimonJniReader::PaimonJniReader(const
std::vector<SlotDescriptor*>& file_slot_d
}
Status PaimonJniReader::get_next_block(Block* block, size_t* read_rows, bool*
eof) {
- RETURN_IF_ERROR(_jni_connector->get_next_block(block, read_rows, eof));
- if (*eof) {
- RETURN_IF_ERROR(_jni_connector->close());
- }
- return Status::OK();
+ return _jni_connector->get_next_block(block, read_rows, eof);
}
Status PaimonJniReader::get_columns(std::unordered_map<std::string,
TypeDescriptor>* name_to_type,
diff --git a/be/src/vec/exec/format/table/trino_connector_jni_reader.cpp
b/be/src/vec/exec/format/table/trino_connector_jni_reader.cpp
index 6fd86385197..b72da757629 100644
--- a/be/src/vec/exec/format/table/trino_connector_jni_reader.cpp
+++ b/be/src/vec/exec/format/table/trino_connector_jni_reader.cpp
@@ -82,11 +82,7 @@ Status TrinoConnectorJniReader::init_reader(
}
Status TrinoConnectorJniReader::get_next_block(Block* block, size_t*
read_rows, bool* eof) {
- RETURN_IF_ERROR(_jni_connector->get_next_block(block, read_rows, eof));
- if (*eof) {
- RETURN_IF_ERROR(_jni_connector->close());
- }
- return Status::OK();
+ return _jni_connector->get_next_block(block, read_rows, eof);
}
Status TrinoConnectorJniReader::get_columns(
diff --git
a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorJniScanner.java
b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorJniScanner.java
index bcc60337429..3e050fc923f 100644
---
a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorJniScanner.java
+++
b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorJniScanner.java
@@ -74,6 +74,7 @@ import java.io.PrintWriter;
import java.io.StringWriter;
import java.time.ZoneId;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
@@ -120,6 +121,7 @@ public class TrinoConnectorJniScanner extends JniScanner {
private List<TrinoColumnMetadata> columnMetadataList =
Lists.newArrayList();
private DynamicFilter dynamicFilter = DynamicFilter.EMPTY;
private List<Type> trinoTypeList;
+ private long[] appendDataTimeNs;
private final TrinoConnectorColumnValue columnValue = new
TrinoConnectorColumnValue();
@@ -199,6 +201,7 @@ public class TrinoConnectorJniScanner extends JniScanner {
break;
}
for (int i = 0; i < page.getChannelCount(); ++i) {
+ long startTime = System.nanoTime();
Block block = page.getBlock(i);
columnValue.setBlock(block);
columnValue.setColumnType(types[i]);
@@ -208,6 +211,7 @@ public class TrinoConnectorJniScanner extends JniScanner {
columnValue.setPosition(j);
appendData(i, columnValue);
}
+ appendDataTimeNs[i] += System.nanoTime() - startTime;
}
rows += page.getPositionCount();
if (rows >= batchSize) {
@@ -227,6 +231,15 @@ public class TrinoConnectorJniScanner extends JniScanner {
return null;
}
+ @Override
+ public Map<String, String> getStatistics() {
+ Map<String, String> mp = new HashMap<>();
+ for (int i = 0; i < appendDataTimeNs.length; ++i) {
+ mp.put("timer:AppendDataTime[" + i + "]",
String.valueOf(appendDataTimeNs[i]));
+ }
+ return mp;
+ }
+
private ConnectorPageSourceProvider getConnectorPageSourceProvider() {
ConnectorPageSourceProvider connectorPageSourceProvider = null;
try {
@@ -323,6 +336,7 @@ public class TrinoConnectorJniScanner extends JniScanner {
private void parseRequiredTypes() {
ColumnType[] columnTypes = new ColumnType[fields.length];
+ appendDataTimeNs = new long[fields.length];
trinoTypeList = Lists.newArrayList();
for (int i = 0; i < fields.length; i++) {
int index = trinoConnectorAllFieldNames.indexOf(fields[i]);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
