This is an automated email from the ASF dual-hosted git repository.
diwu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-spark-connector.git
The following commit(s) were added to refs/heads/master by this push:
new d99ec04 [chord] add log for arrow flight read (#354)
d99ec04 is described below
commit d99ec042910a1e99f896863d19a5474dadb9c321
Author: wudi <[email protected]>
AuthorDate: Tue Apr 7 10:28:19 2026 +0800
[chord] add log for arrow flight read (#354)
add log for arrow flight read
---
.../doris/spark/client/read/DorisFlightSqlReader.java | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
diff --git
a/spark-doris-connector/spark-doris-connector-base/src/main/java/org/apache/doris/spark/client/read/DorisFlightSqlReader.java
b/spark-doris-connector/spark-doris-connector-base/src/main/java/org/apache/doris/spark/client/read/DorisFlightSqlReader.java
index 112b2f8..b691f83 100644
---
a/spark-doris-connector/spark-doris-connector-base/src/main/java/org/apache/doris/spark/client/read/DorisFlightSqlReader.java
+++
b/spark-doris-connector/spark-doris-connector-base/src/main/java/org/apache/doris/spark/client/read/DorisFlightSqlReader.java
@@ -38,6 +38,7 @@ import org.apache.arrow.adbc.driver.flightsql.FlightSqlDriver;
import org.apache.arrow.flight.Location;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
@@ -45,6 +46,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -61,6 +63,8 @@ public class DorisFlightSqlReader extends DorisReader {
private AdbcConnection connection;
private final ArrowReader arrowReader;
private final Boolean datetimeJava8ApiEnabled;
+ private int totalBatches = 0;
+ private long totalRows = 0;
public DorisFlightSqlReader(DorisReaderPartition partition) throws
Exception {
super(partition);
@@ -91,7 +95,18 @@ public class DorisFlightSqlReader extends DorisReader {
public boolean hasNext() throws DorisException {
if (!endOfStream.get() && (rowBatch == null || !rowBatch.hasNext())) {
try {
+ long batchStart = System.currentTimeMillis();
endOfStream.set(!arrowReader.loadNextBatch());
+ if (!endOfStream.get()) {
+ VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+ int rows = root.getRowCount();
+ totalBatches++;
+ totalRows += rows;
+ log.info("Batch loaded: tablet={}, rows={}, cost={}ms",
+ Arrays.toString(partition.getTablets()), rows,
System.currentTimeMillis() - batchStart);
+ } else {
+ log.info("No more data from tablet={}",
Arrays.toString(partition.getTablets()));
+ }
} catch (IOException e) {
throw new DorisException(e);
}
@@ -112,6 +127,8 @@ public class DorisFlightSqlReader extends DorisReader {
@Override
public void close() {
+ log.info("Partition read finished: tablet={}, batches={},
totalRows={}",
+ partition.getTablets(), totalBatches, totalRows);
if (rowBatch != null) {
rowBatch.close();
}
@@ -147,7 +164,9 @@ public class DorisFlightSqlReader extends DorisReader {
String flightSql = generateQuerySql(partition);
log.info("Query SQL Sending to Doris FE is: {}", flightSql);
statement.setSqlQuery(flightSql);
+ long start = System.currentTimeMillis();
AdbcStatement.QueryResult queryResult = statement.executeQuery();
+ log.info("Query submitted, tablet={}, cost={}ms",
Arrays.toString(partition.getTablets()), System.currentTimeMillis() - start);
return queryResult.getReader();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]