This is an automated email from the ASF dual-hosted git repository.

diwu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-spark-connector.git


The following commit(s) were added to refs/heads/master by this push:
     new d99ec04  [chord] add log for arrow flight read (#354)
d99ec04 is described below

commit d99ec042910a1e99f896863d19a5474dadb9c321
Author: wudi <[email protected]>
AuthorDate: Tue Apr 7 10:28:19 2026 +0800

    [chord] add log for arrow flight read (#354)
    
    add log for arrow flight read
---
 .../doris/spark/client/read/DorisFlightSqlReader.java | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git 
a/spark-doris-connector/spark-doris-connector-base/src/main/java/org/apache/doris/spark/client/read/DorisFlightSqlReader.java
 
b/spark-doris-connector/spark-doris-connector-base/src/main/java/org/apache/doris/spark/client/read/DorisFlightSqlReader.java
index 112b2f8..b691f83 100644
--- 
a/spark-doris-connector/spark-doris-connector-base/src/main/java/org/apache/doris/spark/client/read/DorisFlightSqlReader.java
+++ 
b/spark-doris-connector/spark-doris-connector-base/src/main/java/org/apache/doris/spark/client/read/DorisFlightSqlReader.java
@@ -38,6 +38,7 @@ import org.apache.arrow.adbc.driver.flightsql.FlightSqlDriver;
 import org.apache.arrow.flight.Location;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.ipc.ArrowReader;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
@@ -45,6 +46,7 @@ import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -61,6 +63,8 @@ public class DorisFlightSqlReader extends DorisReader {
     private AdbcConnection connection;
     private final ArrowReader arrowReader;
     private final Boolean datetimeJava8ApiEnabled;
+    private int totalBatches = 0;
+    private long totalRows = 0;
 
     public DorisFlightSqlReader(DorisReaderPartition partition) throws 
Exception {
         super(partition);
@@ -91,7 +95,18 @@ public class DorisFlightSqlReader extends DorisReader {
     public boolean hasNext() throws DorisException {
         if (!endOfStream.get() && (rowBatch == null || !rowBatch.hasNext())) {
             try {
+                long batchStart = System.currentTimeMillis();
                 endOfStream.set(!arrowReader.loadNextBatch());
+                if (!endOfStream.get()) {
+                    VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+                    int rows = root.getRowCount();
+                    totalBatches++;
+                    totalRows += rows;
+                    log.info("Batch loaded: tablet={}, rows={}, cost={}ms",
+                            Arrays.toString(partition.getTablets()), rows, 
System.currentTimeMillis() - batchStart);
+                } else {
+                    log.info("No more data from tablet={}", 
Arrays.toString(partition.getTablets()));
+                }
             } catch (IOException e) {
                 throw new DorisException(e);
             }
@@ -112,6 +127,8 @@ public class DorisFlightSqlReader extends DorisReader {
 
     @Override
     public void close() {
+        log.info("Partition read finished: tablet={}, batches={}, 
totalRows={}",
+                partition.getTablets(), totalBatches, totalRows);
         if (rowBatch != null) {
             rowBatch.close();
         }
@@ -147,7 +164,9 @@ public class DorisFlightSqlReader extends DorisReader {
         String flightSql = generateQuerySql(partition);
         log.info("Query SQL Sending to Doris FE is: {}", flightSql);
         statement.setSqlQuery(flightSql);
+        long start = System.currentTimeMillis();
         AdbcStatement.QueryResult queryResult = statement.executeQuery();
+        log.info("Query submitted, tablet={}, cost={}ms", 
Arrays.toString(partition.getTablets()), System.currentTimeMillis() - start);
         return queryResult.getReader();
     }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to