This is an automated email from the ASF dual-hosted git repository. Wei-hao-Li pushed a commit to branch fragmentCSRPC in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a5fa2a56d794007743041617aaa0f6b124823232 Author: Weihao Li <[email protected]> AuthorDate: Thu Nov 12 16:46:35 2026 +0800 draft Signed-off-by: Weihao Li <[email protected]> --- .../v1/handler/QueryDataSetHandlerTest.java | 7 + .../org/apache/iotdb/isession/SessionDataSet.java | 100 +++++++++++- .../main/java/org/apache/iotdb/jdbc/Config.java | 2 + .../org/apache/iotdb/jdbc/IoTDBConnection.java | 2 + .../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java | 100 +++++++++++- .../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 10 +- .../iotdb/jdbc/IoTDBTablePreparedStatement.java | 5 +- iotdb-client/service-rpc/pom.xml | 5 + .../en/org/apache/iotdb/rpc/i18n/RpcMessages.java | 11 ++ .../zh/org/apache/iotdb/rpc/i18n/RpcMessages.java | 11 ++ .../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 172 ++++++++++++++++++++- .../apache/iotdb/session/SessionConnection.java | 32 +++- .../org/apache/iotdb/session/ThriftConnection.java | 7 +- .../iotdb/db/i18n/DataNodeQueryMessages.java | 7 + .../iotdb/db/i18n/DataNodeQueryMessages.java | 7 + .../iotdb/db/protocol/session/IClientSession.java | 11 ++ .../protocol/thrift/impl/ClientRPCServiceImpl.java | 120 ++++++++++++-- .../plan/execution/IQueryExecution.java | 6 + .../queryengine/plan/execution/QueryExecution.java | 8 + .../plan/execution/config/ConfigExecution.java | 9 ++ .../apache/iotdb/db/utils/QueryDataSetUtils.java | 155 +++++++++++++++++++ .../operator/MergeTreeSortOperatorTest.java | 7 + .../iotdb/db/utils/QueryDataSetUtilsTest.java | 154 ++++++++++++++++++ .../conf/iotdb-system.properties.template | 3 + .../thrift-datanode/src/main/thrift/client.thrift | 12 ++ 25 files changed, 920 insertions(+), 43 deletions(-) diff --git a/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/v1/handler/QueryDataSetHandlerTest.java b/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/v1/handler/QueryDataSetHandlerTest.java index 36a2ec1292e..f8aa9c95a2b 100644 --- a/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/v1/handler/QueryDataSetHandlerTest.java +++ b/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/v1/handler/QueryDataSetHandlerTest.java @@ -26,6 +26,7 @@ import org.apache.iotdb.db.queryengine.common.header.DatasetHeader; import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult; import org.apache.iotdb.db.queryengine.plan.execution.IQueryExecution; +import org.apache.iotdb.db.queryengine.plan.execution.PendingResultFragment; import org.apache.iotdb.rest.protocol.model.ExecutionStatus; import org.apache.iotdb.rest.protocol.v1.model.QueryDataSet; import org.apache.iotdb.rpc.TSStatusCode; @@ -88,6 +89,7 @@ public class QueryDataSetHandlerTest { private final Queue<Optional<TsBlock>> batches = new ArrayDeque<>(); private final DatasetHeader datasetHeader; + private final PendingResultFragment pendingResultFragment = new PendingResultFragment(); private TestQueryExecution(TsBlock... tsBlocks) { for (TsBlock tsBlock : tsBlocks) { @@ -128,6 +130,11 @@ public class QueryDataSetHandlerTest { return Optional.empty(); } + @Override + public PendingResultFragment getPendingResultFragment() { + return pendingResultFragment; + } + @Override public boolean hasNextResult() { return !batches.isEmpty(); diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java index 5076afd52d0..511805e5b8f 100644 --- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java @@ -61,7 +61,10 @@ public class SessionDataSet implements ISessionDataSet { ZoneId zoneId, int timeFactor, boolean tableModel, - List<Integer> columnIndex2TsBlockColumnIndexList) { + List<Integer> columnIndex2TsBlockColumnIndexList, + boolean offsetSet, + int offset, + int totalLength) { this.ioTDBRpcDataSet = new IoTDBRpcDataSet( sql, @@ -81,7 +84,48 @@ public class SessionDataSet implements ISessionDataSet { RpcUtils.DEFAULT_TIME_FORMAT, timeFactor, tableModel, - columnIndex2TsBlockColumnIndexList); + columnIndex2TsBlockColumnIndexList, + offsetSet, + offset, + totalLength); + } + + @SuppressWarnings("squid:S107") // ignore Methods should not have too many parameters + public SessionDataSet( + String sql, + List<String> columnNameList, + List<String> columnTypeList, + Map<String, Integer> columnNameIndex, + long queryId, + long statementId, + IClientRPCService.Iface client, + long sessionId, + List<ByteBuffer> queryResult, + boolean ignoreTimeStamp, + boolean moreData, + ZoneId zoneId, + int timeFactor, + boolean tableModel, + List<Integer> columnIndex2TsBlockColumnIndexList) { + this( + sql, + columnNameList, + columnTypeList, + columnNameIndex, + queryId, + statementId, + client, + sessionId, + queryResult, + ignoreTimeStamp, + moreData, + zoneId, + timeFactor, + tableModel, + columnIndex2TsBlockColumnIndexList, + false, + 0, + 0); } @SuppressWarnings("squid:S107") // ignore Methods should not have too many parameters @@ -102,7 +146,10 @@ public class SessionDataSet implements ISessionDataSet { ZoneId zoneId, int timeFactor, boolean tableModel, - List<Integer> columnIndex2TsBlockColumnIndexList) { + List<Integer> columnIndex2TsBlockColumnIndexList, + boolean offsetSet, + int offset, + int totalLength) { this.ioTDBRpcDataSet = new IoTDBRpcDataSet( sql, @@ -122,7 +169,52 @@ public class SessionDataSet implements ISessionDataSet { RpcUtils.DEFAULT_TIME_FORMAT, timeFactor, tableModel, - columnIndex2TsBlockColumnIndexList); + columnIndex2TsBlockColumnIndexList, + offsetSet, + offset, + totalLength); + } + + @SuppressWarnings("squid:S107") // ignore Methods should not have too many parameters + public SessionDataSet( + String sql, + List<String> columnNameList, + List<String> columnTypeList, + Map<String, Integer> columnNameIndex, + long queryId, + long statementId, + IClientRPCService.Iface client, + long sessionId, + List<ByteBuffer> queryResult, + boolean ignoreTimeStamp, + long timeout, + boolean moreData, + int fetchSize, + ZoneId zoneId, + int timeFactor, + boolean tableModel, + List<Integer> columnIndex2TsBlockColumnIndexList) { + this( + sql, + columnNameList, + columnTypeList, + columnNameIndex, + queryId, + statementId, + client, + sessionId, + queryResult, + ignoreTimeStamp, + timeout, + moreData, + fetchSize, + zoneId, + timeFactor, + tableModel, + columnIndex2TsBlockColumnIndexList, + false, + 0, + 0); } public int getFetchSize() { diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/Config.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/Config.java index 54aca784d15..79517b5b938 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/Config.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/Config.java @@ -93,4 +93,6 @@ public class Config { public static final String SQL_DIALECT = "sql_dialect"; public static final String DATABASE = "db"; + + public static final String ENABLE_RESULT_CHUNKING = "enable_result_chunking"; } diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java index 96beecffe1f..f6385249e6b 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java @@ -601,6 +601,8 @@ public class IoTDBConnection implements Connection { openReq.setZoneId(getTimeZone()); openReq.putToConfiguration(Config.VERSION, params.getVersion().toString()); openReq.putToConfiguration(Config.SQL_DIALECT, params.getSqlDialect()); + // this client can assemble fragmented query results, see TSFetchResultsReq#offset + openReq.putToConfiguration(Config.ENABLE_RESULT_CHUNKING, Boolean.TRUE.toString()); params.getDb().ifPresent(db -> openReq.putToConfiguration(Config.DATABASE, db)); TSOpenSessionResp openResp = null; diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java index 5d5cffdf545..8cf3312be12 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java @@ -104,7 +104,10 @@ public class IoTDBJDBCResultSet implements ResultSet { ZoneId zoneId, Charset charset, boolean tableModel, - List<Integer> columnIndex2TsBlockColumnIndexList) + List<Integer> columnIndex2TsBlockColumnIndexList, + boolean offsetSet, + int offset, + int totalLength) throws SQLException { this.ioTDBRpcDataSet = new IoTDBRpcDataSet( @@ -125,7 +128,10 @@ public class IoTDBJDBCResultSet implements ResultSet { timeFormat, statement.getTimeFactor(), tableModel, - columnIndex2TsBlockColumnIndexList); + columnIndex2TsBlockColumnIndexList, + offsetSet, + offset, + totalLength); this.statement = statement; this.columnTypeList = columnTypeList; if (tracingInfo != null) { @@ -135,6 +141,49 @@ public class IoTDBJDBCResultSet implements ResultSet { this.charset = charset; } + @SuppressWarnings("squid:S107") // ignore Methods should not have too many parameters + public IoTDBJDBCResultSet( + IoTDBStatement statement, + List<String> columnNameList, + List<String> columnTypeList, + Map<String, Integer> columnNameIndex, + boolean ignoreTimeStamp, + IClientRPCService.Iface client, + String sql, + long queryId, + long sessionId, + List<ByteBuffer> dataSet, + TSTracingInfo tracingInfo, + long timeout, + boolean moreData, + ZoneId zoneId, + Charset charset, + boolean tableModel, + List<Integer> columnIndex2TsBlockColumnIndexList) + throws SQLException { + this( + statement, + columnNameList, + columnTypeList, + columnNameIndex, + ignoreTimeStamp, + client, + sql, + queryId, + sessionId, + dataSet, + tracingInfo, + timeout, + moreData, + zoneId, + charset, + tableModel, + columnIndex2TsBlockColumnIndexList, + false, + 0, + 0); + } + @SuppressWarnings("squid:S107") // ignore Methods should not have too many parameters public IoTDBJDBCResultSet( Statement statement, @@ -150,7 +199,10 @@ public class IoTDBJDBCResultSet implements ResultSet { TSTracingInfo tracingInfo, long timeout, boolean moreData, - ZoneId zoneId) + ZoneId zoneId, + boolean offsetSet, + int offset, + int totalLength) throws SQLException { this.ioTDBRpcDataSet = new IoTDBRpcDataSet( @@ -171,7 +223,10 @@ public class IoTDBJDBCResultSet implements ResultSet { timeFormat, ((IoTDBStatement) statement).getTimeFactor(), false, - IntStream.range(0, columnNameList.size()).boxed().collect(Collectors.toList())); + IntStream.range(0, columnNameList.size()).boxed().collect(Collectors.toList()), + offsetSet, + offset, + totalLength); this.statement = (IoTDBStatement) statement; this.columnTypeList = columnTypeList; if (tracingInfo != null) { @@ -180,6 +235,43 @@ public class IoTDBJDBCResultSet implements ResultSet { } } + @SuppressWarnings("squid:S107") // ignore Methods should not have too many parameters + public IoTDBJDBCResultSet( + Statement statement, + List<String> columnNameList, + List<String> columnTypeList, + Map<String, Integer> columnNameIndex, + boolean ignoreTimeStamp, + IClientRPCService.Iface client, + String sql, + long queryId, + long sessionId, + List<ByteBuffer> dataSet, + TSTracingInfo tracingInfo, + long timeout, + boolean moreData, + ZoneId zoneId) + throws SQLException { + this( + statement, + columnNameList, + columnTypeList, + columnNameIndex, + ignoreTimeStamp, + client, + sql, + queryId, + sessionId, + dataSet, + tracingInfo, + timeout, + moreData, + zoneId, + false, + 0, + 0); + } + @Override public boolean isWrapperFor(Class<?> iface) throws SQLException { throw new SQLException(Constant.METHOD_NOT_SUPPORTED); diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java index ff790cbd9f4..858462d6b24 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java @@ -412,7 +412,10 @@ public class IoTDBStatement implements Statement { zoneId, charset, execResp.isSetTableModel() && execResp.isTableModel(), - execResp.getColumnIndex2TsBlockColumnIndexList()); + execResp.getColumnIndex2TsBlockColumnIndexList(), + execResp.isSetOffset(), + execResp.offset, + execResp.totalLength); } return true; } @@ -549,7 +552,10 @@ public class IoTDBStatement implements Statement { zoneId, charset, execResp.isSetTableModel() && execResp.isTableModel(), - execResp.getColumnIndex2TsBlockColumnIndexList()); + execResp.getColumnIndex2TsBlockColumnIndexList(), + execResp.isSetOffset(), + execResp.offset, + execResp.totalLength); } return resultSet; } diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBTablePreparedStatement.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBTablePreparedStatement.java index 3c95afa107d..72cd6925498 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBTablePreparedStatement.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBTablePreparedStatement.java @@ -233,7 +233,10 @@ public class IoTDBTablePreparedStatement extends IoTDBStatement implements Prepa resp.tracingInfo, (long) queryTimeout * 1000, resp.isSetMoreData() && resp.isMoreData(), - zoneId); + zoneId, + resp.isSetOffset(), + resp.offset, + resp.totalLength); return resultSet; } return null; diff --git a/iotdb-client/service-rpc/pom.xml b/iotdb-client/service-rpc/pom.xml index 206cc949fe4..d53db34b388 100644 --- a/iotdb-client/service-rpc/pom.xml +++ b/iotdb-client/service-rpc/pom.xml @@ -75,6 +75,11 @@ <artifactId>junit</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <scope>test</scope> + </dependency> <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> diff --git a/iotdb-client/service-rpc/src/main/i18n/en/org/apache/iotdb/rpc/i18n/RpcMessages.java b/iotdb-client/service-rpc/src/main/i18n/en/org/apache/iotdb/rpc/i18n/RpcMessages.java index cd925b4863a..862e15b3b90 100644 --- a/iotdb-client/service-rpc/src/main/i18n/en/org/apache/iotdb/rpc/i18n/RpcMessages.java +++ b/iotdb-client/service-rpc/src/main/i18n/en/org/apache/iotdb/rpc/i18n/RpcMessages.java @@ -127,4 +127,15 @@ public final class RpcMessages { public static final String EXCEPTION_DATA_TYPE_ARG_IS_NOT_SUPPORTED_31213160 = "Data type %s is not supported."; + + public static final String EXCEPTION_CHUNKED_RESULT_EMPTY_FRAGMENT_03A0EF39 = + "Empty fragment received when assembling a chunked query result"; + public static final String + EXCEPTION_CHUNKED_RESULT_FRAGMENT_OFFSET_ARG_NOT_GREATER_THAN_ACCUMULATED_LENGTH_ARG_A0F032E3 = + "Fragment offset %d is not greater than the accumulated length %d"; + public static final String EXCEPTION_CHUNKED_RESULT_MISSING_TOTAL_LENGTH_FOR_FIRST_FRAGMENT_1D11374F = + "Missing totalLength for the first fragment of a chunked TsBlock"; + public static final String + EXCEPTION_CHUNKED_RESULT_ASSEMBLED_LENGTH_ARG_DOES_NOT_MATCH_TOTAL_LENGTH_ARG_925DCB15 = + "Assembled fragment length %d does not match the total length %d"; } diff --git a/iotdb-client/service-rpc/src/main/i18n/zh/org/apache/iotdb/rpc/i18n/RpcMessages.java b/iotdb-client/service-rpc/src/main/i18n/zh/org/apache/iotdb/rpc/i18n/RpcMessages.java index fc059e26f5f..0f448debed5 100644 --- a/iotdb-client/service-rpc/src/main/i18n/zh/org/apache/iotdb/rpc/i18n/RpcMessages.java +++ b/iotdb-client/service-rpc/src/main/i18n/zh/org/apache/iotdb/rpc/i18n/RpcMessages.java @@ -108,4 +108,15 @@ public final class RpcMessages { "通过列名 %s 获取的值为 NULL。"; public static final String EXCEPTION_DATA_TYPE_ARG_IS_NOT_SUPPORTED_31213160 = "不支持数据类型 %s。"; + + public static final String EXCEPTION_CHUNKED_RESULT_EMPTY_FRAGMENT_03A0EF39 = + "拼装分片查询结果时收到空分片"; + public static final String + EXCEPTION_CHUNKED_RESULT_FRAGMENT_OFFSET_ARG_NOT_GREATER_THAN_ACCUMULATED_LENGTH_ARG_A0F032E3 = + "分片 offset %d 不大于已累计的长度 %d"; + public static final String EXCEPTION_CHUNKED_RESULT_MISSING_TOTAL_LENGTH_FOR_FIRST_FRAGMENT_1D11374F = + "分片 TsBlock 的首个分片缺少 totalLength"; + public static final String + EXCEPTION_CHUNKED_RESULT_ASSEMBLED_LENGTH_ARG_DOES_NOT_MATCH_TOTAL_LENGTH_ARG_925DCB15 = + "拼装后的分片长度 %d 与 totalLength %d 不一致"; } diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java index 32f5ea5d45a..f9669d56d7a 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java @@ -34,6 +34,7 @@ import org.apache.tsfile.read.common.block.column.TsBlockSerde; import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.DateUtils; +import org.apache.tsfile.utils.PublicBAOS; import java.nio.ByteBuffer; import java.sql.Timestamp; @@ -84,6 +85,8 @@ public class IoTDBRpcDataSet { private int queryResultIndex; // the index of bytebuffer in queryResult private int tsBlockSize; // the size of current tsBlock private int tsBlockIndex; // the row index in current tsBlock + // the fragment of a TsBlock that is partially received; null if there is no pending fragment + private ResultFragmentBuffer fragmentBuffer; private final ZoneId zoneId; private final String timeFormat; @@ -111,6 +114,53 @@ public class IoTDBRpcDataSet { int timeFactor, boolean tableModel, List<Integer> columnIndex2TsBlockColumnIndexList) { + this( + sql, + columnNameList, + columnTypeList, + columnNameIndex, + ignoreTimeStamp, + moreData, + queryId, + statementId, + client, + sessionId, + queryResult, + fetchSize, + timeout, + zoneId, + timeFormat, + timeFactor, + tableModel, + columnIndex2TsBlockColumnIndexList, + false, + 0, + 0); + } + + @SuppressWarnings({"squid:S3776", "squid:S107"}) // Suppress high Cognitive Complexity warning + public IoTDBRpcDataSet( + String sql, + List<String> columnNameList, + List<String> columnTypeList, + Map<String, Integer> columnNameIndex, + boolean ignoreTimeStamp, + boolean moreData, + long queryId, + long statementId, + IClientRPCService.Iface client, + long sessionId, + List<ByteBuffer> queryResult, + int fetchSize, + long timeout, + ZoneId zoneId, + String timeFormat, + int timeFactor, + boolean tableModel, + List<Integer> columnIndex2TsBlockColumnIndexList, + boolean offsetSet, + int offset, + int totalLength) { this.sessionId = sessionId; this.statementId = statementId; // only used for tree model, table model this field will always be true @@ -184,12 +234,13 @@ public class IoTDBRpcDataSet { } } - this.queryResult = queryResult; + this.queryResult = new ArrayList<>(); + this.queryResultIndex = 0; this.queryResultSize = 0; if (queryResult != null) { - queryResultSize = queryResult.size(); + addResponse(queryResult, offsetSet, offset, totalLength); + this.queryResultSize = this.queryResult.size(); } - this.queryResultIndex = 0; this.tsBlockSize = 0; this.tsBlockIndex = -1; this.zoneId = zoneId; @@ -227,6 +278,7 @@ public class IoTDBRpcDataSet { } client = null; isClosed = true; + fragmentBuffer = null; } public boolean next() throws StatementExecutionException, IoTDBConnectionException { @@ -241,7 +293,13 @@ public class IoTDBRpcDataSet { return true; } - if (moreData && fetchResults() && hasCachedByteBuffer()) { + // one oversized TsBlock may need several fetch RPCs before it is fully assembled + while (moreData && !hasCachedByteBuffer()) { + if (!fetchResults()) { + break; + } + } + if (hasCachedByteBuffer()) { constructOneTsBlock(); constructOneRow(); return true; @@ -262,6 +320,9 @@ public class IoTDBRpcDataSet { TSFetchResultsReq req = new TSFetchResultsReq(sessionId, sql, fetchSize, queryId, true); req.setStatementId(statementId); req.setTimeout(timeout); + // a new client always sets offset (0 if nothing pending), which enables bounded-payload + // responses on the server + req.setOffset(fragmentBuffer == null ? 0 : fragmentBuffer.getNextOffset()); try { TSFetchResultsResp resp = client.fetchResultsV2(req); RpcUtils.verifySuccess(resp.getStatus()); @@ -269,12 +330,11 @@ public class IoTDBRpcDataSet { if (!resp.hasResultSet) { close(); } else { - queryResult = resp.getQueryResult(); + queryResult = new ArrayList<>(); queryResultIndex = 0; queryResultSize = 0; - if (queryResult != null) { - queryResultSize = queryResult.size(); - } + addResponse(resp.getQueryResult(), resp.isSetOffset(), resp.offset, resp.totalLength); + queryResultSize = queryResult.size(); this.tsBlockSize = 0; this.tsBlockIndex = -1; } @@ -286,6 +346,102 @@ public class IoTDBRpcDataSet { } } + /** + * Adds the blocks of one response to {@link #queryResult}. If the response ends with a fragment, + * the fragment is either appended to the pending {@link ResultFragmentBuffer} or starts a new + * one; the corresponding TsBlock is only added to {@link #queryResult} once all of its bytes have + * been received. + * + * @param blocks the serialized TsBlocks in the response + * @param offsetSet whether the last element of {@code blocks} is a fragment + * @param nextOffset the start offset of the next fragment + * @param totalLength the total serialized length of the block the first fragment belongs to + */ + private void addResponse( + List<ByteBuffer> blocks, boolean offsetSet, int nextOffset, int totalLength) { + if (blocks == null || blocks.isEmpty()) { + return; + } + int blockIndex = 0; + + if (fragmentBuffer != null) { + // the first element completes the pending fragment, if it is not the only element + fragmentBuffer.append(blocks.get(blockIndex++)); + if (offsetSet && blockIndex == blocks.size()) { + fragmentBuffer.updateOffset(nextOffset); + return; + } + queryResult.add(fragmentBuffer.toByteBuffer()); + fragmentBuffer = null; + } + + int lastCompleteBlockIndex = offsetSet ? blocks.size() - 1 : blocks.size(); + while (blockIndex < lastCompleteBlockIndex) { + queryResult.add(blocks.get(blockIndex++)); + } + + if (offsetSet) { + if (totalLength <= 0) { + throw new IllegalArgumentException( + RpcMessages.EXCEPTION_CHUNKED_RESULT_MISSING_TOTAL_LENGTH_FOR_FIRST_FRAGMENT_1D11374F); + } + fragmentBuffer = new ResultFragmentBuffer(totalLength); + fragmentBuffer.append(blocks.get(blockIndex)); + fragmentBuffer.updateOffset(nextOffset); + } + } + + /** Accumulates the fragments of one oversized serialized TsBlock until it is complete. */ + private static class ResultFragmentBuffer { + + private final PublicBAOS buffer; + private final int totalLength; + private int nextOffset; + + private ResultFragmentBuffer(int totalLength) { + this.buffer = new PublicBAOS(totalLength); + this.totalLength = totalLength; + } + + private int getNextOffset() { + return nextOffset; + } + + private void append(ByteBuffer fragment) { + if (!fragment.hasRemaining()) { + throw new IllegalArgumentException( + RpcMessages.EXCEPTION_CHUNKED_RESULT_EMPTY_FRAGMENT_03A0EF39); + } + // a fragment may be a slice of a larger buffer, so honor arrayOffset and position + buffer.write( + fragment.array(), fragment.arrayOffset() + fragment.position(), fragment.remaining()); + } + + private void updateOffset(int nextOffset) { + if (nextOffset <= this.nextOffset || nextOffset != buffer.size()) { + throw new IllegalArgumentException( + String.format( + RpcMessages + .EXCEPTION_CHUNKED_RESULT_FRAGMENT_OFFSET_ARG_NOT_GREATER_THAN_ACCUMULATED_LENGTH_ARG_A0F032E3, + nextOffset, + buffer.size())); + } + this.nextOffset = nextOffset; + } + + private ByteBuffer toByteBuffer() { + if (buffer.size() != totalLength) { + throw new IllegalArgumentException( + String.format( + RpcMessages + .EXCEPTION_CHUNKED_RESULT_ASSEMBLED_LENGTH_ARG_DOES_NOT_MATCH_TOTAL_LENGTH_ARG_925DCB15, + buffer.size(), + totalLength)); + } + return ByteBuffer.wrap(buffer.getBuf(), 0, buffer.size()); + } + } + public boolean hasCachedBlock() { return (curTsBlock != null && tsBlockIndex < tsBlockSize - 1); } diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java index f2158ef0ffa..d0b6e8dba52 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java @@ -246,6 +246,8 @@ public class SessionConnection { openReq.setZoneId(zoneId.toString()); openReq.putToConfiguration("version", session.version.toString()); openReq.putToConfiguration("sql_dialect", sqlDialect); + // this client can assemble fragmented query results, see TSFetchResultsReq#offset + openReq.putToConfiguration("enable_result_chunking", Boolean.TRUE.toString()); if (session.useEncryptedPassword) { openReq.putToConfiguration(USE_ENCRYPTED_PASSWORD_KEY, Boolean.TRUE.toString()); } @@ -492,7 +494,10 @@ public class SessionConnection { zoneId, timeFactor, execResp.isSetTableModel() && execResp.isTableModel(), - execResp.getColumnIndex2TsBlockColumnIndexList()); + execResp.getColumnIndex2TsBlockColumnIndexList(), + execResp.isSetOffset(), + execResp.offset, + execResp.totalLength); } protected void executeNonQueryStatement(String sql) @@ -561,7 +566,10 @@ public class SessionConnection { zoneId, timeFactor, execResp.isSetTableModel() && execResp.isTableModel(), - execResp.getColumnIndex2TsBlockColumnIndexList()); + execResp.getColumnIndex2TsBlockColumnIndexList(), + execResp.isSetOffset(), + execResp.offset, + execResp.totalLength); } protected SessionDataSet executeLastDataQueryForOnePrefixPath(final List<String> prefixes) @@ -601,7 +609,10 @@ public class SessionConnection { zoneId, timeFactor, false, - tsExecuteStatementResp.getColumnIndex2TsBlockColumnIndexList()); + tsExecuteStatementResp.getColumnIndex2TsBlockColumnIndexList(), + tsExecuteStatementResp.isSetOffset(), + tsExecuteStatementResp.offset, + tsExecuteStatementResp.totalLength); } protected Pair<SessionDataSet, TEndPoint> executeLastDataQueryForOneDevice( @@ -651,7 +662,10 @@ public class SessionConnection { zoneId, timeFactor, tsExecuteStatementResp.isSetTableModel() && tsExecuteStatementResp.isTableModel(), - tsExecuteStatementResp.getColumnIndex2TsBlockColumnIndexList()), + tsExecuteStatementResp.getColumnIndex2TsBlockColumnIndexList(), + tsExecuteStatementResp.isSetOffset(), + tsExecuteStatementResp.offset, + tsExecuteStatementResp.totalLength), redirectedEndPoint); } @@ -694,7 +708,10 @@ public class SessionConnection { zoneId, timeFactor, tsExecuteStatementResp.isSetTableModel() && tsExecuteStatementResp.isTableModel(), - tsExecuteStatementResp.getColumnIndex2TsBlockColumnIndexList()); + tsExecuteStatementResp.getColumnIndex2TsBlockColumnIndexList(), + tsExecuteStatementResp.isSetOffset(), + tsExecuteStatementResp.offset, + tsExecuteStatementResp.totalLength); } protected SessionDataSet executeAggregationQuery( @@ -776,7 +793,10 @@ public class SessionConnection { zoneId, timeFactor, tsExecuteStatementResp.isSetTableModel() && tsExecuteStatementResp.isTableModel(), - tsExecuteStatementResp.getColumnIndex2TsBlockColumnIndexList()); + tsExecuteStatementResp.getColumnIndex2TsBlockColumnIndexList(), + tsExecuteStatementResp.isSetOffset(), + tsExecuteStatementResp.offset, + tsExecuteStatementResp.totalLength); } private TSAggregationQueryReq createAggregationQueryReq( diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/ThriftConnection.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/ThriftConnection.java index e15320c1a15..dd3a374c21b 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/ThriftConnection.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/ThriftConnection.java @@ -147,6 +147,8 @@ public class ThriftConnection { openReq.setPassword(password); openReq.setZoneId(zoneId.toString()); openReq.putToConfiguration("version", version); + // this client can assemble fragmented query results, see TSFetchResultsReq#offset + openReq.putToConfiguration("enable_result_chunking", Boolean.TRUE.toString()); try { TSOpenSessionResp openResp = client.openSession(openReq); @@ -210,7 +212,10 @@ public class ThriftConnection { zoneId, timeFactor, execResp.isSetTableModel() && execResp.isTableModel(), - execResp.getColumnIndex2TsBlockColumnIndexList()); + execResp.getColumnIndex2TsBlockColumnIndexList(), + execResp.isSetOffset(), + execResp.offset, + execResp.totalLength); } /** Create outside the publication lock: TLS setup itself can perform network I/O. */ diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index b84bebec93d..8514a5c2db0 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -4930,4 +4930,11 @@ public final class DataNodeQueryMessages { public static final String EXCEPTION_ARG_ARG_MS_ARG_INVOCATIONS_ARG_APPLICATIONS_F056BD84 = "%s: %s ms, %s invocations, %s applications"; + + public static final String + EXCEPTION_RESULT_FRAGMENT_REQUEST_OFFSET_ARG_DOES_NOT_MATCH_PENDING_FRAGMENT_OFFSET_ARG_A1B13DBC = + "Request offset %d does not match the pending result fragment offset %d"; + public static final String + EXCEPTION_RESULT_FRAGMENT_REQUEST_OFFSET_ARG_IS_INVALID_WITHOUT_PENDING_FRAGMENT_C403454F = + "Request offset %d is invalid because no result fragment is pending"; } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 1aef0814d92..b9546ce0d6e 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -5630,4 +5630,11 @@ public final class DataNodeQueryMessages { public static final String EXCEPTION_ARG_ARG_MS_ARG_INVOCATIONS_ARG_APPLICATIONS_F056BD84 = "%s:%s ms,调用 %s 次,应用 %s 次"; + + public static final String + EXCEPTION_RESULT_FRAGMENT_REQUEST_OFFSET_ARG_DOES_NOT_MATCH_PENDING_FRAGMENT_OFFSET_ARG_A1B13DBC = + "请求 offset %d 与待发送结果分片的 offset %d 不一致"; + public static final String + EXCEPTION_RESULT_FRAGMENT_REQUEST_OFFSET_ARG_IS_INVALID_WITHOUT_PENDING_FRAGMENT_C403454F = + "请求 offset %d 无效:当前没有待发送的结果分片"; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/IClientSession.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/IClientSession.java index aaf73db5c4d..1307f71741c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/IClientSession.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/IClientSession.java @@ -61,6 +61,9 @@ public abstract class IClientSession { private boolean enableAutoCreateSchema = true; + // set by clients that can assemble fragmented query results, see TSFetchResultsReq#offset + private boolean enableResultChunking = false; + private long lastActiveTime = CommonDateTimeUtils.currentTime(); public abstract String getClientAddress(); @@ -211,6 +214,14 @@ public abstract class IClientSession { this.enableAutoCreateSchema = enableAutoCreateSchema; } + public boolean isEnableResultChunking() { + return enableResultChunking; + } + + public void setEnableResultChunking(boolean enableResultChunking) { + this.enableResultChunking = enableResultChunking; + } + /** * Add a prepared statement to this session. * diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index 796957d1735..614774a42af 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -273,6 +273,7 @@ public class ClientRPCServiceImpl implements IClientRPCServiceWithHandler { public static final String ERROR_CODE = DataNodeMiscMessages.ERROR_CODE; private static final String USE_ENCRYPTED_PASSWORD_KEY = "use_encrypted_password"; private static final String ENABLE_AUTO_CREATE_SCHEMA_KEY = "enable_auto_create_schema"; + private static final String ENABLE_RESULT_CHUNKING_KEY = "enable_result_chunking"; private static final TSProtocolVersion CURRENT_RPC_VERSION = TSProtocolVersion.IOTDB_SERVICE_PROTOCOL_V3; @@ -302,25 +303,55 @@ public class ClientRPCServiceImpl implements IClientRPCServiceWithHandler { @FunctionalInterface public interface SelectResult { - boolean apply(TSExecuteStatementResp resp, IQueryExecution queryExecution, int fetchSize) + boolean apply( + TSExecuteStatementResp resp, + IQueryExecution queryExecution, + int fetchSize, + int maxPayloadSize) throws IoTDBException, IOException; } private static final SelectResult SELECT_RESULT = - (resp, queryExecution, fetchSize) -> { - Pair<List<ByteBuffer>, Boolean> pair = - QueryDataSetUtils.convertQueryResultByFetchSize(queryExecution, fetchSize); - resp.setQueryResult(pair.left); - return pair.right; + (resp, queryExecution, fetchSize, maxPayloadSize) -> { + QueryDataSetUtils.ResultChunk chunk; + if (maxPayloadSize > 0) { + // The execute request carries no offset, so it must start a new result set + chunk = + QueryDataSetUtils.convertQueryResultByFetchSize( + queryExecution, fetchSize, 0, maxPayloadSize); + } else { + Pair<List<ByteBuffer>, Boolean> pair = + QueryDataSetUtils.convertQueryResultByFetchSize(queryExecution, fetchSize); + chunk = QueryDataSetUtils.ResultChunk.complete(pair.left, pair.right); + } + resp.setQueryResult(chunk.getQueryResult()); + if (chunk.hasNextOffset()) { + resp.setOffset(chunk.getNextOffset()); + if (chunk.hasTotalLength()) { + resp.setTotalLength(chunk.getTotalLength()); + } + } + return chunk.isFinished(); }; private static final SelectResult OLD_SELECT_RESULT = - (resp, queryExecution, fetchSize) -> { + (resp, queryExecution, fetchSize, maxPayloadSize) -> { Pair<TSQueryDataSet, Boolean> pair = convertTsBlockByFetchSize(queryExecution, fetchSize); resp.setQueryDataSet(pair.left); return pair.right; }; + /** + * Returns the maximum payload size of one response carrying query results, or 0 if the session's + * client cannot assemble fragmented results. The payload bound is shared with the MPP data + * exchange RPC response, see mpp_data_exchange_max_payload_size_in_bytes. + */ + private static int getMaxResultPayloadSize(IClientSession clientSession) { + return clientSession.isEnableResultChunking() + ? config.getMppDataExchangeMaxPayloadSizeInBytes() + : 0; + } + public ClientRPCServiceImpl() { partitionFetcher = ClusterPartitionFetcher.getInstance(); schemaFetcher = ClusterSchemaFetcher.getInstance(); @@ -494,7 +525,12 @@ public class ClientRPCServiceImpl implements IClientRPCServiceWithHandler { statementType = statementType == null ? StatementType.QUERY : statementType; resp = createResponse(queryExecution.getDatasetHeader(), queryId); resp.setStatus(result.status); - finished = setResult.apply(resp, queryExecution, request.getFetchSize()); + finished = + setResult.apply( + resp, + queryExecution, + request.getFetchSize(), + getMaxResultPayloadSize(clientSession)); resp.setMoreData(!finished); if (quota != null) { quota.addReadResult(resp.getQueryResult()); @@ -767,7 +803,9 @@ public class ClientRPCServiceImpl implements IClientRPCServiceWithHandler { if (queryExecution.isQuery()) { resp = createResponse(queryExecution.getDatasetHeader(), queryId); resp.setStatus(result.status); - finished = setResult.apply(resp, queryExecution, req.fetchSize); + finished = + setResult.apply( + resp, queryExecution, req.fetchSize, getMaxResultPayloadSize(clientSession)); resp.setMoreData(!finished); quota.addReadResult(resp.getQueryResult()); } else { @@ -863,7 +901,9 @@ public class ClientRPCServiceImpl implements IClientRPCServiceWithHandler { if (queryExecution.isQuery()) { resp = createResponse(queryExecution.getDatasetHeader(), queryId); resp.setStatus(result.status); - finished = setResult.apply(resp, queryExecution, req.fetchSize); + finished = + setResult.apply( + resp, queryExecution, req.fetchSize, getMaxResultPayloadSize(clientSession)); resp.setMoreData(!finished); quota.addReadResult(resp.getQueryResult()); } else { @@ -961,7 +1001,9 @@ public class ClientRPCServiceImpl implements IClientRPCServiceWithHandler { if (queryExecution.isQuery()) { resp = createResponse(queryExecution.getDatasetHeader(), queryId); resp.setStatus(result.status); - finished = setResult.apply(resp, queryExecution, req.fetchSize); + finished = + setResult.apply( + resp, queryExecution, req.fetchSize, getMaxResultPayloadSize(clientSession)); resp.setMoreData(!finished); quota.addReadResult(resp.getQueryResult()); } else { @@ -1456,7 +1498,9 @@ public class ClientRPCServiceImpl implements IClientRPCServiceWithHandler { .get(0) .clientRpcEndPoint); resp.setStatus(tsstatus); - finished = SELECT_RESULT.apply(resp, queryExecution, req.fetchSize); + finished = + SELECT_RESULT.apply( + resp, queryExecution, req.fetchSize, getMaxResultPayloadSize(clientSession)); resp.setMoreData(!finished); quota.addReadResult(resp.getQueryResult()); } else { @@ -1571,6 +1615,27 @@ public class ClientRPCServiceImpl implements IClientRPCServiceWithHandler { statementType = queryExecution.getStatementType(); try (SetThreadName queryName = new SetThreadName(queryExecution.getQueryId())) { + if (req.isSetOffset()) { + // The client sets offset (0 for the first request) iff it can assemble fragment results + QueryDataSetUtils.ResultChunk chunk = + QueryDataSetUtils.convertQueryResultByFetchSize( + queryExecution, + req.fetchSize, + req.getOffset(), + config.getMppDataExchangeMaxPayloadSizeInBytes()); + finished = chunk.isFinished(); + resp.setHasResultSet(!chunk.getQueryResult().isEmpty()); + resp.setIsAlign(true); + resp.setQueryResult(chunk.getQueryResult()); + if (chunk.hasNextOffset()) { + resp.setOffset(chunk.getNextOffset()); + if (chunk.hasTotalLength()) { + resp.setTotalLength(chunk.getTotalLength()); + } + } + resp.setMoreData(!finished); + return resp; + } Pair<List<ByteBuffer>, Boolean> pair = QueryDataSetUtils.convertQueryResultByFetchSize(queryExecution, req.fetchSize); List<ByteBuffer> result = pair.left; @@ -1644,6 +1709,7 @@ public class ClientRPCServiceImpl implements IClientRPCServiceWithHandler { clientSession.setDatabaseName(database.get().toLowerCase(Locale.ENGLISH)); } clientSession.setEnableAutoCreateSchema(parseEnableAutoCreateSchema(req)); + clientSession.setEnableResultChunking(parseEnableResultChunking(req)); } TSOpenSessionResp resp = new TSOpenSessionResp(tsStatus, CURRENT_RPC_VERSION); Map<String, String> configuration = new HashMap<>(); @@ -1694,6 +1760,12 @@ public class ClientRPCServiceImpl implements IClientRPCServiceWithHandler { || Boolean.parseBoolean(configuration.get(ENABLE_AUTO_CREATE_SCHEMA_KEY)); } + private boolean parseEnableResultChunking(TSOpenSessionReq req) { + Map<String, String> configuration = req.configuration; + return configuration != null + && Boolean.parseBoolean(configuration.get(ENABLE_RESULT_CHUNKING_KEY)); + } + @Override public TSStatus closeSession(TSCloseSessionReq req) { return new TSStatus( @@ -1805,12 +1877,26 @@ public class ClientRPCServiceImpl implements IClientRPCServiceWithHandler { } private final SelectResult setResultForPrepared = - (resp, queryExecution, fetchSize) -> { + (resp, queryExecution, fetchSize, maxPayloadSize) -> { // Use V2 format (queryResult) to match IoTDBTablePreparedStatement client - Pair<List<ByteBuffer>, Boolean> pair = - QueryDataSetUtils.convertQueryResultByFetchSize(queryExecution, fetchSize); - resp.setQueryResult(pair.left); - return pair.right; + QueryDataSetUtils.ResultChunk chunk; + if (maxPayloadSize > 0) { + chunk = + QueryDataSetUtils.convertQueryResultByFetchSize( + queryExecution, fetchSize, 0, maxPayloadSize); + } else { + Pair<List<ByteBuffer>, Boolean> pair = + QueryDataSetUtils.convertQueryResultByFetchSize(queryExecution, fetchSize); + chunk = QueryDataSetUtils.ResultChunk.complete(pair.left, pair.right); + } + resp.setQueryResult(chunk.getQueryResult()); + if (chunk.hasNextOffset()) { + resp.setOffset(chunk.getNextOffset()); + if (chunk.hasTotalLength()) { + resp.setTotalLength(chunk.getTotalLength()); + } + } + return chunk.isFinished(); }; // ========================= End PreparedStatement RPC Methods ========================= diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/IQueryExecution.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/IQueryExecution.java index f4beb09917c..1a7b96d7728 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/IQueryExecution.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/IQueryExecution.java @@ -45,6 +45,12 @@ public interface IQueryExecution { Optional<ByteBuffer> getByteBufferBatchResult() throws IoTDBException; + /** + * Returns the holder of the serialized TsBlock that is partially sent to the client. It is used + * to continue the fragmented transfer of one oversized TsBlock in the following RPC response. + */ + PendingResultFragment getPendingResultFragment(); + boolean hasNextResult(); int getOutputValueColumnCount(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java index 2324bea7877..a1ff15a98a1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java @@ -108,6 +108,9 @@ public class QueryExecution implements IQueryExecution { // used for cleaning resultHandle up exactly once private final AtomicBoolean resultHandleCleanUp; + // the serialized TsBlock that is partially sent to the client and waits for the next RPC + private final PendingResultFragment pendingResultFragment = new PendingResultFragment(); + private final AtomicBoolean stopped; // cost time in ns of finished rpc @@ -599,6 +602,11 @@ public class QueryExecution implements IQueryExecution { return getResult(this::getSerializedTsBlock); } + @Override + public PendingResultFragment getPendingResultFragment() { + return pendingResultFragment; + } + private ByteBuffer getSerializedTsBlock() throws IoTDBException { return resultHandle.getSerializedTsBlock(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigExecution.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigExecution.java index 148dab7de71..cfc8cfdeb98 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigExecution.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigExecution.java @@ -31,6 +31,7 @@ import org.apache.iotdb.db.queryengine.execution.QueryStateMachine; import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult; import org.apache.iotdb.db.queryengine.plan.execution.IQueryExecution; +import org.apache.iotdb.db.queryengine.plan.execution.PendingResultFragment; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.ClusterConfigTaskExecutor; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; import org.apache.iotdb.db.queryengine.plan.statement.StatementType; @@ -115,6 +116,9 @@ public class ConfigExecution implements IQueryExecution { private final IConfigTask task; private final IConfigTaskExecutor configTaskExecutor; + // the serialized TsBlock that is partially sent to the client and waits for the next RPC + private final PendingResultFragment pendingResultFragment = new PendingResultFragment(); + private final StatementType statementType; private long totalExecutionTime; @@ -289,6 +293,11 @@ public class ConfigExecution implements IQueryExecution { return Optional.empty(); } + @Override + public PendingResultFragment getPendingResultFragment() { + return pendingResultFragment; + } + // According to the execution process of ConfigExecution, there is only one TsBlock for // this execution. Thus, the hasNextResult will be false once the TsBlock is consumed @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java index bb40b38dc24..cb36050adba 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java @@ -20,7 +20,9 @@ package org.apache.iotdb.db.utils; import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.iotdb.db.queryengine.plan.execution.IQueryExecution; +import org.apache.iotdb.db.queryengine.plan.execution.PendingResultFragment; import org.apache.iotdb.service.rpc.thrift.TSQueryDataSet; import org.apache.tsfile.block.column.Column; @@ -272,6 +274,159 @@ public class QueryDataSetUtils { return new Pair<>(res, !queryExecution.hasNextResult()); } + /** + * Fetches query results with a bounded payload size. A serialized TsBlock larger than {@code + * maxPayloadSize} is cut into fragments: the response only carries the bytes that fit into the + * payload, and the remaining bytes are kept in {@link PendingResultFragment} on the query + * execution, to be sent by the following request carrying the returned offset. + * + * @param queryExecution used to get TsBlock from and judge whether there is more data. + * @param fetchSize wanted row size + * @param requestOffset the offset of the pending fragment to continue, 0 if there is none + * @param maxPayloadSize the maximum total bytes of {@link ResultChunk#getQueryResult()} + */ + public static ResultChunk convertQueryResultByFetchSize( + IQueryExecution queryExecution, int fetchSize, int requestOffset, int maxPayloadSize) + throws IoTDBException { + fetchSize = fetchSize > 0 ? fetchSize : TSFLE_CONFIG.getMaxTsBlockLineNumber(); + List<ByteBuffer> res = new ArrayList<>(); + int remainingPayloadSize = maxPayloadSize; + int rowCount = 0; + long memorySize = 0; + + // Continue the oversized TsBlock left by the previous response first + PendingResultFragment pendingFragment = queryExecution.getPendingResultFragment(); + ByteBuffer pendingBlock = pendingFragment.getBlock(); + if (pendingBlock != null) { + int nextOffset = pendingFragment.getNextOffset(); + if (requestOffset != nextOffset) { + throw new IllegalArgumentException( + String.format( + DataNodeQueryMessages + .EXCEPTION_RESULT_FRAGMENT_REQUEST_OFFSET_ARG_DOES_NOT_MATCH_PENDING_FRAGMENT_OFFSET_ARG_A1B13DBC, + requestOffset, + nextOffset)); + } + int fragmentSize = Math.min(remainingPayloadSize, pendingBlock.limit() - nextOffset); + res.add(slice(pendingBlock, nextOffset, fragmentSize)); + if (nextOffset + fragmentSize == pendingBlock.limit()) { + pendingFragment.clear(); + // The fragmented block is now complete. Do not fetch new blocks in the same response, + // otherwise the response would carry the tail of the big block plus up to fetchSize new + // rows, whose row count is not accounted for. One response either starts/continues a + // fragmented block or delivers new blocks, never both. + return ResultChunk.complete(res, !queryExecution.hasNextResult()); + } + pendingFragment.reset(pendingBlock, nextOffset + fragmentSize); + return ResultChunk.fragment(res, nextOffset + fragmentSize, -1, false); + } else if (requestOffset != 0) { + throw new IllegalArgumentException( + String.format( + DataNodeQueryMessages + .EXCEPTION_RESULT_FRAGMENT_REQUEST_OFFSET_ARG_IS_INVALID_WITHOUT_PENDING_FRAGMENT_C403454F, + requestOffset)); + } + + while (rowCount < fetchSize && memorySize < MAX_RETURN_SIZE && remainingPayloadSize > 0) { + Optional<ByteBuffer> optionalByteBuffer = queryExecution.getByteBufferBatchResult(); + if (!optionalByteBuffer.isPresent()) { + break; + } + ByteBuffer byteBuffer = optionalByteBuffer.get(); + int positionCount = readPositionCount(byteBuffer); + if (positionCount == 0) { + continue; + } + // offsets kept in PendingResultFragment are absolute positions of the buffer + int blockStart = byteBuffer.position(); + int blockSize = byteBuffer.remaining(); + rowCount += positionCount; + if (blockSize <= remainingPayloadSize) { + res.add(byteBuffer); + remainingPayloadSize -= blockSize; + memorySize += blockSize; + } else { + // The block exceeds the remaining payload, send a leading fragment and keep the rest + res.add(slice(byteBuffer, blockStart, remainingPayloadSize)); + pendingFragment.reset(byteBuffer, blockStart + remainingPayloadSize); + return ResultChunk.fragment( + res, blockStart + remainingPayloadSize, blockStart == 0 ? blockSize : -1, false); + } + } + return ResultChunk.complete(res, !queryExecution.hasNextResult()); + } + + private static int readPositionCount(ByteBuffer byteBuffer) { + byteBuffer.mark(); + int valueColumnCount = byteBuffer.getInt(); + for (int i = 0; i < valueColumnCount; i++) { + byteBuffer.get(); + } + int positionCount = byteBuffer.getInt(); + byteBuffer.reset(); + return positionCount; + } + + /** Returns a view of {@code [offset, offset + length)} bytes of the buffer without copying. */ + private static ByteBuffer slice(ByteBuffer buffer, int offset, int length) { + ByteBuffer duplicate = buffer.duplicate(); + duplicate.position(offset); + duplicate.limit(offset + length); + return duplicate.slice(); + } + + /** The bounded-payload result of one query result RPC response. */ + public static class ResultChunk { + + private final List<ByteBuffer> queryResult; + private final int nextOffset; + private final int totalLength; + private final boolean finished; + + private ResultChunk( + List<ByteBuffer> queryResult, int nextOffset, int totalLength, boolean finished) { + this.queryResult = queryResult; + this.nextOffset = nextOffset; + this.totalLength = totalLength; + this.finished = finished; + } + + /** The response ends with a fragment, which starts at {@code nextOffset} of its TsBlock. */ + public static ResultChunk fragment( + List<ByteBuffer> queryResult, int nextOffset, int totalLength, boolean finished) { + return new ResultChunk(queryResult, nextOffset, totalLength, finished); + } + + public static ResultChunk complete(List<ByteBuffer> queryResult, boolean finished) { + return new ResultChunk(queryResult, -1, -1, finished); + } + + public List<ByteBuffer> getQueryResult() { + return queryResult; + } + + public boolean hasNextOffset() { + return nextOffset >= 0; + } + + public int getNextOffset() { + return nextOffset; + } + + /** The total serialized length of the last block, set only when the fragment starts it. */ + public boolean hasTotalLength() { + return totalLength >= 0; + } + + public int getTotalLength() { + return totalLength; + } + + public boolean isFinished() { + return finished; + } + } + public static long[] readTimesFromBuffer(ByteBuffer buffer, int size) { long[] times = new long[size]; for (int i = 0; i < size; i++) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/MergeTreeSortOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/MergeTreeSortOperatorTest.java index cfbcf0e2630..ca22be2e6be 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/MergeTreeSortOperatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/MergeTreeSortOperatorTest.java @@ -50,6 +50,7 @@ import org.apache.iotdb.db.queryengine.plan.Coordinator; import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult; import org.apache.iotdb.db.queryengine.plan.execution.IQueryExecution; +import org.apache.iotdb.db.queryengine.plan.execution.PendingResultFragment; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; import org.apache.iotdb.db.queryengine.plan.statement.component.OrderByKey; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; @@ -1808,6 +1809,7 @@ public class MergeTreeSortOperatorTest { private final long startTime; private final String queryId; private final String sql; + private final PendingResultFragment pendingResultFragment = new PendingResultFragment(); FakeQueryExecution(long startTime, String queryId, String sql) { this.startTime = startTime; @@ -1893,6 +1895,11 @@ public class MergeTreeSortOperatorTest { return Optional.empty(); } + @Override + public PendingResultFragment getPendingResultFragment() { + return pendingResultFragment; + } + @Override public boolean hasNextResult() { return false; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/QueryDataSetUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/QueryDataSetUtilsTest.java index 7b65562ce61..3ea0cce94b6 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/QueryDataSetUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/QueryDataSetUtilsTest.java @@ -21,6 +21,7 @@ package org.apache.iotdb.db.utils; import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.db.queryengine.plan.execution.IQueryExecution; +import org.apache.iotdb.db.queryengine.plan.execution.PendingResultFragment; import org.apache.iotdb.service.rpc.thrift.TSQueryDataSet; import org.apache.tsfile.common.conf.TSFileConfig; @@ -281,4 +282,157 @@ public class QueryDataSetUtilsTest { new Binary(BINARY_STR, TSFileConfig.STRING_CHARSET), tsBlock.getColumn(5).getBinary(0)); assertTrue(tsBlock.getColumn(5).isNull(1)); } + + @Test + public void testConvertQueryResultByFetchSizeWithSingleOversizedBlock() + throws IoTDBException, IOException { + TsBlock tsBlock = buildTsBlock(); + ByteBuffer serialized = new TsBlockSerde().serialize(tsBlock); + int totalLength = serialized.remaining(); + // one byte smaller than the block so that the block has to be split into two fragments + int payloadSize = totalLength - 1; + + PendingResultFragment pendingFragment = new PendingResultFragment(); + IQueryExecution queryExecution = Mockito.mock(IQueryExecution.class); + Mockito.when(queryExecution.getPendingResultFragment()).thenReturn(pendingFragment); + Mockito.when(queryExecution.getByteBufferBatchResult()) + .thenReturn(Optional.of(serialized.duplicate()), Optional.empty()); + Mockito.when(queryExecution.hasNextResult()).thenReturn(false); + + QueryDataSetUtils.ResultChunk firstChunk = + QueryDataSetUtils.convertQueryResultByFetchSize(queryExecution, 10, 0, payloadSize); + assertFalse(firstChunk.isFinished()); + assertTrue(firstChunk.hasNextOffset()); + assertTrue(firstChunk.hasTotalLength()); + assertEquals(totalLength, firstChunk.getTotalLength()); + assertEquals(payloadSize, firstChunk.getNextOffset()); + assertEquals(1, firstChunk.getQueryResult().size()); + assertEquals(payloadSize, firstChunk.getQueryResult().get(0).remaining()); + assertTrue(pendingFragment.getBlock() != null); + + QueryDataSetUtils.ResultChunk secondChunk = + QueryDataSetUtils.convertQueryResultByFetchSize( + queryExecution, 10, firstChunk.getNextOffset(), payloadSize); + assertTrue(secondChunk.isFinished()); + assertFalse(secondChunk.hasNextOffset()); + assertFalse(secondChunk.hasTotalLength()); + assertTrue(pendingFragment.getBlock() == null); + + TsBlock result = deserialize(firstChunk.getQueryResult().get(0), secondChunk.getQueryResult()); + assertEquals(2, result.getPositionCount()); + assertEquals(1L, result.getTimeColumn().getLong(0)); + assertEquals(2L, result.getTimeColumn().getLong(1)); + } + + @Test + public void testConvertQueryResultByFetchSizeWithTrailingFragment() + throws IoTDBException, IOException { + TsBlock tsBlock = buildTsBlock(); + ByteBuffer serialized = new TsBlockSerde().serialize(tsBlock); + int blockSize = serialized.remaining(); + // a small block that fits into the payload, followed by the oversized block + ByteBuffer smallBlock = serialized.duplicate(); + int payloadSize = smallBlock.remaining() + blockSize / 2; + + PendingResultFragment pendingFragment = new PendingResultFragment(); + IQueryExecution queryExecution = Mockito.mock(IQueryExecution.class); + Mockito.when(queryExecution.getPendingResultFragment()).thenReturn(pendingFragment); + Mockito.when(queryExecution.getByteBufferBatchResult()) + .thenReturn(Optional.of(smallBlock), Optional.of(serialized.duplicate()), Optional.empty()); + Mockito.when(queryExecution.hasNextResult()).thenReturn(false); + + QueryDataSetUtils.ResultChunk firstChunk = + QueryDataSetUtils.convertQueryResultByFetchSize(queryExecution, 10, 0, payloadSize); + assertFalse(firstChunk.isFinished()); + assertTrue(firstChunk.hasNextOffset()); + assertEquals(2, firstChunk.getQueryResult().size()); + // the last element is the fragment of the second block + assertEquals(blockSize / 2, firstChunk.getQueryResult().get(1).remaining()); + assertTrue(firstChunk.hasTotalLength()); + assertEquals(blockSize, firstChunk.getTotalLength()); + + QueryDataSetUtils.ResultChunk secondChunk = + QueryDataSetUtils.convertQueryResultByFetchSize( + queryExecution, 10, firstChunk.getNextOffset(), payloadSize); + assertTrue(secondChunk.isFinished()); + assertFalse(secondChunk.hasNextOffset()); + // only the rest of the oversized block remains + assertEquals(1, secondChunk.getQueryResult().size()); + assertEquals(blockSize - blockSize / 2, secondChunk.getQueryResult().get(0).remaining()); + } + + @Test + public void testContinuationCompletionDoesNotFetchNewBlocks() throws IoTDBException, IOException { + TsBlock tsBlock = buildTsBlock(); + ByteBuffer serialized = new TsBlockSerde().serialize(tsBlock); + int blockSize = serialized.remaining(); + int payloadSize = blockSize / 2; + + PendingResultFragment pendingFragment = new PendingResultFragment(); + IQueryExecution queryExecution = Mockito.mock(IQueryExecution.class); + Mockito.when(queryExecution.getPendingResultFragment()).thenReturn(pendingFragment); + Mockito.when(queryExecution.getByteBufferBatchResult()) + .thenReturn( + Optional.of(serialized.duplicate()), + Optional.of(serialized.duplicate()), + Optional.empty()); + Mockito.when(queryExecution.hasNextResult()).thenReturn(true, false); + + QueryDataSetUtils.ResultChunk firstChunk = + QueryDataSetUtils.convertQueryResultByFetchSize(queryExecution, 10, 0, payloadSize); + assertFalse(firstChunk.isFinished()); + assertEquals(payloadSize, firstChunk.getNextOffset()); + + // The response that completes the fragmented block must not fetch new blocks, otherwise its + // row count would exceed fetchSize (the tail of the big block is not accounted). + QueryDataSetUtils.ResultChunk continuationChunk = + QueryDataSetUtils.convertQueryResultByFetchSize( + queryExecution, 10, firstChunk.getNextOffset(), blockSize); + assertFalse(continuationChunk.isFinished()); + assertFalse(continuationChunk.hasNextOffset()); + assertEquals(1, continuationChunk.getQueryResult().size()); + assertEquals(blockSize - payloadSize, continuationChunk.getQueryResult().get(0).remaining()); + assertTrue(pendingFragment.getBlock() == null); + + // New blocks are delivered by the following request only + QueryDataSetUtils.ResultChunk nextChunk = + QueryDataSetUtils.convertQueryResultByFetchSize(queryExecution, 10, 0, blockSize); + assertTrue(nextChunk.isFinished()); + assertFalse(nextChunk.hasNextOffset()); + assertEquals(1, nextChunk.getQueryResult().size()); + assertEquals(blockSize, nextChunk.getQueryResult().get(0).remaining()); + } + + @Test(expected = IllegalArgumentException.class) + public void testConvertQueryResultByFetchSizeRejectsMismatchedOffset() throws IoTDBException { + PendingResultFragment pendingFragment = new PendingResultFragment(); + pendingFragment.reset(ByteBuffer.wrap(new byte[10]), 5); + IQueryExecution queryExecution = Mockito.mock(IQueryExecution.class); + Mockito.when(queryExecution.getPendingResultFragment()).thenReturn(pendingFragment); + + QueryDataSetUtils.convertQueryResultByFetchSize(queryExecution, 10, 3, 8); + } + + @Test(expected = IllegalArgumentException.class) + public void testConvertQueryResultByFetchSizeRejectsOffsetWithoutPendingFragment() + throws IoTDBException { + IQueryExecution queryExecution = Mockito.mock(IQueryExecution.class); + Mockito.when(queryExecution.getPendingResultFragment()).thenReturn(new PendingResultFragment()); + + QueryDataSetUtils.convertQueryResultByFetchSize(queryExecution, 10, 5, 8); + } + + private TsBlock deserialize(ByteBuffer head, List<ByteBuffer> tail) { + int size = head.remaining(); + for (ByteBuffer buffer : tail) { + size += buffer.remaining(); + } + ByteBuffer assembled = ByteBuffer.allocate(size); + assembled.put(head.duplicate()); + for (ByteBuffer buffer : tail) { + assembled.put(buffer.duplicate()); + } + assembled.flip(); + return new TsBlockSerde().deserialize(assembled); + } } diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index a3fe0970318..49a23190cf8 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1241,6 +1241,9 @@ mpp_data_exchange_max_pool_size=10 mpp_data_exchange_keep_alive_time_in_ms=1000 # The maximum payload size of one MPP data exchange RPC response. +# It also bounds the payload of one query-result RPC response between client and DataNode: a +# serialized TsBlock larger than this value is transferred in fragments, so that one oversized +# row (e.g. an OBJECT/BLOB column) cannot inflate the RPC buffer of server and client connections. # <=0 use default value # The effective value is between 131072(128KB) and dn_thrift_max_frame_size minus 1024 bytes reserved for the RPC response envelope. # effectiveMode: hot_reload diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift index 7ff12ceb021..46d2e54554e 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift @@ -74,6 +74,11 @@ struct TSExecuteStatementResp { 15: optional string database 16: optional bool tableModel 17: optional list<i32> columnIndex2TsBlockColumnIndexList + // Start offset of the next fragment. Set only when the last element in queryResult is a + // fragment. Only set for sessions that enabled chunked query results at openSession. + 18: optional i32 offset + // Total serialized length of the TsBlock when the response starts its first fragment. + 19: optional i32 totalLength } enum TSProtocolVersion { @@ -203,6 +208,9 @@ struct TSFetchResultsReq{ 5: required bool isAlign 6: optional i64 timeout 7: optional i64 statementId + // Start offset of the fragment to continue. The client sets this field (including 0 for the + // first request) iff it supports chunked query results. It is optional for rolling upgrade. + 8: optional i32 offset } struct TSFetchResultsResp{ @@ -213,6 +221,10 @@ struct TSFetchResultsResp{ 5: optional TSQueryNonAlignDataSet nonAlignQueryDataSet 6: optional list<binary> queryResult 7: optional bool moreData + // Start offset of the next fragment. Set only when the last element in queryResult is a fragment. + 8: optional i32 offset + // Total serialized length of the TsBlock when the response starts its first fragment. + 9: optional i32 totalLength } struct TSFetchMetadataResp{
