jt2594838 commented on code in PR #18901:
URL: https://github.com/apache/iotdb/pull/18901#discussion_r4332678700


##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java:
##########
@@ -362,6 +392,82 @@ protected void encodeAndBuildChunks() throws IOException, 
PageException {
     }
   }
 
+  /**
+   * Reassembles a chunk from pages that were stored without decoding. Returns 
{@code null} when the
+   * chunk has a decoded page, an empty page whose statistics cannot be 
merged, or no page at all;
+   * the caller then falls back to {@link AlignedChunkWriterImpl}.
+   */
+  protected Chunk buildChunkFromUndecodedPages(
+      final ChunkHeader sourceHeader, final int mask, final List<PageBuffer> 
chunkPages)
+      throws IOException {
+    // A time chunk header carries VECTOR, and its pages use TimeStatistics 
rather than the
+    // LongStatistics that Statistics.getStatsByType(INT64) would create.
+    final TSDataType statisticsType = sourceHeader.getDataType();
+    Statistics<?> statistics = null;
+    int totalSize = 0;
+    int pageCount = 0;
+    for (final PageBuffer page : chunkPages) {
+      if (page.needDecode || page.pageHeader == null || 
page.pageHeader.getStatistics() == null) {
+        return null;
+      }

Review Comment:
   So the only-one-page chunk must be decoded? Why?



##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/ChunkPayloadRef.java:
##########
@@ -112,32 +112,59 @@ public static ChunkPayloadRef deserializeFrom(final 
InputStream stream) throws I
    *     outside of this DataNode's LOAD directories
    */
   public byte[] readPayload() {
-    final File file = locateFile();
-    // Checked against the file before the buffer is allocated: without it a 
reference to bytes past
-    // the end of the staged file would still allocate its full size and fail 
later, and a huge size
-    // would be allocated only to be thrown away.
-    if (offset + size > file.length()) {
+    final File file = locatePayloadFile();
+    try (final RandomAccessFile input = new RandomAccessFile(file, "r")) {
+      final byte[] payload = new byte[Math.toIntExact(size)];
+      input.seek(offset);
+      input.readFully(payload);
+      return payload;
+    } catch (final EOFException e) {
+      throw new ChunkPayloadUnavailableException(
+          String.format(
+              
StorageEngineMessages.EXCEPTION_LOAD_CONSENSUS_STAGED_FILE_EOF_8743387D,
+              file.getAbsolutePath(),
+              offset),
+          e);
+    } catch (final IOException | ArithmeticException e) {
       throw new ChunkPayloadUnavailableException(
           String.format(
               StorageEngineMessages
-                  
.EXCEPTION_THE_CHUNK_PAYLOAD_OF_ARG_BYTES_AT_OFFSET_ARG_IS_LONGER_THAN_THE_STAGED_FILE_ARG_14081256,
-              size,
-              offset,
-              file.getAbsolutePath()));
+                  
.EXCEPTION_FAILED_TO_READ_BACK_THE_STAGED_PIECE_FILE_ARG_ARG_3F54CB90,
+              file.getAbsolutePath(),
+              e.getMessage()),
+          e);
     }
+  }
+
+  /**
+   * Streams the referenced bytes into {@code stream} without materializing 
them in a byte array.
+   * Used when a payload has to be inlined into a request but is still stored 
in the staged file.
+   */
+  public void writePayloadTo(final DataOutputStream stream) throws IOException 
{
+    final File file = locatePayloadFile();
+    if (size == 0) {
+      return;
+    }

Review Comment:
   Check the size before locating.



##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/ChunkPayloadRef.java:
##########
@@ -112,32 +112,59 @@ public static ChunkPayloadRef deserializeFrom(final 
InputStream stream) throws I
    *     outside of this DataNode's LOAD directories
    */
   public byte[] readPayload() {
-    final File file = locateFile();
-    // Checked against the file before the buffer is allocated: without it a 
reference to bytes past
-    // the end of the staged file would still allocate its full size and fail 
later, and a huge size
-    // would be allocated only to be thrown away.
-    if (offset + size > file.length()) {
+    final File file = locatePayloadFile();
+    try (final RandomAccessFile input = new RandomAccessFile(file, "r")) {
+      final byte[] payload = new byte[Math.toIntExact(size)];
+      input.seek(offset);
+      input.readFully(payload);
+      return payload;
+    } catch (final EOFException e) {
+      throw new ChunkPayloadUnavailableException(
+          String.format(
+              
StorageEngineMessages.EXCEPTION_LOAD_CONSENSUS_STAGED_FILE_EOF_8743387D,
+              file.getAbsolutePath(),
+              offset),
+          e);
+    } catch (final IOException | ArithmeticException e) {
       throw new ChunkPayloadUnavailableException(
           String.format(
               StorageEngineMessages
-                  
.EXCEPTION_THE_CHUNK_PAYLOAD_OF_ARG_BYTES_AT_OFFSET_ARG_IS_LONGER_THAN_THE_STAGED_FILE_ARG_14081256,
-              size,
-              offset,
-              file.getAbsolutePath()));
+                  
.EXCEPTION_FAILED_TO_READ_BACK_THE_STAGED_PIECE_FILE_ARG_ARG_3F54CB90,
+              file.getAbsolutePath(),
+              e.getMessage()),
+          e);
     }
+  }
+
+  /**
+   * Streams the referenced bytes into {@code stream} without materializing 
them in a byte array.
+   * Used when a payload has to be inlined into a request but is still stored 
in the staged file.
+   */
+  public void writePayloadTo(final DataOutputStream stream) throws IOException 
{
+    final File file = locatePayloadFile();
+    if (size == 0) {
+      return;
+    }
+    final byte[] buffer = new byte[(int) Math.min(size, 8192L)];
     try (final RandomAccessFile input = new RandomAccessFile(file, "r")) {
-      final byte[] payload = new byte[(int) size];
       input.seek(offset);
-      input.readFully(payload);
-      return payload;
+      long remaining = size;
+      while (remaining > 0) {
+        final int read = input.read(buffer, 0, (int) Math.min(buffer.length, 
remaining));
+        if (read < 0) {
+          throw new EOFException();
+        }

Review Comment:
   Provide the path and the current position, etc.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to