jt2594838 commented on code in PR #18627:
URL: https://github.com/apache/iotdb/pull/18627#discussion_r4034631632


##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java:
##########
@@ -629,10 +629,30 @@ public TSchemaFetchResponse fetchSchema(final 
TSchemaFetchRequest req) {
 
   @Override
   public TLoadResp sendTsFilePieceNode(final TTsFilePieceReq req) {
-    LOGGER.info(DataNodeMiscMessages.RECEIVE_LOAD_NODE, req.uuid);
+    if (!req.isSetSliceIndex() || req.sliceIndex == 0) {
+      LOGGER.info(DataNodeMiscMessages.RECEIVE_LOAD_NODE, req.uuid);
+    }
 
     final ConsensusGroupId groupId =
         
ConsensusGroupId.Factory.createFromTConsensusGroupId(req.consensusGroupId);
+    final boolean isSliced =
+        req.isSetSliceIndex() || req.isSetSliceCount() || 
req.isSetOriginBodySize();
+    if (isSliced) {
+      if (!req.isSetSliceIndex() || !req.isSetSliceCount() || 
!req.isSetOriginBodySize()) {
+        return createTLoadResp(
+            new 
TSStatus(TSStatusCode.DESERIALIZE_PIECE_OF_TSFILE_ERROR.getStatusCode()));
+      }

Review Comment:
   Indicate which one is missing to help debugging



##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileDispatcherImpl.java:
##########
@@ -143,26 +146,83 @@ public Future<FragInstanceDispatchResult> dispatch(
 
   private void dispatchOneInstance(FragmentInstance instance)
       throws FragmentInstanceDispatchException {
-    TTsFilePieceReq loadTsFileReq = null;
+    List<TTsFilePieceReq> loadTsFileReqs = null;
 
     for (TDataNodeLocation dataNodeLocation :
         instance.getRegionReplicaSet().getDataNodeLocations()) {
       TEndPoint endPoint = dataNodeLocation.getInternalEndPoint();
       if (isDispatchedToLocal(endPoint)) {
         dispatchLocally(instance);
       } else {
-        if (loadTsFileReq == null) {
-          loadTsFileReq =
-              new TTsFilePieceReq(
+        if (loadTsFileReqs == null) {
+          loadTsFileReqs =
+              splitTsFilePieceReq(
                   
instance.getFragment().getPlanNodeTree().serializeToByteBuffer(),
                   uuid,
-                  instance.getRegionReplicaSet().getRegionId());
+                  instance.getRegionReplicaSet().getRegionId(),
+                  getLoadTsFilePieceBodySizeLimit());
         }
-        dispatchRemote(loadTsFileReq, endPoint);
+        dispatchRemote(loadTsFileReqs, endPoint);
       }
     }
   }
 
+  private static int getLoadTsFilePieceBodySizeLimit() {
+    final int thriftMaxFrameSize =
+        IoTDBDescriptor.getInstance().getConfig().getThriftMaxFrameSize();
+    return Math.max(1, thriftMaxFrameSize - 
LOAD_TSFILE_PIECE_RPC_FRAME_RESERVED_BYTES);
+  }

Review Comment:
   The receiver and the sender may have different maxSize.
   May parse it from the exception.



##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFilePieceNodeAssembler.java:
##########
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.load;
+
+import org.apache.tsfile.utils.PublicBAOS;
+
+import java.nio.ByteBuffer;
+
+public final class LoadTsFilePieceNodeAssembler {
+
+  private final int sliceCount;
+  private final int originBodySize;
+  private final PublicBAOS assembledBody = new PublicBAOS();
+
+  private int nextSliceIndex;
+
+  LoadTsFilePieceNodeAssembler(final int sliceCount, final int originBodySize) 
{
+    this.sliceCount = sliceCount;
+    this.originBodySize = originBodySize;
+  }
+
+  synchronized Result append(
+      final ByteBuffer sliceBody,
+      final int sliceIndex,
+      final int requestSliceCount,
+      final int requestOriginBodySize) {
+    if (sliceBody == null
+        || !sliceBody.hasRemaining()
+        || sliceCount <= 1
+        || originBodySize <= 0
+        || sliceCount != requestSliceCount
+        || originBodySize != requestOriginBodySize
+        || sliceIndex != nextSliceIndex
+        || sliceIndex < 0
+        || sliceIndex >= sliceCount
+        || assembledBody.size() > originBodySize - sliceBody.remaining()) {
+      return Result.invalid();
+    }
+
+    final ByteBuffer duplicatedBody = sliceBody.duplicate();
+    if (duplicatedBody.hasArray()) {
+      assembledBody.write(
+          duplicatedBody.array(),
+          duplicatedBody.arrayOffset() + duplicatedBody.position(),
+          duplicatedBody.remaining());
+    } else {
+      final byte[] bytes = new byte[Math.min(duplicatedBody.remaining(), 
8192)];
+      while (duplicatedBody.hasRemaining()) {
+        final int size = Math.min(duplicatedBody.remaining(), bytes.length);
+        duplicatedBody.get(bytes, 0, size);
+        assembledBody.write(bytes, 0, size);
+      }

Review Comment:
   Check which is faster:
   copy and write
   write byte-by-byte



##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFilePieceNodeAssembler.java:
##########
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.load;
+
+import org.apache.tsfile.utils.PublicBAOS;
+
+import java.nio.ByteBuffer;
+
+public final class LoadTsFilePieceNodeAssembler {
+
+  private final int sliceCount;
+  private final int originBodySize;
+  private final PublicBAOS assembledBody = new PublicBAOS();
+
+  private int nextSliceIndex;
+
+  LoadTsFilePieceNodeAssembler(final int sliceCount, final int originBodySize) 
{
+    this.sliceCount = sliceCount;
+    this.originBodySize = originBodySize;
+  }

Review Comment:
   Better to use originBodySize to init assembledBody, avoiding growing the 
buffer.



##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFilePieceNodeAssembler.java:
##########
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.load;
+
+import org.apache.tsfile.utils.PublicBAOS;
+
+import java.nio.ByteBuffer;
+
+public final class LoadTsFilePieceNodeAssembler {
+
+  private final int sliceCount;
+  private final int originBodySize;
+  private final PublicBAOS assembledBody = new PublicBAOS();
+
+  private int nextSliceIndex;
+
+  LoadTsFilePieceNodeAssembler(final int sliceCount, final int originBodySize) 
{
+    this.sliceCount = sliceCount;
+    this.originBodySize = originBodySize;
+  }
+
+  synchronized Result append(
+      final ByteBuffer sliceBody,
+      final int sliceIndex,
+      final int requestSliceCount,
+      final int requestOriginBodySize) {
+    if (sliceBody == null
+        || !sliceBody.hasRemaining()
+        || sliceCount <= 1
+        || originBodySize <= 0
+        || sliceCount != requestSliceCount
+        || originBodySize != requestOriginBodySize
+        || sliceIndex != nextSliceIndex
+        || sliceIndex < 0
+        || sliceIndex >= sliceCount
+        || assembledBody.size() > originBodySize - sliceBody.remaining()) {
+      return Result.invalid();
+    }

Review Comment:
   Better to give specific reason or leave a warn log here.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to