TanYuxin-tyx commented on code in PR #22652:
URL: https://github.com/apache/flink/pull/22652#discussion_r1211224068
##########
flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/storage/TieredStorageProducerClient.java:
##########
@@ -119,7 +145,66 @@ private void writeAccumulatedBuffers(
private void writeAccumulatedBuffer(
TieredStorageSubpartitionId subpartitionId, Buffer
accumulatedBuffer)
throws IOException {
- // TODO, Try to write the accumulated buffer to the appropriate tier.
After the tier is
- // decided, then write the accumulated buffer to the tier.
+ int subpartitionIndex = subpartitionId.getSubpartitionId();
+ int tierIndex = chooseStorageTierIfNeeded(subpartitionId);
+
+ Buffer compressedBuffer = compressBufferIfPossible(accumulatedBuffer);
+ updateStatistics(compressedBuffer);
+ boolean isLastBufferInSegment =
+ tierProducerAgents.get(tierIndex).write(subpartitionId,
compressedBuffer);
+ if (isLastBufferInSegment) {
+ isSubpartitionSegmentFinished[subpartitionIndex] = true;
+ }
+ }
+
+ private int chooseStorageTierIfNeeded(TieredStorageSubpartitionId
subpartitionId)
+ throws IOException {
+ int subpartitionIndex = subpartitionId.getSubpartitionId();
+ if (isSubpartitionSegmentFinished[subpartitionIndex]) {
+ // If the current segment is finished, this subpartition need
choose a new storage tier.
+ internalChooseStorageTier(subpartitionId);
+ isSubpartitionSegmentFinished[subpartitionIndex] = false;
+ }
+ return currentSubpartitionTierIndex[subpartitionIndex];
+ }
+
+ private void internalChooseStorageTier(TieredStorageSubpartitionId
subpartitionId)
+ throws IOException {
+ int subpartitionIndex = subpartitionId.getSubpartitionId();
+ int segmentIndex = currentSubpartitionSegmentId[subpartitionIndex];
+ int nextSegmentIndex = segmentIndex + 1;
+
+ for (int tierIndex = 0; tierIndex < tierProducerAgents.size();
++tierIndex) {
Review Comment:
Fixed. Added a doc for the tierProducerAgents to show that they are sorted
by priority.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
