This is an automated email from the ASF dual-hosted git repository.
xbli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new d5c4730ff7 add offloadSegmentUnafe to allow separate threads to manage
segment offloading (#13517)
d5c4730ff7 is described below
commit d5c4730ff76acba480af3edcb2c2bcdea006c490
Author: Xiaobing <[email protected]>
AuthorDate: Mon Jul 1 15:08:07 2024 -0700
add offloadSegmentUnafe to allow separate threads to manage segment
offloading (#13517)
* add offloadSegmentUnafe to allow separate threads to manage segment
offloading
* refine log msg a bit
---
.../apache/pinot/controller/BaseControllerStarter.java | 1 +
.../pinot/core/data/manager/BaseTableDataManager.java | 16 ++++++++++++++++
.../segment/local/data/manager/TableDataManager.java | 7 +++++++
.../local/upsert/BasePartitionUpsertMetadataManager.java | 4 ++--
4 files changed, 26 insertions(+), 2 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java
index d8e4547a77..101ec08e7b 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java
@@ -309,6 +309,7 @@ public abstract class BaseControllerStarter implements
ServiceStartable {
protected PinotHelixResourceManager createHelixResourceManager() {
return new PinotHelixResourceManager(_config);
}
+
public PinotHelixResourceManager getHelixResourceManager() {
return _helixResourceManager;
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
index bced2704db..97b7530b04 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
@@ -446,6 +446,22 @@ public abstract class BaseTableDataManager implements
TableDataManager {
}
}
+ @Override
+ public void offloadSegmentUnsafe(String segmentName) {
+ if (_shutDown) {
+ _logger.info("Table data manager is already shut down, skipping
offloading segment: {} unsafe", segmentName);
+ return;
+ }
+ _logger.info("Offloading segment: {} unsafe", segmentName);
+ try {
+ doOffloadSegment(segmentName);
+ } catch (Exception e) {
+ addSegmentError(segmentName,
+ new SegmentErrorInfo(System.currentTimeMillis(), "Caught exception
while offloading segment unsafe", e));
+ throw e;
+ }
+ }
+
protected void doOffloadSegment(String segmentName) {
SegmentDataManager segmentDataManager = unregisterSegment(segmentName);
if (segmentDataManager != null) {
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/data/manager/TableDataManager.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/data/manager/TableDataManager.java
index d31dfdd2d5..677d659fff 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/data/manager/TableDataManager.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/data/manager/TableDataManager.java
@@ -151,6 +151,13 @@ public interface TableDataManager {
void offloadSegment(String segmentName)
throws Exception;
+ /**
+ * Offloads a segment from table like the method {@link
#offloadSegment(String)}, but this method doesn't take
+ * segment lock internally to allow separate threads to manage segment
offloading w/o the risk of deadlocks.
+ */
+ void offloadSegmentUnsafe(String segmentName)
+ throws Exception;
+
/**
* Reloads an existing immutable segment for the table, which can be an
OFFLINE or REALTIME table.
* A new segment may be downloaded if the local one has a different CRC; or
can be forced to download
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/BasePartitionUpsertMetadataManager.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/BasePartitionUpsertMetadataManager.java
index b0703d2501..d483a65c22 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/BasePartitionUpsertMetadataManager.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/BasePartitionUpsertMetadataManager.java
@@ -309,8 +309,8 @@ public abstract class BasePartitionUpsertMetadataManager
implements PartitionUps
}
}
}
- _logger.info("Preloaded segments from partition: {} of table: {} for fast
upsert metadata recovery", _partitionId,
- _tableNameWithType);
+ _logger.info("Preloaded {} segments from partition: {} of table: {} for
fast upsert metadata recovery",
+ futures.size(), _partitionId, _tableNameWithType);
}
private String getInstanceId(TableDataManager tableDataManager) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
