This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 16ab9c1945 Remove the old deprecated commit end without metadata
(#11662)
16ab9c1945 is described below
commit 16ab9c1945f7ac868f88c97070c4fb4b909f6ba3
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Sun Sep 24 17:58:39 2023 -0700
Remove the old deprecated commit end without metadata (#11662)
---
.../resources/LLCSegmentCompletionHandlers.java | 71 ----------------------
.../ServerSegmentCompletionProtocolHandler.java | 12 ----
2 files changed, 83 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/LLCSegmentCompletionHandlers.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/LLCSegmentCompletionHandlers.java
index 93fd7b1896..e774f91436 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/LLCSegmentCompletionHandlers.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/LLCSegmentCompletionHandlers.java
@@ -42,7 +42,6 @@ import
org.apache.pinot.common.protocols.SegmentCompletionProtocol;
import org.apache.pinot.common.utils.LLCSegmentName;
import org.apache.pinot.common.utils.TarGzCompressionUtils;
import org.apache.pinot.common.utils.URIUtils;
-import org.apache.pinot.common.utils.fetcher.SegmentFetcherFactory;
import org.apache.pinot.controller.api.access.AccessType;
import org.apache.pinot.controller.api.access.Authenticate;
import
org.apache.pinot.controller.helix.core.realtime.SegmentCompletionManager;
@@ -214,57 +213,6 @@ public class LLCSegmentCompletionHandlers {
return responseStr;
}
- // TODO: remove this API. Should not allow committing without metadata
- @GET
- @Path(SegmentCompletionProtocol.MSG_TYPE_COMMIT_END)
- @Authorize(targetType = TargetType.CLUSTER, action =
Actions.Cluster.GET_ADMIN_INFO)
- @Produces(MediaType.APPLICATION_JSON)
- public String
segmentCommitEnd(@QueryParam(SegmentCompletionProtocol.PARAM_INSTANCE_ID)
String instanceId,
- @QueryParam(SegmentCompletionProtocol.PARAM_SEGMENT_NAME) String
segmentName,
- @QueryParam(SegmentCompletionProtocol.PARAM_SEGMENT_LOCATION) String
segmentLocation,
- @QueryParam(SegmentCompletionProtocol.PARAM_OFFSET) long offset,
- @QueryParam(SegmentCompletionProtocol.PARAM_STREAM_PARTITION_MSG_OFFSET)
String streamPartitionMsgOffset,
- @QueryParam(SegmentCompletionProtocol.PARAM_MEMORY_USED_BYTES) long
memoryUsedBytes,
- @QueryParam(SegmentCompletionProtocol.PARAM_BUILD_TIME_MILLIS) long
buildTimeMillis,
- @QueryParam(SegmentCompletionProtocol.PARAM_WAIT_TIME_MILLIS) long
waitTimeMillis,
- @QueryParam(SegmentCompletionProtocol.PARAM_ROW_COUNT) int numRows,
- @QueryParam(SegmentCompletionProtocol.PARAM_SEGMENT_SIZE_BYTES) long
segmentSizeBytes) {
- if (instanceId == null || segmentName == null || segmentLocation == null
|| (offset == -1
- && streamPartitionMsgOffset == null)) {
- LOGGER.error(
- "Invalid call: offset={}, segmentName={}, instanceId={},
segmentLocation={}, streamPartitionMsgOffset={}",
- offset, segmentName, instanceId, segmentLocation,
streamPartitionMsgOffset);
- // TODO: memoryUsedInBytes = 0 if not present in params. Add validation
when we start using it
- return SegmentCompletionProtocol.RESP_FAILED.toJsonString();
- }
-
- SegmentMetadataImpl segmentMetadata;
- try {
- segmentMetadata = extractMetadataFromSegmentFileURI(new
URI(segmentLocation), segmentName);
- } catch (Exception e) {
- LOGGER.error("Caught exception while extracting metadata from segment:
{} at location: {} from instance: {}",
- segmentName, segmentLocation, instanceId, e);
- return SegmentCompletionProtocol.RESP_FAILED.toJsonString();
- }
- SegmentCompletionProtocol.Request.Params requestParams = new
SegmentCompletionProtocol.Request.Params();
-
requestParams.withInstanceId(instanceId).withSegmentName(segmentName).withSegmentLocation(segmentLocation)
-
.withSegmentSizeBytes(segmentSizeBytes).withBuildTimeMillis(buildTimeMillis).withWaitTimeMillis(waitTimeMillis)
- .withNumRows(numRows).withMemoryUsedBytes(memoryUsedBytes);
- extractOffsetFromParams(requestParams, streamPartitionMsgOffset, offset);
- LOGGER.info("Processing segmentCommitEnd:{}", requestParams.toString());
-
- final boolean isSuccess = true;
- final boolean isSplitCommit = true;
-
- CommittingSegmentDescriptor committingSegmentDescriptor =
-
CommittingSegmentDescriptor.fromSegmentCompletionReqParamsAndMetadata(requestParams,
segmentMetadata);
- SegmentCompletionProtocol.Response response = _segmentCompletionManager
- .segmentCommitEnd(requestParams, isSuccess, isSplitCommit,
committingSegmentDescriptor);
- final String responseStr = response.toJsonString();
- LOGGER.info("Response to segmentCommitEnd for segment:{} is:{}",
segmentName, responseStr);
- return responseStr;
- }
-
@POST
@Path(SegmentCompletionProtocol.MSG_TYPE_COMMIT)
@Authorize(targetType = TargetType.CLUSTER, action =
Actions.Cluster.COMMIT_SEGMENT)
@@ -542,25 +490,6 @@ public class LLCSegmentCompletionHandlers {
}
}
- /**
- * Extracts the segment metadata from the segment file URI (could be
remote). Copy the segment file to local under
- * file upload temporary directory first, then use the untarred file
temporary directory to store the metadata files
- * temporarily.
- */
- private static SegmentMetadataImpl extractMetadataFromSegmentFileURI(URI
segmentFileURI, String segmentName)
- throws Exception {
- File localTempFile =
org.apache.pinot.common.utils.FileUtils.concatAndValidateFile(
- ControllerFilePathProvider.getInstance().getFileUploadTempDir(),
getTempSegmentFileName(segmentName),
- "Invalid segment name: %s", segmentName);
-
- try {
- SegmentFetcherFactory.fetchSegmentToLocal(segmentFileURI, localTempFile);
- return extractMetadataFromLocalSegmentFile(localTempFile);
- } finally {
- FileUtils.deleteQuietly(localTempFile);
- }
- }
-
private static String getTempSegmentFileName(String segmentName) {
return segmentName + "." + UUID.randomUUID();
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/server/realtime/ServerSegmentCompletionProtocolHandler.java
b/pinot-core/src/main/java/org/apache/pinot/server/realtime/ServerSegmentCompletionProtocolHandler.java
index 03227f2920..6e1e24a688 100644
---
a/pinot-core/src/main/java/org/apache/pinot/server/realtime/ServerSegmentCompletionProtocolHandler.java
+++
b/pinot-core/src/main/java/org/apache/pinot/server/realtime/ServerSegmentCompletionProtocolHandler.java
@@ -128,18 +128,6 @@ public class ServerSegmentCompletionProtocolHandler {
return request.getUrl(hostPort, protocol);
}
- // Replaced by segmentCommitEndWithMetadata().
- @Deprecated
- public SegmentCompletionProtocol.Response
segmentCommitEnd(SegmentCompletionProtocol.Request.Params params) {
- SegmentCompletionProtocol.SegmentCommitEndRequest request =
- new SegmentCompletionProtocol.SegmentCommitEndRequest(params);
- String url = createSegmentCompletionUrl(request);
- if (url == null) {
- return SegmentCompletionProtocol.RESP_NOT_SENT;
- }
- return sendRequest(url);
- }
-
public SegmentCompletionProtocol.Response segmentCommitEndWithMetadata(
SegmentCompletionProtocol.Request.Params params, final Map<String, File>
metadataFiles) {
SegmentCompletionProtocol.SegmentCommitEndWithMetadataRequest request =
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
