This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 5dae99c312b Remove APIs deprecated in 1.1.0 (#19141)
5dae99c312b is described below
commit 5dae99c312b0698f0fa8cd62dc9e30f03983b181
Author: Xiang Fu <[email protected]>
AuthorDate: Sun Aug 2 14:30:22 2026 -0700
Remove APIs deprecated in 1.1.0 (#19141)
- Remove deprecated SegmentName class (delegated to LLCSegmentName) and
LLCSegmentName.isLowLevelConsumerSegmentName
- Remove deprecated no-op TableConfigBuilder.setLLC and its test usage
- Remove deprecated JsonIgnore getters getReplicationNumber and
getReplicasPerPartitionNumber from SegmentsValidationAndRetentionConfig
- Remove deprecated ZNRecord-based ZKMetadataProvider.setTableConfig,
setRealtimeTableConfig and setOfflineTableConfig
- Remove deprecated PinotHelixResourceManager.getServersForSegment
- Remove deprecated MinionContext.getMinionMetrics/setMinionMetrics
(MinionMetrics.register already initializes the singleton)
- Remove deprecated validDocIds legacy read path: server endpoints
GET /segments/{table}/{segment}/validDocIds and
GET /tables/{table}/validDocIdMetadata, plus
ServerSegmentMetadataReader.getValidDocIdsFromServer; migrate tests to
the validDocIdsBitmap and POST validDocIdsMetadata replacements
- Remove dead SegmentCompletionProtocol.SegmentCommitEndRequest and
MSG_TYPE_COMMIT_END (controller endpoint removed in #11662)
---
.../pinot/common/metadata/ZKMetadataProvider.java | 19 ---
.../protocols/SegmentCompletionProtocol.java | 7 --
.../apache/pinot/common/utils/LLCSegmentName.java | 5 -
.../org/apache/pinot/common/utils/SegmentName.java | 33 ------
.../helix/core/PinotHelixResourceManager.java | 5 -
.../util/ServerSegmentMetadataReader.java | 37 ------
.../PinotLLCRealtimeSegmentManagerTest.java | 1 -
.../org/apache/pinot/minion/BaseMinionStarter.java | 1 -
.../org/apache/pinot/minion/MinionContext.java | 12 --
.../pinot/server/api/resources/TablesResource.java | 79 -------------
.../pinot/server/api/TablesResourceTest.java | 128 +--------------------
.../SegmentsValidationAndRetentionConfig.java | 17 ---
.../spi/utils/builder/TableConfigBuilder.java | 7 --
13 files changed, 2 insertions(+), 349 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/metadata/ZKMetadataProvider.java
b/pinot-common/src/main/java/org/apache/pinot/common/metadata/ZKMetadataProvider.java
index e2fd61fa0df..dc8a91a01b3 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/metadata/ZKMetadataProvider.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/metadata/ZKMetadataProvider.java
@@ -161,13 +161,6 @@ public class ZKMetadataProvider {
}
}
- @Deprecated
- public static void setTableConfig(ZkHelixPropertyStore<ZNRecord>
propertyStore, String tableNameWithType,
- ZNRecord znRecord) {
-
propertyStore.set(constructPropertyStorePathForResourceConfig(tableNameWithType),
znRecord,
- AccessOption.PERSISTENT);
- }
-
/// Create table config, fail if existed.
///
/// @return true if creation is successful.
@@ -210,18 +203,6 @@ public class ZKMetadataProvider {
expectedVersion, AccessOption.PERSISTENT);
}
- @Deprecated
- public static void setRealtimeTableConfig(ZkHelixPropertyStore<ZNRecord>
propertyStore, String realtimeTableName,
- ZNRecord znRecord) {
- setTableConfig(propertyStore, realtimeTableName, znRecord);
- }
-
- @Deprecated
- public static void setOfflineTableConfig(ZkHelixPropertyStore<ZNRecord>
propertyStore, String offlineTableName,
- ZNRecord znRecord) {
- setTableConfig(propertyStore, offlineTableName, znRecord);
- }
-
public static String constructPropertyStorePathForSegment(String
resourceName, String segmentName) {
return StringUtil.join("/", PROPERTYSTORE_SEGMENTS_PREFIX, resourceName,
segmentName);
}
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/protocols/SegmentCompletionProtocol.java
b/pinot-common/src/main/java/org/apache/pinot/common/protocols/SegmentCompletionProtocol.java
index 79f5c91cfd1..e989d95957d 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/protocols/SegmentCompletionProtocol.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/protocols/SegmentCompletionProtocol.java
@@ -115,7 +115,6 @@ public class SegmentCompletionProtocol {
public static final String MSG_TYPE_COMMIT = "segmentCommit";
public static final String MSG_TYPE_COMMIT_START = "segmentCommitStart";
public static final String MSG_TYPE_SEGMENT_UPLOAD = "segmentUpload";
- public static final String MSG_TYPE_COMMIT_END = "segmentCommitEnd";
public static final String MSG_TYPE_COMMIT_END_METADATA =
"segmentCommitEndWithMetadata";
public static final String MSG_TYPE_STOPPED_CONSUMING =
"segmentStoppedConsuming";
public static final String MSG_TYPE_EXTEND_BUILD_TIME = "extendBuildTime";
@@ -413,12 +412,6 @@ public class SegmentCompletionProtocol {
}
}
- public static class SegmentCommitEndRequest extends Request {
- public SegmentCommitEndRequest(Params params) {
- super(params, MSG_TYPE_COMMIT_END);
- }
- }
-
public static class SegmentCommitEndWithMetadataRequest extends Request {
public SegmentCommitEndWithMetadataRequest(Params params) {
super(params, MSG_TYPE_COMMIT_END_METADATA);
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/utils/LLCSegmentName.java
b/pinot-common/src/main/java/org/apache/pinot/common/utils/LLCSegmentName.java
index 5e8e47fa550..6be056889b6 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/utils/LLCSegmentName.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/utils/LLCSegmentName.java
@@ -81,11 +81,6 @@ public class LLCSegmentName implements
Comparable<LLCSegmentName> {
return numSeparators == 3;
}
- @Deprecated
- public static boolean isLowLevelConsumerSegmentName(String segmentName) {
- return isLLCSegment(segmentName);
- }
-
/// Returns the sequence number of the given segment name.
public static int getSequenceNumber(String segmentName) {
return Integer.parseInt(StringUtils.splitByWholeSeparator(segmentName,
SEPARATOR)[2]);
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/utils/SegmentName.java
b/pinot-common/src/main/java/org/apache/pinot/common/utils/SegmentName.java
deleted file mode 100644
index 2395e18e4ef..00000000000
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/SegmentName.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.common.utils;
-
-@Deprecated
-public class SegmentName {
- private SegmentName() {
- }
-
- public static boolean isLowLevelConsumerSegmentName(String segmentName) {
- return LLCSegmentName.isLLCSegment(segmentName);
- }
-
- public static boolean isRealtimeSegmentName(String segmentName) {
- return LLCSegmentName.isLLCSegment(segmentName);
- }
-}
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
index 599a35f70c4..69135c8da94 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
@@ -3660,11 +3660,6 @@ public class PinotHelixResourceManager {
return consumingSegments;
}
- @Deprecated
- public Set<String> getServersForSegment(String tableNameWithType, String
segmentName) {
- return getServers(tableNameWithType, segmentName);
- }
-
public synchronized Map<String, String> getSegmentsCrcForTable(String
tableNameWithType) {
// Get the segment list for this table
IdealState is = _helixAdmin.getResourceIdealState(_helixClusterName,
tableNameWithType);
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java
index f4e8a812551..324b2df9ee6 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java
@@ -39,7 +39,6 @@ import java.util.stream.Collectors;
import javax.annotation.Nullable;
import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
import org.apache.hc.client5.http.io.HttpClientConnectionManager;
@@ -48,14 +47,12 @@ import
org.apache.pinot.common.restlet.resources.TableMetadataInfo;
import org.apache.pinot.common.restlet.resources.TableSegments;
import org.apache.pinot.common.restlet.resources.ValidDocIdsBitmapResponse;
import org.apache.pinot.common.restlet.resources.ValidDocIdsMetadataInfo;
-import org.apache.pinot.common.utils.RoaringBitmapUtils;
import org.apache.pinot.controller.api.resources.TableStaleSegmentResponse;
import org.apache.pinot.segment.local.data.manager.StaleSegment;
import org.apache.pinot.spi.utils.JsonUtils;
import org.apache.pinot.spi.utils.builder.UrlBuilderUtils;
import org.glassfish.jersey.client.ClientConfig;
import org.glassfish.jersey.client.ClientProperties;
-import org.roaringbitmap.RoaringBitmap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -393,28 +390,6 @@ public class ServerSegmentMetadataReader {
}
}
- /// This method is called when the API request is to fetch validDocIds for a
segment of the given table. This method
- /// will pick a server that hosts the target segment and fetch the
validDocIds result.
- ///
- /// @return a bitmap of validDocIds
- @Deprecated
- public RoaringBitmap getValidDocIdsFromServer(String tableNameWithType,
String segmentName, String validDocIdsType,
- String endpoint, int timeoutMs) {
- // Build the endpoint url
- String url = generateValidDocIdsURL(tableNameWithType, segmentName,
validDocIdsType, endpoint);
-
- // Set timeout
- ClientConfig clientConfig = new ClientConfig();
- clientConfig.property(ClientProperties.CONNECT_TIMEOUT, timeoutMs);
- clientConfig.property(ClientProperties.READ_TIMEOUT, timeoutMs);
-
- Response response =
ClientBuilder.newClient(clientConfig).target(url).request().get(Response.class);
- Preconditions.checkState(response.getStatus() ==
Response.Status.OK.getStatusCode(),
- "Unable to retrieve validDocIds from %s", url);
- byte[] validDocIds = response.readEntity(byte[].class);
- return RoaringBitmapUtils.deserialize(validDocIds);
- }
-
/// This method is called when the API request is to fetch validDocIds for a
segment of the given table. This method
/// will pick a server that hosts the target segment and fetch the
validDocIds result.
///
@@ -503,18 +478,6 @@ public class ServerSegmentMetadataReader {
return String.format("%s/tables/%s/segments/needReload", endpoint,
tableNameWithType);
}
- @Deprecated
- private String generateValidDocIdsURL(String tableNameWithType, String
segmentName, String validDocIdsType,
- String endpoint) {
- tableNameWithType = encode(tableNameWithType);
- segmentName = encode(segmentName);
- String url = String.format("%s/segments/%s/%s/validDocIds", endpoint,
tableNameWithType, segmentName);
- if (validDocIdsType != null) {
- url = url + "?validDocIdsType=" + validDocIdsType;
- }
- return url;
- }
-
private String generateValidDocIdsBitmapURL(String tableNameWithType, String
segmentName, String validDocIdsType,
String endpoint) {
tableNameWithType = encode(tableNameWithType);
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManagerTest.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManagerTest.java
index 47182b1787a..ce20061ce2f 100644
---
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManagerTest.java
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManagerTest.java
@@ -1904,7 +1904,6 @@ public class PinotLLCRealtimeSegmentManagerTest {
PinotHelixResourceManager helixResourceManager =
mock(PinotHelixResourceManager.class);
when(helixResourceManager.getTableConfig(REALTIME_TABLE_NAME)).thenReturn(
new TableConfigBuilder(TableType.REALTIME).setTableName(RAW_TABLE_NAME)
- .setLLC(true)
.setStreamConfigs(FakeStreamConfigUtils.getDefaultLowLevelStreamConfigs().getStreamConfigsMap())
.build());
PinotLLCRealtimeSegmentManager segmentManager =
diff --git
a/pinot-minion/src/main/java/org/apache/pinot/minion/BaseMinionStarter.java
b/pinot-minion/src/main/java/org/apache/pinot/minion/BaseMinionStarter.java
index 03e0897b5e3..05ccbd4f6eb 100644
--- a/pinot-minion/src/main/java/org/apache/pinot/minion/BaseMinionStarter.java
+++ b/pinot-minion/src/main/java/org/apache/pinot/minion/BaseMinionStarter.java
@@ -240,7 +240,6 @@ public abstract class BaseMinionStarter implements
ServiceStartable {
minionMetrics.setValueOfGlobalGauge(MinionGauge.ZK_JUTE_MAX_BUFFER,
Integer.getInteger(ZkSystemPropertyKeys.JUTE_MAXBUFFER, 0xfffff));
MinionMetrics.register(minionMetrics);
- minionContext.setMinionMetrics(minionMetrics);
minionContext.setAllowDownloadFromServer(_config.isAllowDownloadFromServer());
// Install default SSL context if necessary (even if not force-enabled
everywhere)
diff --git
a/pinot-minion/src/main/java/org/apache/pinot/minion/MinionContext.java
b/pinot-minion/src/main/java/org/apache/pinot/minion/MinionContext.java
index 41273afb25a..189d6814b08 100644
--- a/pinot-minion/src/main/java/org/apache/pinot/minion/MinionContext.java
+++ b/pinot-minion/src/main/java/org/apache/pinot/minion/MinionContext.java
@@ -23,7 +23,6 @@ import javax.net.ssl.SSLContext;
import org.apache.helix.HelixManager;
import org.apache.helix.store.zk.ZkHelixPropertyStore;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
-import org.apache.pinot.common.metrics.MinionMetrics;
import org.apache.pinot.core.minion.SegmentPurger;
import org.apache.pinot.spi.auth.AuthProvider;
@@ -40,7 +39,6 @@ public class MinionContext {
}
private File _dataDir;
- private MinionMetrics _minionMetrics;
private ZkHelixPropertyStore<ZNRecord> _helixPropertyStore;
private HelixManager _helixManager;
@@ -61,16 +59,6 @@ public class MinionContext {
_dataDir = dataDir;
}
- @Deprecated
- public MinionMetrics getMinionMetrics() {
- return _minionMetrics;
- }
-
- @Deprecated
- public void setMinionMetrics(MinionMetrics minionMetrics) {
- _minionMetrics = minionMetrics;
- }
-
public ZkHelixPropertyStore<ZNRecord> getHelixPropertyStore() {
return _helixPropertyStore;
}
diff --git
a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
index b57a9485e6d..67c98124611 100644
---
a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
+++
b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
@@ -672,85 +672,6 @@ public class TablesResource {
}
}
- /// Download snapshot for the given immutable segment for upsert table. This
endpoint is used when get snapshot from
- /// peer to avoid recompute when reload segments.
- @Deprecated
- @GET
- @Produces(MediaType.APPLICATION_OCTET_STREAM)
- @Path("/segments/{tableNameWithType}/{segmentName}/validDocIds")
- @ApiOperation(value = "Download validDocIds for an REALTIME immutable
segment", notes = "Download validDocIds for "
- + "an immutable segment in bitmap format.")
- public Response downloadValidDocIds(
- @ApiParam(value = "Name of the table with type REALTIME", required =
true, example = "myTable_REALTIME")
- @PathParam("tableNameWithType") String tableNameWithType,
- @ApiParam(value = "Name of the segment", required = true)
@PathParam("segmentName") @Encoded String segmentName,
- @ApiParam(value = "Valid doc ids type") @QueryParam("validDocIdsType")
@Nullable String validDocIdsType,
- @Context HttpHeaders httpHeaders) {
- tableNameWithType = DatabaseUtils.translateTableName(tableNameWithType,
httpHeaders);
- segmentName = URIUtils.decode(segmentName);
- LOGGER.info("Received a request to download validDocIds for segment {}
table {}", segmentName, tableNameWithType);
- // Validate data access
- ServerResourceUtils.validateDataAccess(_accessControlFactory,
tableNameWithType, httpHeaders);
-
- TableDataManager tableDataManager =
- ServerResourceUtils.checkGetTableDataManager(_serverInstance,
tableNameWithType);
- SegmentDataManager segmentDataManager =
tableDataManager.acquireSegment(segmentName);
- if (segmentDataManager == null) {
- throw new WebApplicationException(
- String.format("Table %s segment %s does not exist",
tableNameWithType, segmentName),
- Response.Status.NOT_FOUND);
- }
-
- try {
- IndexSegment indexSegment = segmentDataManager.getSegment();
- if (!(indexSegment instanceof ImmutableSegmentImpl)) {
- throw new WebApplicationException(
- String.format("Table %s segment %s is not a immutable segment",
tableNameWithType, segmentName),
- Response.Status.BAD_REQUEST);
- }
-
- final Pair<ValidDocIdsType, MutableRoaringBitmap> validDocIdSnapshotPair
=
- getValidDocIds(indexSegment, validDocIdsType);
- MutableRoaringBitmap validDocIdSnapshot =
validDocIdSnapshotPair.getRight();
- if (validDocIdSnapshot == null) {
- String msg = String.format(
- "Found that validDocIds is missing while fetching validDocIds for
table %s segment %s while "
- + "reading the validDocIds with validDocIdType %s",
- tableNameWithType, segmentDataManager.getSegmentName(),
validDocIdsType);
- LOGGER.warn(msg);
- throw new WebApplicationException(msg, Response.Status.NOT_FOUND);
- }
-
- byte[] validDocIdsBytes =
RoaringBitmapUtils.serialize(validDocIdSnapshot);
- Response.ResponseBuilder builder = Response.ok(validDocIdsBytes);
- builder.header(HttpHeaders.CONTENT_LENGTH, validDocIdsBytes.length);
- return builder.build();
- } finally {
- tableDataManager.releaseSegment(segmentDataManager);
- }
- }
-
- @Deprecated
- @GET
- @Path("/tables/{tableNameWithType}/validDocIdMetadata")
- @Produces(MediaType.APPLICATION_JSON)
- @ApiOperation(value = "Provides segment validDocId metadata", notes =
"Provides segment validDocId metadata")
- @ApiResponses(value = {
- @ApiResponse(code = 200, message = "Success"),
- @ApiResponse(code = 500, message = "Internal server error", response =
ErrorInfo.class),
- @ApiResponse(code = 404, message = "Table or segment not found",
response = ErrorInfo.class)
- })
- public String getValidDocIdsMetadata(
- @ApiParam(value = "Table name including type", required = true, example
= "myTable_REALTIME")
- @PathParam("tableNameWithType") String tableNameWithType,
- @ApiParam(value = "Valid doc ids type") @QueryParam("validDocIdsType")
String validDocIdsType,
- @ApiParam(value = "Segment name", allowMultiple = true)
@QueryParam("segmentNames") List<String> segmentNames,
- @Context HttpHeaders headers) {
- tableNameWithType = DatabaseUtils.translateTableName(tableNameWithType,
headers);
- return ResourceUtils.convertToJsonString(
- processValidDocIdsMetadata(tableNameWithType, segmentNames,
validDocIdsType));
- }
-
@POST
@Path("/tables/{tableNameWithType}/validDocIdsMetadata")
@Produces(MediaType.APPLICATION_JSON)
diff --git
a/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
b/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
index b95395189a4..fe912f3055c 100644
---
a/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
+++
b/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
@@ -377,51 +377,26 @@ public class TablesResourceTest extends BaseResourceTest {
public void testDownloadValidDocIdsSnapshot()
throws Exception {
// Verify the content of the downloaded snapshot from a realtime table.
- downLoadAndVerifyValidDocIdsSnapshot(REALTIME_TABLE_NAME,
(ImmutableSegmentImpl) _realtimeIndexSegments.get(0));
downLoadAndVerifyValidDocIdsSnapshotBitmap(REALTIME_TABLE_NAME,
(ImmutableSegmentImpl) _realtimeIndexSegments.get(0));
// Verify non-existent table and segment download return NOT_FOUND status.
Response response =
-
_webTarget.path("/tables/UNKNOWN_REALTIME/segments/segmentname/validDocIds").request().get(Response.class);
+
_webTarget.path("/segments/UNKNOWN_REALTIME/segmentname/validDocIdsBitmap").request().get(Response.class);
Assert.assertEquals(response.getStatus(),
Response.Status.NOT_FOUND.getStatusCode());
response =
- _webTarget.path(String.format("/tables/%s/segments/%s/validDocIds",
REALTIME_TABLE_NAME, "UNKNOWN_SEGMENT"))
+ _webTarget.path(String.format("/segments/%s/%s/validDocIdsBitmap",
REALTIME_TABLE_NAME, "UNKNOWN_SEGMENT"))
.request()
.get(Response.class);
Assert.assertEquals(response.getStatus(),
Response.Status.NOT_FOUND.getStatusCode());
}
- @Deprecated
- @Test
- public void testValidDocIdMetadata()
- throws IOException {
- IndexSegment segment = _realtimeIndexSegments.get(0);
- // Verify the content of the downloaded snapshot from a realtime table.
- downLoadAndVerifyValidDocIdsSnapshot(REALTIME_TABLE_NAME,
(ImmutableSegmentImpl) segment);
- downLoadAndVerifyValidDocIdsSnapshotBitmap(REALTIME_TABLE_NAME,
(ImmutableSegmentImpl) segment);
-
- String validDocIdMetadataPath = "/tables/" + REALTIME_TABLE_NAME +
"/validDocIdMetadata";
- String metadataResponse = _webTarget.path(validDocIdMetadataPath)
- .queryParam("segmentNames", segment.getSegmentName())
- .request()
- .get(String.class);
- JsonNode validDocIdMetadata =
JsonUtils.stringToJsonNode(metadataResponse).get(0);
-
- Assert.assertEquals(validDocIdMetadata.get("totalDocs").asInt(), 200000);
- Assert.assertEquals(validDocIdMetadata.get("totalValidDocs").asInt(), 8);
- Assert.assertEquals(validDocIdMetadata.get("totalInvalidDocs").asInt(),
199992);
- Assert.assertEquals(validDocIdMetadata.get("segmentCrc").asText(),
segment.getSegmentMetadata().getCrc());
- Assert.assertEquals(validDocIdMetadata.get("validDocIdsType").asText(),
"SNAPSHOT");
- }
-
@Test
public void testValidDocIdsMetadataPost()
throws IOException {
IndexSegment segment = _realtimeIndexSegments.get(0);
// Verify the content of the downloaded snapshot from a realtime table.
- downLoadAndVerifyValidDocIdsSnapshot(REALTIME_TABLE_NAME,
(ImmutableSegmentImpl) segment);
downLoadAndVerifyValidDocIdsSnapshotBitmap(REALTIME_TABLE_NAME,
(ImmutableSegmentImpl) segment);
List<String> segments = List.of(segment.getSegmentName());
@@ -455,7 +430,6 @@ public class TablesResourceTest extends BaseResourceTest {
throws IOException {
IndexSegment segment = _realtimeIndexSegments.get(0);
// Verify the content of the downloaded snapshot from a realtime table.
- downLoadAndVerifyValidDocIdsSnapshot(REALTIME_TABLE_NAME,
(ImmutableSegmentImpl) segment);
downLoadAndVerifyValidDocIdsSnapshotBitmap(REALTIME_TABLE_NAME,
(ImmutableSegmentImpl) segment);
List<String> segments = List.of(segment.getSegmentName());
@@ -515,78 +489,6 @@ public class TablesResourceTest extends BaseResourceTest {
FileUtils.forceDelete(tempMetadataDir);
}
- // Verify metadata file from segments.
- private void downLoadAndVerifyValidDocIdsSnapshot(String tableNameWithType,
ImmutableSegmentImpl segment)
- throws IOException {
- String snapshotPath = "/segments/" + tableNameWithType + "/" +
segment.getSegmentName() + "/validDocIds";
-
- PartitionUpsertMetadataManager upsertMetadataManager =
mock(PartitionUpsertMetadataManager.class);
- ThreadSafeMutableRoaringBitmap validDocIds = new
ThreadSafeMutableRoaringBitmap();
- ThreadSafeMutableRoaringBitmap queryableDocIds = new
ThreadSafeMutableRoaringBitmap();
- ThreadSafeMutableRoaringBitmap validDocIdsSnapshot = new
ThreadSafeMutableRoaringBitmap();
-
- int[] docIds = new int[]{1, 4, 6, 10, 15, 17, 18, 20};
- for (int docId : docIds) {
- validDocIds.add(docId);
- queryableDocIds.add(docId + 1);
- validDocIdsSnapshot.add(docId + 2);
- }
- segment.enableUpsert(upsertMetadataManager, validDocIds, queryableDocIds);
- File validDocIdsSnapshotFile =
- new
File(SegmentDirectoryPaths.findSegmentDirectory(segment.getSegmentMetadata().getIndexDir()),
- V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME);
- FileUtils.writeByteArrayToFile(validDocIdsSnapshotFile,
-
RoaringBitmapUtils.serialize(validDocIdsSnapshot.getMutableRoaringBitmap()));
-
- // Create the queryableDocIds snapshot file needed for SNAPSHOT_WITH_DELETE
- File queryableDocIdsSnapshotFile =
- new
File(SegmentDirectoryPaths.findSegmentDirectory(segment.getSegmentMetadata().getIndexDir()),
- V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME);
- FileUtils.writeByteArrayToFile(queryableDocIdsSnapshotFile,
-
RoaringBitmapUtils.serialize(queryableDocIds.getMutableRoaringBitmap()));
-
- // Check no type (default should be validDocIdsSnapshot)
- Response response =
_webTarget.path(snapshotPath).request().get(Response.class);
- Assert.assertEquals(response.getStatus(),
Response.Status.OK.getStatusCode());
- byte[] validDocIdsSnapshotBitmap = response.readEntity(byte[].class);
- Assert.assertNotNull(validDocIdsSnapshotBitmap);
- Assert.assertEquals(new
ImmutableRoaringBitmap(ByteBuffer.wrap(validDocIdsSnapshotBitmap)).toMutableRoaringBitmap(),
- validDocIdsSnapshot.getMutableRoaringBitmap());
-
- // Check snapshot type
- response = _webTarget.path(snapshotPath)
- .queryParam("validDocIdsType", ValidDocIdsType.SNAPSHOT.toString())
- .request()
- .get(Response.class);
- Assert.assertEquals(response.getStatus(),
Response.Status.OK.getStatusCode());
- validDocIdsSnapshotBitmap = response.readEntity(byte[].class);
- Assert.assertNotNull(validDocIdsSnapshotBitmap);
- Assert.assertEquals(new
ImmutableRoaringBitmap(ByteBuffer.wrap(validDocIdsSnapshotBitmap)).toMutableRoaringBitmap(),
- validDocIdsSnapshot.getMutableRoaringBitmap());
-
- // Check onHeap type
- response = _webTarget.path(snapshotPath)
- .queryParam("validDocIdsType", ValidDocIdsType.IN_MEMORY)
- .request()
- .get(Response.class);
- Assert.assertEquals(response.getStatus(),
Response.Status.OK.getStatusCode());
- validDocIdsSnapshotBitmap = response.readEntity(byte[].class);
- Assert.assertNotNull(validDocIdsSnapshotBitmap);
- Assert.assertEquals(new
ImmutableRoaringBitmap(ByteBuffer.wrap(validDocIdsSnapshotBitmap)).toMutableRoaringBitmap(),
- validDocIds.getMutableRoaringBitmap());
-
- // Check onHeapWithDelete type
- response = _webTarget.path(snapshotPath)
- .queryParam("validDocIdsType",
ValidDocIdsType.IN_MEMORY_WITH_DELETE.toString())
- .request()
- .get(Response.class);
- Assert.assertEquals(response.getStatus(),
Response.Status.OK.getStatusCode());
- validDocIdsSnapshotBitmap = response.readEntity(byte[].class);
- Assert.assertNotNull(validDocIdsSnapshotBitmap);
- Assert.assertEquals(new
ImmutableRoaringBitmap(ByteBuffer.wrap(validDocIdsSnapshotBitmap)).toMutableRoaringBitmap(),
- queryableDocIds.getMutableRoaringBitmap());
- }
-
private void downLoadAndVerifyValidDocIdsSnapshotBitmap(String
tableNameWithType, ImmutableSegmentImpl segment)
throws IOException {
String snapshotPath = "/segments/" + tableNameWithType + "/" +
segment.getSegmentName() + "/validDocIdsBitmap";
@@ -666,37 +568,11 @@ public class TablesResourceTest extends BaseResourceTest {
queryableDocIds.getMutableRoaringBitmap());
}
- @Test
- public void testValidDocIdsMetadataGetForSnapshotWithDelete()
- throws IOException {
- IndexSegment segment = _realtimeIndexSegments.get(0);
- // Verify the content of the downloaded snapshot from a realtime table.
- downLoadAndVerifyValidDocIdsSnapshot(REALTIME_TABLE_NAME,
(ImmutableSegmentImpl) segment);
- downLoadAndVerifyValidDocIdsSnapshotBitmap(REALTIME_TABLE_NAME,
(ImmutableSegmentImpl) segment);
-
- String validDocIdsMetadataPath = "/tables/" + REALTIME_TABLE_NAME +
"/validDocIdMetadata";
-
- // Test GET endpoint with SNAPSHOT_WITH_DELETE validDocIdsType
- String response = _webTarget.path(validDocIdsMetadataPath)
- .queryParam("segmentNames", segment.getSegmentName())
- .queryParam("validDocIdsType",
ValidDocIdsType.SNAPSHOT_WITH_DELETE.toString())
- .request()
- .get(String.class);
- JsonNode validDocIdsMetadata = JsonUtils.stringToJsonNode(response).get(0);
-
- Assert.assertEquals(validDocIdsMetadata.get("totalDocs").asInt(), 200000);
- Assert.assertEquals(validDocIdsMetadata.get("totalValidDocs").asInt(), 8);
- Assert.assertEquals(validDocIdsMetadata.get("totalInvalidDocs").asInt(),
199992);
- Assert.assertEquals(validDocIdsMetadata.get("segmentCrc").asText(),
segment.getSegmentMetadata().getCrc());
- Assert.assertEquals(validDocIdsMetadata.get("validDocIdsType").asText(),
"SNAPSHOT_WITH_DELETE");
- }
-
@Test
public void testValidDocIdsBitmapForSnapshotWithDelete()
throws IOException {
IndexSegment segment = _realtimeIndexSegments.get(0);
// Verify the content of the downloaded snapshot from a realtime table.
- downLoadAndVerifyValidDocIdsSnapshot(REALTIME_TABLE_NAME,
(ImmutableSegmentImpl) segment);
downLoadAndVerifyValidDocIdsSnapshotBitmap(REALTIME_TABLE_NAME,
(ImmutableSegmentImpl) segment);
String validDocIdsBitmapPath =
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/SegmentsValidationAndRetentionConfig.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/SegmentsValidationAndRetentionConfig.java
index 89a1525ebe7..9027fd3ec57 100644
---
a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/SegmentsValidationAndRetentionConfig.java
+++
b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/SegmentsValidationAndRetentionConfig.java
@@ -18,7 +18,6 @@
*/
package org.apache.pinot.spi.config.table;
-import com.fasterxml.jackson.annotation.JsonIgnore;
import java.util.concurrent.TimeUnit;
import org.apache.pinot.spi.config.BaseJsonConfig;
import org.apache.pinot.spi.utils.TimeUtils;
@@ -198,22 +197,6 @@ public class SegmentsValidationAndRetentionConfig extends
BaseJsonConfig {
_completionConfig = completionConfig;
}
- /// Try to Use [TableConfig#getReplication()]
- @Deprecated
- @JsonIgnore
- public int getReplicationNumber() {
- return Integer.parseInt(_replication);
- }
-
- /// Try to Use [TableConfig#getReplication()]
- ///
- /// Will be deleted in future version of Pinot
- @Deprecated
- @JsonIgnore
- public int getReplicasPerPartitionNumber() {
- return Integer.parseInt(_replicasPerPartition);
- }
-
public String getPeerSegmentDownloadScheme() {
return _peerSegmentDownloadScheme;
}
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/TableConfigBuilder.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/TableConfigBuilder.java
index f6503246c07..77a246ab20c 100644
---
a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/TableConfigBuilder.java
+++
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/TableConfigBuilder.java
@@ -175,13 +175,6 @@ public class TableConfigBuilder {
return this;
}
- @Deprecated
- public TableConfigBuilder setLLC(boolean isLLC) {
- Preconditions.checkState(_tableType == TableType.REALTIME);
- Preconditions.checkArgument(isLLC, "Real-time table must use LLC");
- return this;
- }
-
public TableConfigBuilder setNumReplicas(int numReplicas) {
Preconditions.checkArgument(numReplicas > 0);
_numReplicas = String.valueOf(numReplicas);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
