raghavgautam commented on code in PR #10639:
URL: https://github.com/apache/pinot/pull/10639#discussion_r1171726625
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/dedup/PartitionDedupMetadataManager.java:
##########
@@ -39,46 +44,73 @@ public class PartitionDedupMetadataManager {
private final List<String> _primaryKeyColumns;
private final int _partitionId;
private final ServerMetrics _serverMetrics;
- private final HashFunction _hashFunction;
-
- @VisibleForTesting
- final ConcurrentHashMap<Object, IndexSegment> _primaryKeyToSegmentMap = new
ConcurrentHashMap<>();
+ private final DedupConfig _dedupConfig;
+ @VisibleForTesting final LocalKeyValueStore _keyValueStore;
public PartitionDedupMetadataManager(String tableNameWithType, List<String>
primaryKeyColumns, int partitionId,
- ServerMetrics serverMetrics, HashFunction hashFunction) {
+ ServerMetrics serverMetrics, DedupConfig dedupConfig) {
_tableNameWithType = tableNameWithType;
_primaryKeyColumns = primaryKeyColumns;
_partitionId = partitionId;
_serverMetrics = serverMetrics;
- _hashFunction = hashFunction;
+ _dedupConfig = dedupConfig;
+ try {
+ byte[] id = (tableNameWithType + "@" + partitionId).getBytes();
+ _keyValueStore = StringUtils.isEmpty(dedupConfig.getKeyStore())
+ ? new ConcurrentHashMapKeyValueStore(id)
+ : PluginManager.get().createInstance(
+ dedupConfig.getKeyStore(),
+ new Class[]{byte[].class},
+ new Object[]{id}
+ );
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
}
public void addSegment(IndexSegment segment) {
- // Add all PKs to _primaryKeyToSegmentMap
Iterator<PrimaryKey> primaryKeyIterator = getPrimaryKeyIterator(segment);
+ byte[] serializedSegment = serializeSegment(segment);
+ List<Pair<byte[], byte[]>> keyValues = new ArrayList<>();
while (primaryKeyIterator.hasNext()) {
PrimaryKey pk = primaryKeyIterator.next();
- _primaryKeyToSegmentMap.put(HashUtils.hashPrimaryKey(pk, _hashFunction),
segment);
+ byte[] serializedPrimaryKey =
serializePrimaryKey(HashUtils.hashPrimaryKey(pk,
_dedupConfig.getHashFunction()));
+ keyValues.add(Pair.of(serializedPrimaryKey, serializedSegment));
}
+ _keyValueStore.putBatch(keyValues);
_serverMetrics.setValueOfPartitionGauge(_tableNameWithType, _partitionId,
ServerGauge.DEDUP_PRIMARY_KEYS_COUNT,
- _primaryKeyToSegmentMap.size());
+ _keyValueStore.getKeyCount());
+ }
+
+ @VisibleForTesting
+ static byte[] serializeSegment(IndexSegment segment) {
+ return segment.getSegmentName().getBytes();
+ }
+
+ @VisibleForTesting
+ static byte[] serializePrimaryKey(Object pk) {
+ if (pk instanceof PrimaryKey) {
+ return ((PrimaryKey) pk).asBytes();
+ }
+ if (pk instanceof ByteArray) {
+ return ((ByteArray) pk).getBytes();
+ }
+ throw new RuntimeException("Invalid primary key: " + pk);
Review Comment:
Will make the change.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]