[
https://issues.apache.org/jira/browse/PHOENIX-7108?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17816704#comment-17816704
]
ASF GitHub Bot commented on PHOENIX-7108:
-----------------------------------------
jpisaac commented on code in PR #1799:
URL: https://github.com/apache/phoenix/pull/1799#discussion_r1486627123
##########
phoenix-core/src/main/java/org/apache/phoenix/coprocessor/CompactionScanner.java:
##########
@@ -160,17 +241,312 @@ public void close() throws IOException {
storeScanner.close();
}
+ private interface TTLTracker {
+ void setTTL(Cell firstCell);
+ RowContext getRowContext();
+ }
+ private class NonPartitionedTableTTLTracker implements TTLTracker {
+
+ private long ttl;
+ private RowContext rowContext;
+
+ public NonPartitionedTableTTLTracker(PTable pTable,
RegionCoprocessorEnvironment env, Store store) {
+ boolean isSystemTable = pTable.getType() == PTableType.SYSTEM;
+ if (isSystemTable) {
+ ColumnFamilyDescriptor cfd = store.getColumnFamilyDescriptor();
+ ttl = cfd.getTimeToLive();
+ } else {
+ ttl = pTable.getTTL() != TTL_NOT_DEFINED ? pTable.getTTL() :
DEFAULT_TTL;
+ }
+ LOGGER.info(String.format("NonPartitionedTableTTLTracker params:-
(physical-name=%s, ttl=%d, isSystemTable=%s)",
+ pTable.getName().toString(), ttl*1000, isSystemTable));
+ }
+
+ @Override
+ public void setTTL(Cell firstCell) {
+ this.rowContext = new RowContext();
+ this.rowContext.setTTL(ttl);
+
+ }
+
+ @Override
+ public RowContext getRowContext() {
+ if (this.rowContext == null) {
+ this.rowContext = new RowContext();
+ this.rowContext.setTTL(ttl);
+ }
+ return rowContext;
+ }
+ }
+
+ private class PartitionedTableTTLTracker implements TTLTracker {
+ private final Logger LOGGER = LoggerFactory.getLogger(
+ PartitionedTableTTLTracker.class);
+ private PTable baseTable;
+ private TableTTLInfoCache ttlCache;
+ private PrefixIndex index;
+
+ // Default or Table-Level TTL
+ private long ttl;
+ private RowContext rowContext;
+
+ private boolean isIndexTable = false;
+ private boolean isMultiTenant = false;
+ private boolean isSalted = false;
+ private int startingPKPosition;
+ private RowKeyParser rowKeyParser;
+
+ public PartitionedTableTTLTracker(PTable table,
RegionCoprocessorEnvironment env,
+ boolean isSalted, boolean isIndexTable) {
+
+ try {
+ this.baseTable = table;
+ this.ttlCache = new TableTTLInfoCache();
+ this.index = new PrefixIndex();
+ this.rowKeyParser = new RowKeyParser(baseTable);
+ this.ttl = table.getTTL() != TTL_NOT_DEFINED ? table.getTTL()
: DEFAULT_TTL;
+ this.isIndexTable = isIndexTable || localIndex ;
+ this.isSalted = isSalted;
+ this.isMultiTenant = table.isMultiTenant();
+
+ int startingPKPosition = 0;
+ if (this.isMultiTenant && this.isSalted && this.isIndexTable) {
+ // case multi-tenant + salted + index-table =>
+ // startingPKPosition = 1 skip the salt-byte and starting
at the viewIndexId
+ startingPKPosition = 1;
+ } else if (this.isMultiTenant && this.isSalted) {
+ // case multi-tenant + salted =>
+ // startingPKPosition = 2 skip salt byte + tenant-id to
search the global space
+ // if above search returned no results
+ // then search using the following start position
+ // startingPKPosition = 1 skip salt-byte to search the
tenant space
+ startingPKPosition = 2;
+ } else if (this.isMultiTenant && this.isIndexTable) {
+ // case multi-tenant + index-table =>
+ // startingPKPosition = 0, the first key will the
viewIndexId
+ startingPKPosition = 0;
+ } else if (this.isSalted && this.isIndexTable) {
+ // case salted + index-table =>
+ // startingPKPosition = 1 skip salt-byte search using the
viewIndexId
+ startingPKPosition = 1;
+ } else if (this.isSalted) {
+ // case salted =>
+ // startingPKPosition = 1 skip salt-byte
+ startingPKPosition = 1;
+ } else if (this.isIndexTable) {
+ // case index-table =>
+ // startingPKPosition = 0 the first key will the
viewIndexId
+ startingPKPosition = 0;
+ } else if (this.isMultiTenant) {
+ // case multi-tenant =>
+ // startingPKPosition = 1 skip tenant-id to search the
global space
+ // if above search returned no results
+ // then search using the following start position
+ // startingPKPosition = 0 to search the tenant space
+ startingPKPosition = 1;
+ } else {
+ // case not multi-tenant + not salted + not index-table =>
+ // startingPKPosition = 0
+ startingPKPosition = 0;
+ }
+
+ this.startingPKPosition = startingPKPosition;
+
+ LOGGER.info(String.format("PartitionedTableTTLTracker params:-
" +
+ "region-name = %s, table-name = %s, " +
+ "multi-tenant = %s, index-table = %s, salted =
%s, " +
+ "default-ttl = %d, startingPKPosition = %d",
+ region.getRegionInfo().getEncodedName(),
+ region.getRegionInfo().getTable().getNameAsString(),
+ this.isMultiTenant,
+ this.isIndexTable,
+ this.isSalted,
+ this.ttl,
+ this.startingPKPosition));
+
+ List<TableTTLInfo> tableList;
+ tableList = ViewUtil.getRowKeyPrefixesForPartitionedTables(
+ table.getName().getString(),
+ env.getConfiguration(),
+ this.isIndexTable);
+
+ tableList.forEach(m -> {
+ if (m.getTTL() != TTL_NOT_DEFINED) {
+ // add the ttlInfo to the cache.
+ // each new/unique ttlInfo object added returns a
unique tableId.
+ int tableId = ttlCache.addTable(m);
+ // map the row-prefix to the tableId using prefix
index.
+ index.put(m.getPrefix(), tableId);
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(String.format("Updated index : " +
m.toString()));
+ }
+
+ }
+ });
+ } catch (Exception e) {
+ LOGGER.error(String.format("Failed to read from catalog: " +
e.getMessage()));
+ } finally {
+ LOGGER.info(String.format("PartitionedTableTTLTracker " +
+ "index-entries = %d, cache-entries = %d for
region = %s",
+ index.getValidPrefixes(),
ttlCache.getNumTablesInCache(),
+
CompactionScanner.this.store.getRegionInfo().getEncodedName()));
+ }
+
+ }
+ @Override
+ public void setTTL(Cell firstCell) {
+
+ boolean matched = false;
+ TableTTLInfo tableTTLInfo = null;
+ List<Integer> pkPositions = null;
+ long rowTTLInSecs = ttl;
+ long searchOffset = -1;
+ int pkPosition = startingPKPosition;
+ try {
+ if (ttlCache.getNumTablesInCache() == 0) {
+ // case: no views in the hierarchy had TTL set
+ // Use the default TTL
+ this.rowContext = new RowContext();
+ this.rowContext.setTTL(rowTTLInSecs);
+ return;
+ }
+ // pkPositions holds the byte offsets for the PKs of the base
table
+ // for the current row
+ pkPositions = isIndexTable ?
+ (isSalted ?
+ Arrays.asList(0, 1) :
+ Arrays.asList(0)) :
+ rowKeyParser.parsePKPositions(firstCell);
+ // The startingPKPosition was initialized(constructor) in the
following manner
+ // case multi-tenant + salted + index-table =>
startingPKPosition = 1
+ // case multi-tenant + salted => startingPKPosition = 2, 1
+ // case multi-tenant + index-table => startingPKPosition = 0
+ // case salted + index-table => startingPKPosition = 1
+ // case salted => startingPKPosition = 1
+ // case index-table => startingPKPosition = 0
+ // case multi-tenant => startingPKPosition = 1, 0
+ int offset = pkPositions.get(pkPosition);
+ byte[] rowKey = CellUtil.cloneRow(firstCell);
+ // Search using the starting offset (startingPKPosition offset)
+ Integer tableId = index.getTableIdWithPrefix(rowKey, offset);
Review Comment:
This is an interesting case of overlap between the global space and tenant
space, Currently KP1Tenant#KP2 row will match the global keyprefix "KP1". The
fix will be to keep the two prefix indexes (global and tenant) separate.
> Provide support for pruning expired rows of views using Phoenix level
> compactions
> ---------------------------------------------------------------------------------
>
> Key: PHOENIX-7108
> URL: https://issues.apache.org/jira/browse/PHOENIX-7108
> Project: Phoenix
> Issue Type: Sub-task
> Reporter: Jacob Isaac
> Assignee: Jacob Isaac
> Priority: Major
>
> Modify Phoenix compaction framework introduced in PHOENIX-6888 to prune TTL
> expired rows of views.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
