[
https://issues.apache.org/jira/browse/PHOENIX-7108?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17817933#comment-17817933
]
ASF GitHub Bot commented on PHOENIX-7108:
-----------------------------------------
jpisaac commented on code in PR #1799:
URL: https://github.com/apache/phoenix/pull/1799#discussion_r1492361003
##########
phoenix-core/src/main/java/org/apache/phoenix/coprocessor/CompactionScanner.java:
##########
@@ -84,45 +110,99 @@ public CompactionScanner(RegionCoprocessorEnvironment env,
Store store,
InternalScanner storeScanner,
long maxLookbackInMillis,
- byte[] emptyCF,
- byte[] emptyCQ,
- int phoenixTTL,
- boolean isSystemTable) {
+ PTable table) throws IOException {
this.storeScanner = storeScanner;
this.region = env.getRegion();
this.store = store;
this.env = env;
- this.emptyCF = emptyCF;
- this.emptyCQ = emptyCQ;
+ this.emptyCF = SchemaUtil.getEmptyColumnFamily(table);
+ this.emptyCQ = table.getEncodingScheme() ==
PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS ?
+ QueryConstants.EMPTY_COLUMN_BYTES :
table.getEncodingScheme().encode(QueryConstants.ENCODED_EMPTY_COLUMN_NAME);
this.config = env.getConfiguration();
compactionTime = EnvironmentEdgeManager.currentTimeMillis();
- this.maxLookbackInMillis = maxLookbackInMillis;
String columnFamilyName = store.getColumnFamilyName();
storeColumnFamily = columnFamilyName.getBytes();
String tableName = region.getRegionInfo().getTable().getNameAsString();
Long overriddenMaxLookback =
maxLookbackMap.remove(tableName + SEPARATOR +
columnFamilyName);
maxLookbackInMillis = overriddenMaxLookback == null ?
maxLookbackInMillis : Math.max(maxLookbackInMillis,
overriddenMaxLookback);
+
+ this.maxLookbackInMillis = maxLookbackInMillis;
// The oldest scn is current time - maxLookbackInMillis. Phoenix sets
the scan time range
// for scn queries [0, scn). This means that the maxlookback size
should be
// maxLookbackInMillis + 1 so that the oldest scn does not return
empty row
- this.maxLookbackWindowStart = maxLookbackInMillis == 0 ?
- compactionTime : compactionTime - (maxLookbackInMillis + 1);
+ this.maxLookbackWindowStart = maxLookbackInMillis == 0 ?
compactionTime : compactionTime - (maxLookbackInMillis + 1);
ColumnFamilyDescriptor cfd = store.getColumnFamilyDescriptor();
- this.ttl = isSystemTable ? cfd.getTimeToLive() : phoenixTTL;
- this.ttlWindowStart = ttl == HConstants.FOREVER ? 1 : compactionTime -
ttl * 1000;
- ttl *= 1000;
- this.maxLookbackWindowStart = Math.max(ttlWindowStart,
maxLookbackWindowStart);
this.minVersion = cfd.getMinVersions();
this.maxVersion = cfd.getMaxVersions();
this.keepDeletedCells = cfd.getKeepDeletedCells();
familyCount = region.getTableDescriptor().getColumnFamilies().length;
localIndex =
columnFamilyName.startsWith(LOCAL_INDEX_COLUMN_FAMILY_PREFIX);
- emptyCFStore = familyCount == 1 ||
columnFamilyName.equals(Bytes.toString(emptyCF))
- || localIndex;
- phoenixLevelRowCompactor = new PhoenixLevelRowCompactor();
- hBaseLevelRowCompactor = new HBaseLevelRowCompactor();
+ emptyCFStore = familyCount == 1 ||
columnFamilyName.equals(Bytes.toString(emptyCF)) || localIndex;
+ // TODO: check if is it appropriate to throw an IOException here
+ TTLTracker ttlTracker = createTTLTrackerFor(env, store, table);
+ phoenixLevelRowCompactor = new PhoenixLevelRowCompactor(ttlTracker);
+ hBaseLevelRowCompactor = new HBaseLevelRowCompactor(ttlTracker);
+ LOGGER.info(String.format("CompactionScanner params:- (" +
+ "physical-data-tablename = %s, compaction-tablename =
%s, " +
+ "emptyCF = %s, emptyCQ = %s, " +
+ "minVersion = %d, maxVersion = %d, keepDeletedCells =
%s, " +
+ "familyCount = %d, localIndex = %s, emptyCFStore = %s,
" +
+ "compactionTime = %d, maxLookbackWindowStart = %d,
maxLookbackInMillis = %d)",
+ table.getName().toString(), tableName,
+ Bytes.toString(this.emptyCF), Bytes.toString(emptyCQ),
+ this.minVersion, this.maxVersion, this.keepDeletedCells.name(),
+ this.familyCount, this.localIndex, this.emptyCFStore,
+ compactionTime, maxLookbackWindowStart, maxLookbackInMillis));
+
+ }
+
+ /**
+ * Helper method to create TTL tracker for various phoenix data model
objects
+ * i.e views, view indexes ...
+ * @param env
+ * @param store
+ * @param baseTable
+ * @return
+ */
+ private TTLTracker createTTLTrackerFor(RegionCoprocessorEnvironment env,
+ Store store, PTable baseTable) throws IOException {
+
+ long currentTime = EnvironmentEdgeManager.currentTimeMillis();
+ String compactionTableName =
env.getRegion().getRegionInfo().getTable().getNameAsString();
+ String schemaName =
SchemaUtil.getSchemaNameFromFullName(baseTable.getName().toString());
+ String tableName =
SchemaUtil.getTableNameFromFullName(baseTable.getName().toString());
+
+ boolean isPartitionedIndexTable = false;
+ if
(compactionTableName.startsWith(MetaDataUtil.VIEW_INDEX_TABLE_PREFIX)) {
+ isPartitionedIndexTable = true;
+ }
+
+ // NonPartitioned: Salt bucket property can be separately set for base
tables and indexes.
+ // Partitioned: Salt bucket property can be set only for the base
table.
+ // Global views, Tenant views, view indexes inherit the salt bucket
property from their
+ // base table.
+ boolean isSalted = baseTable.getBucketNum() != null;
+ try (PhoenixConnection serverConnection =
QueryUtil.getConnectionOnServer(new Properties(),
+ env.getConfiguration()).unwrap(PhoenixConnection.class)) {
+
+ Table childLinkHTable =
serverConnection.getQueryServices().getTable(SYSTEM_CHILD_LINK_NAME_BYTES);
+ // If there is atleast one child view for this table then it is a
partitioned table.
+ boolean isPartitioned = ViewUtil.hasChildViews(
+ childLinkHTable,
+ EMPTY_BYTE_ARRAY,
+ Bytes.toBytes(schemaName),
+ Bytes.toBytes(tableName),
+ currentTime);
+
+ return isPartitioned ?
+ new PartitionedTableTTLTracker(baseTable, env, isSalted,
isPartitionedIndexTable) :
Review Comment:
>> View indexes are partitioned table but they do not have any children in
the child link table
Correct!, but if there are no views defined in SYSCAT for this table, we do
not expect any rows to be in them.
> Provide support for pruning expired rows of views using Phoenix level
> compactions
> ---------------------------------------------------------------------------------
>
> Key: PHOENIX-7108
> URL: https://issues.apache.org/jira/browse/PHOENIX-7108
> Project: Phoenix
> Issue Type: Sub-task
> Reporter: Jacob Isaac
> Assignee: Jacob Isaac
> Priority: Major
>
> Modify Phoenix compaction framework introduced in PHOENIX-6888 to prune TTL
> expired rows of views.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
