[
https://issues.apache.org/jira/browse/HIVE-26716?focusedWorklogId=828890&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-828890
]
ASF GitHub Bot logged work on HIVE-26716:
-----------------------------------------
Author: ASF GitHub Bot
Created on: 25/Nov/22 11:02
Start Date: 25/Nov/22 11:02
Worklog Time Spent: 10m
Work Description: deniskuzZ commented on code in PR #3746:
URL: https://github.com/apache/hive/pull/3746#discussion_r1032313195
##########
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorFactory.java:
##########
@@ -48,39 +56,54 @@ private QueryCompactorFactory() {
* <br>
* {@link MmMinorQueryCompactor} - handles query based minor compaction for
micro-managed tables
* <br>
+ * {@link MRCompactor} - handles MR based minor, major, or rebalance
compaction
+ * <br>
+ * {@link RebalanceQueryCompactor} - handles query based rebalance compaction
+ * <br>
* </p>
+ * @param msc The {@link IMetaStoreClient} instance is used only by the
{@link MRCompactor}.
* @param table the table, on which the compaction should be running, must
be not null.
* @param configuration the hive configuration, must be not null.
* @param compactionInfo provides insight about the type of compaction, must
be not null.
* @return {@link QueryCompactor} or null.
*/
- static QueryCompactor getQueryCompactor(Table table, HiveConf configuration,
CompactionInfo compactionInfo)
+ public Compactor getQueryCompactor(IMetaStoreClient msc, Table table,
HiveConf configuration, CompactionInfo compactionInfo)
throws HiveException {
- if (!AcidUtils.isInsertOnlyTable(table.getParameters())
- && HiveConf.getBoolVar(configuration,
HiveConf.ConfVars.COMPACTOR_CRUD_QUERY_BASED)) {
- if (!"tez".equalsIgnoreCase(HiveConf.getVar(configuration,
HiveConf.ConfVars.HIVE_EXECUTION_ENGINE))) {
+ if (AcidUtils.isFullAcidTable(table.getParameters())) {
+ if (!"tez".equalsIgnoreCase(HiveConf.getVar(configuration,
HiveConf.ConfVars.HIVE_EXECUTION_ENGINE)) ||
+ !HiveConf.getBoolVar(configuration,
HiveConf.ConfVars.COMPACTOR_CRUD_QUERY_BASED)) {
+ if (CompactionType.REBALANCE.equals(compactionInfo.type)) {
+ throw new HiveException("Rebalancing compaction is only supported in
Tez, and via Query based compaction. " +
+ "Set hive.execution.engine=tez and
hive.compactor.crud.query.based=true to enable it.");
+ }
LOG.info("Query-based compaction is only supported on tez. Falling
back to MR compaction.");
- return null;
+ return new MRCompactor(msc);
}
- if (compactionInfo.isMajorCompaction()) {
- return new MajorQueryCompactor();
- } else {
- return new MinorQueryCompactor();
+ switch (compactionInfo.type) {
+ case MINOR:
+ return new MinorQueryCompactor();
+ case MAJOR:
+ return new MajorQueryCompactor();
+ case REBALANCE:
+ return new RebalanceQueryCompactor();
}
- }
- if (AcidUtils.isInsertOnlyTable(table.getParameters())) {
+ } else if (AcidUtils.isInsertOnlyTable(table.getParameters())) {
if
(!configuration.getBoolVar(HiveConf.ConfVars.HIVE_COMPACTOR_COMPACT_MM)) {
throw new HiveException(
- "Insert only compaction is disabled. Set
hive.compactor.compact.insert.only to true to enable it.");
+ "Insert only compaction is disabled. Set
hive.compactor.compact.insert.only=true to enable it.");
}
- if (compactionInfo.isMajorCompaction()) {
- return new MmMajorQueryCompactor();
- } else {
- return new MmMinorQueryCompactor();
+ switch (compactionInfo.type) {
+ case MINOR:
+ return new MmMinorQueryCompactor();
+ case MAJOR:
+ return new MmMajorQueryCompactor();
+ case REBALANCE:
Review Comment:
should be under default:
compactionInfo.type + compaction is not supported on insert only tables.
Issue Time Tracking
-------------------
Worklog Id: (was: 828890)
Time Spent: 7.5h (was: 7h 20m)
> Query based Rebalance compaction on full acid tables
> ----------------------------------------------------
>
> Key: HIVE-26716
> URL: https://issues.apache.org/jira/browse/HIVE-26716
> Project: Hive
> Issue Type: Sub-task
> Components: Hive
> Reporter: László Végh
> Assignee: László Végh
> Priority: Major
> Labels: ACID, compaction, pull-request-available
> Time Spent: 7.5h
> Remaining Estimate: 0h
>
> Support rebalancing compaction on fully ACID tables.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
