This is an automated email from the ASF dual-hosted git repository.
dengzhhu653 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new d00655d64de HIVE-30194: HiveMetaTool drops silently without clear
error message (#6887)
d00655d64de is described below
commit d00655d64de27864f9e32adf17d8e28b2243ff59
Author: dengzh <[email protected]>
AuthorDate: Wed Nov 4 07:41:19 2026 +0800
HIVE-30194: HiveMetaTool drops silently without clear error message (#6887)
---
.../hive/metastore/tools/ColumnDeduplicator.java | 124 +++++++++++++++------
.../hive/metastore/tools/MetaToolObjectStore.java | 4 +-
.../metastore/tools/metatool/HiveMetaTool.java | 1 +
.../tools/metatool/HiveMetaToolCommandLine.java | 26 ++++-
.../tools/metatool/MetaToolTaskDedupColumns.java | 37 ++++--
.../metastore/TestHMSColumnDescriptorReuse.java | 118 +++++++++++++++++++-
.../metatool/TestHiveMetaToolCommandLine.java | 23 ++++
7 files changed, 287 insertions(+), 46 deletions(-)
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/ColumnDeduplicator.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/ColumnDeduplicator.java
index 16cb5f29136..27e31a3f361 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/ColumnDeduplicator.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/ColumnDeduplicator.java
@@ -20,6 +20,7 @@
package org.apache.hadoop.hive.metastore.tools;
import org.apache.commons.collections4.CollectionUtils;
+import org.apache.hadoop.hive.common.TableName;
import org.apache.hadoop.hive.metastore.Batchable;
import org.apache.hadoop.hive.metastore.Deadline;
import org.apache.hadoop.hive.metastore.RawStore;
@@ -33,6 +34,7 @@
import org.apache.hadoop.hive.metastore.model.MPartition;
import org.apache.hadoop.hive.metastore.model.MStorageDescriptor;
import org.apache.hadoop.hive.metastore.model.MTable;
+import javax.jdo.JDOObjectNotFoundException;
import javax.jdo.PersistenceManager;
import javax.jdo.Query;
import java.util.ArrayList;
@@ -56,54 +58,75 @@
* the same CD, reducing metadata bloat that can accumulate during replication.
*/
public final class ColumnDeduplicator {
+ public static final String COLUMN_DEDUPLICATOR_MARKER =
"__metatool_column_deduplication__";
private final RawStore store;
private final PersistenceManager pm;
private final AtomicReference<String> progress;
private final boolean isDryRun;
private final boolean isVerbose;
+ private final boolean clearMarker;
public ColumnDeduplicator(RawStoreBundle bundle, AtomicReference<String>
progress,
- boolean isDryRun, boolean isVerbose) {
+ boolean isDryRun, boolean isVerbose, boolean clearMarker) {
this.store = bundle.getBaseStore();
this.pm = bundle.getPersistentManager();
this.progress = progress;
this.isDryRun = isDryRun;
this.isVerbose = isVerbose;
+ this.clearMarker = clearMarker;
}
MetaToolObjectStore.DedupColumnsResult run(String catalogFilter, String
dbFilter, String tableFilter) {
- List<TableInfo> tables = findPartitionedTables(catalogFilter, dbFilter,
tableFilter);
+ List<Long> tables = findPartitionedTables(catalogFilter, dbFilter,
tableFilter);
MetaToolObjectStore.DedupColumnsResult result = new
MetaToolObjectStore.DedupColumnsResult(tables.size());
long start = System.currentTimeMillis();
for (int i = 0; i < tables.size() && result.getException() == null; i++) {
boolean committed = false;
- TableInfo table = tables.get(i);
+ long tableId = tables.get(i);
PostCommitValidation validation = null;
+ TableInfo table = null;
store.openTransaction();
try {
Deadline.checkTimeout();
- validation = deduplicateTable(table, result);
+ table = getTable(tableId);
+ // This target table might be dropped or deduplicated, ignore
+ // If this is the stage to clean up the marker, ignore
+ if (table != null) {
+ validation = deduplicateCols(table, result);
+ if (validation == null) {
+ // Nothing has been changed that requires a post-commit
validation, mark the table right away.
+ // Otherwise the marker is added once the validation succeeds.
+ markProcessed(tableId);
+ }
+ }
committed = store.commitTransaction();
} catch (Exception ex) {
+ System.err.println("Failed to deduplicate the column in this table: "
+ (table == null ? tableId : table));
result.catchException(ex);
} finally {
if (!committed) {
store.rollbackTransaction();
- if (result.getException() == null) {
- result.catchException(
- new MetaException("Failed to apply column descriptor
de-duplication updates for table " + table));
- }
}
}
+ boolean success = false;
+ store.openTransaction();
try {
if (committed && validation != null) {
validateTableAfterDedup(table, validation.schemaSnapshots());
result.incrementTablesValidated();
+ // Only a table that passed the validation can be skipped by later
runs
+ markProcessed(tableId);
}
+ success = store.commitTransaction();
} catch (Exception e) {
- result.catchException(new IllegalStateException("Failed to validate
the column deduplication for this table: " + table, e));
+ result.catchException(new IllegalStateException(
+ "Failed to validate the column deduplication or mark this table: "
+ table, e));
+ } finally {
+ if (!success) {
+ store.rollbackTransaction();
+ }
}
if (progress != null) {
@@ -119,7 +142,56 @@ MetaToolObjectStore.DedupColumnsResult run(String
catalogFilter, String dbFilter
return result;
}
- private PostCommitValidation deduplicateTable(TableInfo table,
MetaToolObjectStore.DedupColumnsResult result)
+ public TableInfo getTable(long tableId) {
+ final MTable table;
+ try {
+ table = pm.getObjectById(MTable.class, new MTable.PK(tableId));
+ } catch (JDOObjectNotFoundException e) {
+ // This table might have been dropped since the beginning
+ return null;
+ }
+ Map<String, String> params = table.getParameters() == null ?
+ new HashMap<>() : table.getParameters();
+ if (clearMarker) {
+ // Clean-up only mode: never add the marker or deduplicate anything, and
don't touch anything in a dry run
+ if (!isDryRun && params.containsKey(COLUMN_DEDUPLICATOR_MARKER)) {
+ params.remove(COLUMN_DEDUPLICATOR_MARKER);
+ table.setParameters(params);
+ }
+ return null;
+ }
+ if (params.containsKey(COLUMN_DEDUPLICATOR_MARKER)) {
+ // This table has been processed before
+ return null;
+ }
+
+ pm.retrieve(table);
+ pm.retrieve(table.getSd().getCD());
+ pm.retrieve(table.getDatabase());
+ return new TableInfo(tableId, table.getSd().getCD().getId(),
+ new TableName(table.getDatabase().getCatalogName(),
table.getDatabase().getName(), table.getTableName()));
+ }
+
+ /**
+ * Marks the table as processed, so later runs skip it until the marker is
cleared. Must be called within a
+ * transaction. It is a no-op in a dry run.
+ */
+ private void markProcessed(long tableId) {
+ if (isDryRun || clearMarker) {
+ return;
+ }
+ try {
+ MTable table = pm.getObjectById(MTable.class, new MTable.PK(tableId));
+ Map<String, String> params = table.getParameters() == null ?
+ new HashMap<>() : table.getParameters();
+ params.put(COLUMN_DEDUPLICATOR_MARKER,
String.valueOf(System.currentTimeMillis() / 1000));
+ table.setParameters(params);
+ } catch (JDOObjectNotFoundException e) {
+ // This table might have been dropped in the meantime, nothing to mark
+ }
+ }
+
+ private PostCommitValidation deduplicateCols(TableInfo table,
MetaToolObjectStore.DedupColumnsResult result)
throws MetaException {
List<PartitionSdInfo> partitionSds =
loadPartitionStorageDescriptors(table.tableId);
if (partitionSds.isEmpty()) {
@@ -153,8 +225,8 @@ private PostCommitValidation deduplicateTable(TableInfo
table, MetaToolObjectSto
if (isVerbose) {
PartitionSdInfo partSd = update.getKey();
long newCdId = update.getValue();
- result.addDetail(String.format("table %s.%s.%s: partition %d CD %d ->
%d",
- table.catalogName, table.dbName, table.tableName, partSd.partId,
partSd.cdId, newCdId));
+ result.addDetail(String.format("table %s: partition %d CD %d -> %d",
+ table.tableName, partSd.partId, partSd.cdId, newCdId));
}
}
if (isDryRun) {
@@ -213,7 +285,7 @@ private long pickCanonicalCdId(Set<Long> group, long
tableCdId, List<PartitionSd
.orElse(group.iterator().next());
}
- private List<TableInfo> findPartitionedTables(String catalogFilter, String
dbFilter, String tableFilter) {
+ private List<Long> findPartitionedTables(String catalogFilter, String
dbFilter, String tableFilter) {
StringBuilder filter = new StringBuilder();
List<String> parameterVals = new ArrayList<>();
if (!isEmpty(catalogFilter)) {
@@ -229,24 +301,12 @@ private List<TableInfo> findPartitionedTables(String
catalogFilter, String dbFil
Query query = !filter.isEmpty() ?
pm.newQuery(MPartition.class, filter.toString()) :
pm.newQuery(MPartition.class);
- query.setResult("DISTINCT this.table");
+ query.setResult("DISTINCT this.table.id");
boolean success = false;
- List<TableInfo> tables = new ArrayList<>();
+ List<Long> tables = new ArrayList<>();
store.openTransaction();
try {
- List<MTable> mTables = (List<MTable>)
query.executeWithArray(parameterVals.toArray(new String[0]));
- pm.retrieveAll(mTables);
- for (MTable mTable : mTables) {
- pm.retrieve(mTable.getDatabase());
- pm.retrieve(mTable.getSd());
- pm.retrieve(mTable.getSd().getCD());
- tables.add(new TableInfo(
- mTable.getId(),
- mTable.getSd().getCD().getId(),
- mTable.getDatabase().getCatalogName(),
- mTable.getDatabase().getName(),
- mTable.getTableName()));
- }
+ tables.addAll((List<Long>)
query.executeWithArray(parameterVals.toArray(new String[0])));
success = store.commitTransaction();
} finally {
query.closeAll();
@@ -273,7 +333,7 @@ private List<PartitionSdInfo>
loadPartitionStorageDescriptors(long tableId) thro
if (sd == null) {
continue;
}
- pm.retrieve(sd);
+ pm.retrieve(sd.getCD());
partitionSds.add(new PartitionSdInfo(partId, sd, sd.getCD().getId()));
}
} finally {
@@ -418,11 +478,11 @@ private void removeConstraintsForCd(MColumnDescriptor cd)
{
}
}
- public record TableInfo(long tableId, long tableCdId, String catalogName,
String dbName, String tableName) {
+ public record TableInfo(long tableId, long tableCdId, TableName tableName) {
@Override
public String toString() {
- return catalogName + "." + dbName + "." + tableName;
- }
+ return tableName.toString();
+ }
}
private record PartitionSdInfo(long partId, MStorageDescriptor sd, long
cdId) {
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/MetaToolObjectStore.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/MetaToolObjectStore.java
index 5c5a6124270..4322c5f80b5 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/MetaToolObjectStore.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/MetaToolObjectStore.java
@@ -1035,8 +1035,8 @@ public Exception getException() {
}
public DedupColumnsResult dedupColumns(String catalogFilter, String
dbFilter, String tableFilter,
- AtomicReference<String> progress, boolean isDryRun, boolean isVerbose) {
- return new ColumnDeduplicator(this.createRawStoreBundle(), progress,
isDryRun, isVerbose)
+ AtomicReference<String> progress, boolean isDryRun, boolean isVerbose,
boolean clearMarker) {
+ return new ColumnDeduplicator(this.createRawStoreBundle(), progress,
isDryRun, isVerbose, clearMarker)
.run(catalogFilter, dbFilter, tableFilter);
}
}
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/HiveMetaTool.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/HiveMetaTool.java
index 2ab17c67edc..6e615628354 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/HiveMetaTool.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/HiveMetaTool.java
@@ -81,6 +81,7 @@ public static void main(String[] args) {
execute(args);
} catch (Exception e) {
status = -1;
+ e.printStackTrace(System.err);
LOGGER.error("Exception occured", e);
} finally {
ExitUtil.terminate(status);
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/HiveMetaToolCommandLine.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/HiveMetaToolCommandLine.java
index 5bec7e3bd97..5d80e97d45e 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/HiveMetaToolCommandLine.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/HiveMetaToolCommandLine.java
@@ -93,7 +93,10 @@ class HiveMetaToolCommandLine {
.hasOptionalArgs(3)
.withDescription("De-duplicate column descriptors for partitioned tables
to reduce metadata bloat. "
+ "Optional catalog, database, and table filters support *
wildcards. Use -dryRun to preview changes, "
- + "-verbose to print per-partition details, and -timeout to set a
timeout in seconds. "
+ + "-verbose to print per-partition details, -clearMarker to drop the
marker that has been added in previous "
+ + "runs (cannot be combined with -dryRun), and -timeout to set a
timeout in seconds. "
+ + "Live runs mark each successfully processed table with the table
property "
+ + "__metatool_column_deduplication__, and marked tables are skipped
on re-run until -clearMarker is used. "
+ "Live runs validate committed state (partition column schemas
unchanged, canonical CD IDs applied) "
+ "and print a post-commit validation summary.")
.create("dedupColumns");
@@ -102,6 +105,11 @@ class HiveMetaToolCommandLine {
.withDescription("Print per-partition de-duplication details. Valid only
with -dedupColumns.")
.create("verbose");
+ private static final Option CLEAR_MARKER = OptionBuilder
+ .withDescription("Clear the column de-duplication table marker made by
previous runs, no de-duplication "
+ + "is performed. Valid only with -dedupColumns, and not with
-dryRun.")
+ .create("clearMarker");
+
@SuppressWarnings("static-access")
private static final Option TIMEOUT = OptionBuilder
.withArgName("timeout-seconds")
@@ -153,6 +161,7 @@ class HiveMetaToolCommandLine {
OPTIONS.addOption(METADATA_SUMMARY);
OPTIONS.addOption(DEDUP_COLUMNS);
OPTIONS.addOption(VERBOSE);
+ OPTIONS.addOption(CLEAR_MARKER);
OPTIONS.addOption(TIMEOUT);
}
@@ -163,6 +172,7 @@ class HiveMetaToolCommandLine {
private String[] diffExtTblLocsParams;
private boolean dryRun;
private boolean verbose;
+ private boolean clearMarker;
private Long dedupColumnsTimeoutSeconds;
private String serdePropKey;
private String tablePropKey;
@@ -204,6 +214,7 @@ private void parseCommandLine(String[] args) throws
ParseException {
diffExtTblLocsParams = cl.getOptionValues(DIFF_EXT_TBL_LOCS.getOpt());
dryRun = cl.hasOption(DRY_RUN.getOpt());
verbose = cl.hasOption(VERBOSE.getOpt());
+ clearMarker = cl.hasOption(CLEAR_MARKER.getOpt());
if (cl.hasOption(TIMEOUT.getOpt())) {
dedupColumnsTimeoutSeconds =
Long.parseLong(cl.getOptionValue(TIMEOUT.getOpt()));
if (dedupColumnsTimeoutSeconds <= 0) {
@@ -260,6 +271,14 @@ private void parseCommandLine(String[] args) throws
ParseException {
throw new IllegalArgumentException("-timeout may be used only for the
-dedupColumns command");
}
+ if (clearMarker && !isDedupColumns()) {
+ throw new IllegalArgumentException("-clearMarker may be used only for
the -dedupColumns command");
+ }
+
+ if (clearMarker && dryRun) {
+ throw new IllegalArgumentException("-clearMarker cannot be used together
with -dryRun");
+ }
+
if (metadataSummaryParams != null && metadataSummaryParams.length < 1) {
throw new IllegalArgumentException("HiveMetaTool:metadataSummary takes
in 1 required arguments but was passed " +
metadataSummaryParams.length + " arguments");
@@ -288,6 +307,7 @@ private void printArguments() {
"\tdiffExtTblLocs: " + Arrays.toString(diffExtTblLocsParams) + "\n" +
"\tdryRun : " + dryRun + "\n" +
"\tverbose : " + verbose + "\n" +
+ "\tclearMarker : " + clearMarker + "\n" +
"\ttimeout : " + dedupColumnsTimeoutSeconds + "\n" +
"\tserdePropKey : " + serdePropKey + "\n" +
"\ttablePropKey : " + tablePropKey + "\n" +
@@ -339,6 +359,10 @@ boolean isVerbose() {
return verbose;
}
+ boolean isClearMarker() {
+ return clearMarker;
+ }
+
Long getDedupColumnsTimeoutSeconds() {
return dedupColumnsTimeoutSeconds;
}
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/MetaToolTaskDedupColumns.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/MetaToolTaskDedupColumns.java
index f86c7b9480c..caf8824b756 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/MetaToolTaskDedupColumns.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/MetaToolTaskDedupColumns.java
@@ -27,6 +27,7 @@
import org.apache.hadoop.hive.metastore.DeadlineException;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.tools.ColumnDeduplicator;
import org.apache.hadoop.hive.metastore.tools.MetaToolObjectStore;
class MetaToolTaskDedupColumns extends MetaToolTask {
@@ -41,6 +42,7 @@ void execute() {
Thread daemon = null;
MetaToolObjectStore.DedupColumnsResult result = null;
+ Exception failure = null;
AtomicReference<String> progress = new AtomicReference<>();
AtomicBoolean stopped = new AtomicBoolean(false);
long timeoutMs = getCl().getDedupColumnsTimeoutSeconds() != null ?
@@ -69,10 +71,11 @@ void execute() {
daemon.setDaemon(true);
daemon.start();
}
- result = getObjectStore().dedupColumns(catalogFilter, dbFilter,
tableFilter, progress, isDryRun, isVerbose);
- } catch (MetaException ex) {
- // Deadline.startTimer throws MetaException, this shouldn't happen as
the Deadline has been registered.
- System.err.println(ex.getMessage());
+ result = getObjectStore().dedupColumns(catalogFilter, dbFilter,
tableFilter, progress, isDryRun, isVerbose,
+ getCl().isClearMarker());
+ } catch (Exception ex) {
+ // Remember the failure and rethrow it once the clean-up is done, so
that the tool exits with a failure status
+ failure = ex;
} finally {
if (daemon != null) {
stopped.set(true);
@@ -86,12 +89,23 @@ void execute() {
}
}
+ if (failure != null) {
+ throw new IllegalStateException("HiveMetaTool: failed to run
-dedupColumns", failure);
+ }
+
if (result != null) {
printSummary(result, isDryRun, isVerbose);
if (result.getException() != null) {
- if (result.getException().getCause() instanceof DeadlineException
deadline) {
- // Just print the timeout message
- System.err.println(deadline.getMessage());
+ DeadlineException deadline = findDeadlineCause(result.getException());
+ if (deadline != null) {
+ long timeoutSeconds = Math.max(1, timeoutMs / 1000);
+ System.err.println("HiveMetaTool -dedupColumns timed out after " +
timeoutSeconds
+ + " second(s). The configured limit is -timeout <seconds> or "
+ + MetastoreConf.ConfVars.DEDUP_COLUMNS_TIMEOUT.getVarname() + "
(default in minutes).");
+ System.err.println("Increase the limit with -timeout <seconds> or
set "
+ + MetastoreConf.ConfVars.DEDUP_COLUMNS_TIMEOUT.getVarname()
+ + " to a larger value (minutes), then re-run. Tables already
processed can be skipped if marked "
+ + "with " + ColumnDeduplicator.COLUMN_DEDUPLICATOR_MARKER + ".");
return;
}
throw new IllegalStateException("HiveMetaTool: failed to de-duplicate
column descriptors for all tables",
@@ -100,6 +114,15 @@ void execute() {
}
}
+ private static DeadlineException findDeadlineCause(Throwable ex) {
+ for (Throwable t = ex; t != null; t = t.getCause()) {
+ if (t instanceof DeadlineException) {
+ return (DeadlineException) t;
+ }
+ }
+ return null;
+ }
+
private void printSummary(MetaToolObjectStore.DedupColumnsResult result,
boolean isDryRun, boolean isVerbose) {
System.out.println(isDryRun ?
"Dry run of -dedupColumns.." :
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHMSColumnDescriptorReuse.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHMSColumnDescriptorReuse.java
index 0f63bda5f80..886574e2106 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHMSColumnDescriptorReuse.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHMSColumnDescriptorReuse.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.metastore;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.TableName;
import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -37,6 +38,7 @@
import org.apache.hadoop.hive.metastore.tools.ColumnDeduplicator;
import org.apache.hadoop.hive.metastore.tools.MetaToolObjectStore;
import org.apache.hadoop.hive.metastore.utils.TestTxnDbUtil;
+
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -54,7 +56,9 @@
import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
@@ -98,6 +102,14 @@ public void setUp() throws Exception {
@After
public void tearDown() {
+ if (objectStore != null) {
+ objectStore.shutdown();
+ }
+ // The PersistenceManagerFactory and its connection pools are static and
survive across tests,
+ // as the JDBC url is the same. Close them before dropping the in-memory
Derby database, otherwise
+ // the pool silently re-creates an empty database (create=true) that races
with the next schema init.
+ PersistenceManagerProvider.closePmfIfNeeded(false, true);
+ PersistenceManagerProvider.closePmfIfNeeded(true, true);
db.after();
}
@@ -198,7 +210,7 @@ public void testDeduplicateColumnDescriptorsTool() throws
Exception {
Deadline.startTimer("testDeduplicateColumnDescriptorsTool");
try {
MetaToolObjectStore.DedupColumnsResult result =
metaToolStore.dedupColumns(null, "default", "person", progress,
- false, false);
+ false, false, false);
assertEquals(0, result.getTablesWithDuplicates());
assertEquals(0, result.getTablesValidated());
@@ -211,12 +223,31 @@ public void testDeduplicateColumnDescriptorsTool() throws
Exception {
objectStore.addPartition(newPart(tbl2, "England"));
cdsBeforeDedup = countColumnDescriptors();
assertTrue(cdsBeforeDedup > 2);
+ Table table = objectStore.getTable("hive", "default", "person");
+
assertFalse(table.getParameters().containsKey(ColumnDeduplicator.COLUMN_DEDUPLICATOR_MARKER));
- result = metaToolStore.dedupColumns(null, "default", "person", progress,
false, false);
+ result = metaToolStore.dedupColumns(null, "default", "person", progress,
false, false, false);
assertTrue(result.getStorageDescriptorsUpdated() > 0);
assertEquals(1, result.getTablesValidated());
assertEquals(2, countColumnDescriptors());
assertNotNull(progress.get());
+ table = objectStore.getTable("hive", "default", "person");
+
assertTrue(table.getParameters().containsKey(ColumnDeduplicator.COLUMN_DEDUPLICATOR_MARKER));
+
+ result = metaToolStore.dedupColumns(null, "default", "person", progress,
false, false, false);
+ assertEquals(0, result.getStorageDescriptorsUpdated());
+ assertEquals(0, result.getTablesValidated());
+
+ result = metaToolStore.dedupColumns(null, "default", "person", progress,
false, false, true);
+ assertEquals(0, result.getStorageDescriptorsUpdated());
+ table = objectStore.getTable("hive", "default", "person");
+ // the marker has been cleaned
+
assertFalse(table.getParameters().containsKey(ColumnDeduplicator.COLUMN_DEDUPLICATOR_MARKER));
+
+ result = metaToolStore.dedupColumns(null, "default", "person", progress,
false, false, true);
+ assertEquals(0, result.getStorageDescriptorsUpdated());
+ table = objectStore.getTable("hive", "default", "person");
+
assertFalse(table.getParameters().containsKey(ColumnDeduplicator.COLUMN_DEDUPLICATOR_MARKER));
Deadline.registerIfNot(30 * 1000);
Deadline.startTimer("testDeduplicateColumnDescriptorsTool");
@@ -236,6 +267,85 @@ public void testDeduplicateColumnDescriptorsTool() throws
Exception {
}
}
+ @Test
+ public void testColDupGetTable() throws Exception {
+ FieldSchema id = new FieldSchema("id", ColumnType.STRING_TYPE_NAME, "");
+ FieldSchema fname = new FieldSchema("fname", ColumnType.STRING_TYPE_NAME,
"");
+ FieldSchema country = new FieldSchema("country",
ColumnType.STRING_TYPE_NAME, "");
+ Table tbl = newTable(Arrays.asList(id, fname),
Collections.singletonList(country));
+ objectStore.createTable(tbl);
+ long tableId = objectStore.getTable("hive", "default", "person").getId();
+
+ MetaToolObjectStore metaToolStore = new MetaToolObjectStore();
+ metaToolStore.setConf(conf);
+ ColumnDeduplicator deduplicator =
+ new ColumnDeduplicator(metaToolStore.createRawStoreBundle(), null,
false, false, false);
+ assertNull(getTableInTrx(metaToolStore, deduplicator, 9_999_999_999L));
+ ColumnDeduplicator.TableInfo tableInfo = getTableInTrx(metaToolStore,
deduplicator, tableId);
+ assertNotNull(tableInfo);
+ assertEquals("person", tableInfo.tableName().getTable());
+ // Looking up the table doesn't mark it, the marker is only added once the
table has been processed
+ tbl = objectStore.getTable("hive", "default", "person");
+
assertFalse(tbl.getParameters().containsKey(ColumnDeduplicator.COLUMN_DEDUPLICATOR_MARKER));
+
+ Table newTable = newTable(Arrays.asList(id, fname),
Collections.singletonList(country));
+ newTable.setTableName("person1");
+ objectStore.createTable(newTable);
+ long person1TableId = objectStore.getTable("hive", "default",
"person1").getId();
+
+ // Process the table "person", it has no duplicates but should be marked,
and be skipped afterwards
+ objectStore.addPartition(newPart(tbl, "US"));
+ Deadline.registerIfNot(30 * 1000);
+ Deadline.startTimer("testColDupGetTable");
+ try {
+ MetaToolObjectStore.DedupColumnsResult result =
+ metaToolStore.dedupColumns(null, "default", "person", new
AtomicReference<>(), false, false, false);
+ assertNull(result.getException());
+ assertEquals(1, result.getTablesScanned());
+ } finally {
+ Deadline.stopTimer();
+ }
+ tbl = objectStore.getTable("hive", "default", "person");
+
assertTrue(tbl.getParameters().containsKey(ColumnDeduplicator.COLUMN_DEDUPLICATOR_MARKER));
+ assertNull(getTableInTrx(metaToolStore, deduplicator, tableId));
+
+ // A dry run of the clean-up shouldn't remove the marker
+ ColumnDeduplicator dryRunCleaner =
+ new ColumnDeduplicator(metaToolStore.createRawStoreBundle(), null,
true, false, true);
+ assertNull(getTableInTrx(metaToolStore, dryRunCleaner, tableId));
+ tbl = objectStore.getTable("hive", "default", "person");
+
assertTrue(tbl.getParameters().containsKey(ColumnDeduplicator.COLUMN_DEDUPLICATOR_MARKER));
+
+ // Marker should be cleared
+ deduplicator =
+ new ColumnDeduplicator(metaToolStore.createRawStoreBundle(), null,
false, false, true);
+ assertNull(getTableInTrx(metaToolStore, deduplicator, tableId));
+ tbl = objectStore.getTable("hive", "default", "person");
+
assertFalse(tbl.getParameters().containsKey(ColumnDeduplicator.COLUMN_DEDUPLICATOR_MARKER));
+
+ // A table without the deduplication marker shouldn't be affected
+ newTable = objectStore.getTable("hive", "default", "person1");
+
assertFalse(newTable.getParameters().containsKey(ColumnDeduplicator.COLUMN_DEDUPLICATOR_MARKER));
+ assertNull(getTableInTrx(metaToolStore, deduplicator, person1TableId));
+ newTable = objectStore.getTable("hive", "default", "person1");
+
assertFalse(newTable.getParameters().containsKey(ColumnDeduplicator.COLUMN_DEDUPLICATOR_MARKER));
+ }
+
+ private static ColumnDeduplicator.TableInfo
getTableInTrx(MetaToolObjectStore store,
+ ColumnDeduplicator deduplicator, long tableId) {
+ boolean committed = false;
+ store.openTransaction();
+ try {
+ ColumnDeduplicator.TableInfo tableInfo = deduplicator.getTable(tableId);
+ committed = store.commitTransaction();
+ return tableInfo;
+ } finally {
+ if (!committed) {
+ store.rollbackTransaction();
+ }
+ }
+ }
+
@Test
public void testValidateTableAfterDedupReloadsPartitionById() throws
Exception {
MetastoreConf.setBoolVar(conf,
MetastoreConf.ConfVars.PARTITION_REUSE_COLUMN_DESCRIPTORS, false);
@@ -332,8 +442,8 @@ private long lookupPartitionCdId(long partId) throws
Exception {
private void validate(List<ColumnDeduplicator.PartitionSchemaSnapshot>
snapshots)
throws Exception {
ColumnDeduplicator deduplicator =
- new ColumnDeduplicator(objectStore.createRawStoreBundle(), null,
false, false);
- ColumnDeduplicator.TableInfo tableInfo = new
ColumnDeduplicator.TableInfo(0L, 0L, "hive", "default", "person");
+ new ColumnDeduplicator(objectStore.createRawStoreBundle(), null,
false, false, false);
+ ColumnDeduplicator.TableInfo tableInfo = new
ColumnDeduplicator.TableInfo(0L, 0L, new TableName("hive", "default",
"person"));
deduplicator.validateTableAfterDedup(tableInfo, snapshots);
}
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/metatool/TestHiveMetaToolCommandLine.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/metatool/TestHiveMetaToolCommandLine.java
index f28b177eb02..36e12b7a334 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/metatool/TestHiveMetaToolCommandLine.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/metatool/TestHiveMetaToolCommandLine.java
@@ -154,6 +154,21 @@ public void testParseDedupColumns() throws ParseException {
cl = new HiveMetaToolCommandLine(new String[] {"-dedupColumns",
"-timeout", "3600"});
assertEquals(Long.valueOf(3600L), cl.getDedupColumnsTimeoutSeconds());
+
+ cl = new HiveMetaToolCommandLine(new String[] {"-dedupColumns", "hive",
"default", "person", "-clearMarker"});
+ assertTrue(cl.isClearMarker());
+
+ cl = new HiveMetaToolCommandLine(new String[] {"-dedupColumns",
"-clearMarker"});
+ assertTrue(cl.isClearMarker());
+ assertEquals(0, cl.getDedupColumnsParams().length);
+ }
+
+ @Test
+ public void testClearMarkerWithDryRunNotAllowed() throws ParseException {
+ exception.expect(IllegalArgumentException.class);
+ exception.expectMessage("-clearMarker cannot be used together with
-dryRun");
+
+ new HiveMetaToolCommandLine(new String[] {"-dedupColumns", "-clearMarker",
"-dryRun"});
}
@Test
@@ -172,6 +187,14 @@ public void testInvalidTimeout() throws ParseException {
new HiveMetaToolCommandLine(new String[] {"-dedupColumns", "-timeout",
"0"});
}
+ @Test
+ public void testClearMarkerNotAllowed() throws ParseException {
+ exception.expect(IllegalArgumentException.class);
+ exception.expectMessage("-clearMarker may be used only for the
-dedupColumns command");
+
+ new HiveMetaToolCommandLine(new String[] {"-listFSRoot", "-clearMarker"});
+ }
+
@Test
public void testVerboseNotAllowed() throws ParseException {
exception.expect(IllegalArgumentException.class);