dengzhhu653 commented on code in PR #6450:
URL: https://github.com/apache/hive/pull/6450#discussion_r3331547275
##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java:
##########
@@ -2974,6192 +2120,1269 @@ private Partition convertToPart(String catName,
String dbName, String tblName,
return p;
}
- @Override
- public boolean dropPartition(String catName, String dbName, String
tableName, String partName)
- throws MetaException, NoSuchObjectException, InvalidObjectException,
InvalidInputException {
- boolean success = false;
- try {
- openTransaction();
- dropPartitionsInternal(catName, dbName, tableName,
Arrays.asList(partName), true, true);
- success = commitTransaction();
- } finally {
- rollbackAndCleanup(success, null);
+ public static List<Partition> convertToParts(String catName, String dbName,
String tblName,
+ List<MPartition> mparts, boolean isAcidTable, Configuration conf,
GetPartitionsArgs args)
+ throws MetaException {
+ List<Partition> parts = new ArrayList<>(mparts.size());
+ for (MPartition mp : mparts) {
+ parts.add(convertToPart(catName, dbName, tblName, mp, isAcidTable, conf,
args));
+ Deadline.checkTimeout();
}
- return success;
+ return parts;
}
- @Override
- public void dropPartitions(String catName, String dbName, String tblName,
List<String> partNames)
- throws MetaException, NoSuchObjectException {
- dropPartitionsInternal(catName, dbName, tblName, partNames, true, true);
+ public static Pair<Query, Map<String, String>> getPartQueryWithParams(
+ PersistenceManager pm,
+ String catName, String dbName, String tblName,
+ List<String> partNames) {
+ Query query = pm.newQuery();
+ Map<String, String> params = new HashMap<>();
+ String filterStr = getJDOFilterStrForPartitionNames(catName, dbName,
tblName, partNames, params);
+ query.setFilter(filterStr);
+ LOG.debug(" JDOQL filter is {}", filterStr);
+ query.declareParameters(makeParameterDeclarationString(params));
+ return Pair.of(query, params);
}
- @VisibleForTesting
- void dropPartitionsInternal(String catName, String dbName, String tblName,
- List<String> partNames, boolean allowSql, boolean allowJdo)
- throws MetaException, NoSuchObjectException {
- if (CollectionUtils.isEmpty(partNames)) {
- return;
+ public static String getJDOFilterStrForPartitionNames(String catName, String
dbName, String tblName,
+ List<String> partNames, Map params) {
+ StringBuilder sb = new StringBuilder(
+ "table.tableName == t1 && table.database.name == t2 &&" + "
table.database.catalogName == t3 && (");
+ params.put("t1", normalizeIdentifier(tblName));
+ params.put("t2", normalizeIdentifier(dbName));
+ params.put("t3", normalizeIdentifier(catName));
+ int n = 0;
+ for (Iterator<String> itr = partNames.iterator(); itr.hasNext(); ) {
+ String pn = "p" + n;
+ n++;
+ String part = itr.next();
+ params.put(pn, part);
+ sb.append("partitionName == ").append(pn);
+ sb.append(" || ");
}
- new GetListHelper<Void>(catName, dbName, tblName, allowSql, allowJdo) {
- @Override
- protected List<Void> getSqlResult(GetHelper<List<Void>> ctx) throws
MetaException {
- directSql.dropPartitionsViaSqlFilter(catName, dbName, tblName,
partNames);
- return Collections.emptyList();
- }
- @Override
- protected List<Void> getJdoResult(GetHelper<List<Void>> ctx) throws
MetaException {
- dropPartitionsViaJdo(catName, dbName, tblName, partNames, new
AtomicReference<>());
- return Collections.emptyList();
- }
- }.run(false);
+ sb.setLength(sb.length() - 4); // remove the last " || "
+ sb.append(')');
+ return sb.toString();
}
- private void dropPartitionsViaJdo(String catName, String dbName, String
tblName,
- List<String> partNames, AtomicReference<String> message) throws
MetaException {
- boolean success = false;
-
- if (partNames.isEmpty()) {
- return;
- }
- openTransaction();
-
- int batch = batchSize == NO_BATCHING ? 1 : (partNames.size() + batchSize)
/ batchSize;
- AtomicLong batchIdx = new AtomicLong(1);
- AtomicLong timeSpent = new AtomicLong(0);
- try {
- Batchable.runBatched(batchSize, partNames, new Batchable<String, Void>()
{
- @Override
- public List<Void> run(List<String> input) throws MetaException {
- StringBuilder progress = new StringBuilder("Dropping partitions,
batch: ");
- long start = System.currentTimeMillis();
- progress.append(batchIdx.get()).append("/").append(batch);
- if (batchIdx.get() > 1) {
- long leftTime = (batch - batchIdx.get()) * timeSpent.get() /
batchIdx.get();
- progress.append(", time left: ").append(leftTime).append("ms");
- }
- message.set(progress.toString());
- // Delete all things.
- dropPartitionGrantsNoTxn(catName, dbName, tblName, input);
- dropPartitionAllColumnGrantsNoTxn(catName, dbName, tblName, input);
- dropPartitionColumnStatisticsNoTxn(catName, dbName, tblName, input);
-
- // CDs are reused; go try partition SDs, detach all CDs from SDs,
then remove unused CDs.
- for (MColumnDescriptor mcd : detachCdsFromSdsNoTxn(catName, dbName,
tblName, input)) {
- removeUnusedColumnDescriptor(mcd);
- }
- dropPartitionsNoTxn(catName, dbName, tblName, input);
- timeSpent.addAndGet(System.currentTimeMillis() - start);
- batchIdx.incrementAndGet();
- return Collections.emptyList();
- }
- });
-
- if (!(success = commitTransaction())) {
- throw new MetaException("Failed to drop partitions");
- }
- } finally {
- rollbackAndCleanup(success, null);
+ public static String makeParameterDeclarationString(Map<String, String>
params) {
+ //Create the parameter declaration string
+ StringBuilder paramDecl = new StringBuilder();
+ for (String key : params.keySet()) {
+ paramDecl.append(", java.lang.String ").append(key);
}
+ return paramDecl.toString();
}
- @Override
- public List<Partition> getPartitions(String catName, String dbName, String
tableName,
- GetPartitionsArgs args) throws MetaException, NoSuchObjectException {
- List<Partition> results = Collections.emptyList();
- boolean success = false;
-
- LOG.debug("Executing getPartitions");
+ /** Helper class for getting stuff w/transaction, direct SQL, perf logging,
etc. */
+ @VisibleForTesting
+ public abstract class GetHelper<T> {
+ private final boolean isInTxn, doTrace, allowJdo;
+ private boolean doUseDirectSql;
+ private long start;
+ private Table table;
+ protected final List<String> partitionFields;
+ protected final String catName, dbName, tblName;
+ private boolean success = false;
+ protected T results = null;
- try {
- openTransaction();
- results = getPartitionsInternal(catName, dbName, tableName, true, true,
args);
- success = commitTransaction();
- } finally {
- rollbackAndCleanup(success, null);
+ public GetHelper(String catalogName, String dbName, String tblName,
+ boolean allowSql, boolean allowJdo) throws MetaException {
+ this(catalogName, dbName, tblName, null, allowSql, allowJdo);
}
- return results;
- }
- @Override
- public Map<String, String> getPartitionLocations(String catName, String
dbName, String tblName,
- String baseLocationToNotShow, int max) {
- catName = normalizeIdentifier(catName);
- dbName = normalizeIdentifier(dbName);
- tblName = normalizeIdentifier(tblName);
+ public GetHelper(String catalogName, String dbName, String tblName,
+ List<String> fields, boolean allowSql, boolean allowJdo) throws
MetaException {
+ assert allowSql || allowJdo;
+ this.allowJdo = allowJdo;
+ this.catName = (catalogName != null) ? normalizeIdentifier(catalogName)
: null;
+ this.dbName = (dbName != null) ? normalizeIdentifier(dbName) : null;
+ this.partitionFields = fields;
+ if (tblName != null) {
+ this.tblName = normalizeIdentifier(tblName);
+ } else {
+ // tblName can be null in cases of Helper being used at a higher
+ // abstraction level, such as with datbases
+ this.tblName = null;
+ this.table = null;
+ }
+ this.doTrace = LOG.isDebugEnabled();
+ this.isInTxn = isActiveTransaction();
- boolean success = false;
- Query query = null;
- Map<String, String> partLocations = new HashMap<>();
- try {
- openTransaction();
- LOG.debug("Executing getPartitionLocations");
-
- query = pm.newQuery(MPartition.class);
- query.setFilter("this.table.database.catalogName == t1 &&
this.table.database.name == t2 "
- + "&& this.table.tableName == t3");
- query.declareParameters("String t1, String t2, String t3");
- query.setResult("this.partitionName, this.sd.location");
- if (max >= 0) {
- //Row limit specified, set it on the Query
- query.setRange(0, max);
- }
-
- List<Object[]> result = (List<Object[]>)query.execute(catName, dbName,
tblName);
- for(Object[] row:result) {
- String location = (String)row[1];
- if (baseLocationToNotShow != null && location != null
- && FileUtils.isSubdirectory(baseLocationToNotShow, location)) {
- location = null;
- }
- partLocations.put((String)row[0], location);
+ boolean isConfigEnabled = MetastoreConf.getBoolVar(getConf(),
ConfVars.TRY_DIRECT_SQL);
+ if (isConfigEnabled && directSql == null) {
+ directSql = new MetaStoreDirectSql(pm, getConf(), "");
}
- LOG.debug("Done executing query for getPartitionLocations");
- success = commitTransaction();
- } finally {
- rollbackAndCleanup(success, query);
- }
- return partLocations;
- }
- protected List<Partition> getPartitionsInternal(String catName, String
dbName, String tblName,
- boolean allowSql, boolean allowJdo, GetPartitionsArgs args)
- throws MetaException, NoSuchObjectException {
- return new GetListHelper<Partition>(catName, dbName, tblName, allowSql,
allowJdo) {
- @Override
- protected List<Partition> getSqlResult(GetHelper<List<Partition>> ctx)
throws MetaException {
- return directSql.getPartitions(catName, dbName, tblName, args);
+ if (!allowJdo && isConfigEnabled && !directSql.isCompatibleDatastore()) {
+ throw new MetaException("SQL is not operational"); // test path; SQL
is enabled and broken.
}
- @Override
- protected List<Partition> getJdoResult(GetHelper<List<Partition>> ctx)
throws MetaException {
- try {
- return convertToParts(catName, dbName, tblName,
- listMPartitions(catName, dbName, tblName, args.getMax()), false,
args);
- } catch (Exception e) {
- LOG.error("Failed to convert to parts", e);
- throw new MetaException(e.getMessage());
- }
- }
- }.run(false);
- }
-
- @Override
- public Partition getPartitionWithAuth(String catName, String dbName, String
tblName,
- List<String> partVals, String user_name, List<String> group_names)
- throws NoSuchObjectException, MetaException, InvalidObjectException {
- boolean success = false;
- try {
- openTransaction();
- MPartition mpart = getMPartition(catName, dbName, tblName, partVals,
null);
- if (mpart == null) {
- commitTransaction();
- throw new NoSuchObjectException("partition values="
- + partVals.toString());
- }
- MTable mtbl = mpart.getTable();
-
- Partition part = convertToPart(catName, dbName, tblName, mpart,
TxnUtils.isAcidTable(mtbl.getParameters()));
- if
("TRUE".equalsIgnoreCase(mtbl.getParameters().get("PARTITION_LEVEL_PRIVILEGE")))
{
- String partName =
Warehouse.makePartName(this.convertToFieldSchemas(mtbl
- .getPartitionKeys()), partVals);
- PrincipalPrivilegeSet partAuth =
this.getPartitionPrivilegeSet(catName, dbName,
- tblName, partName, user_name, group_names);
- part.setPrivileges(partAuth);
- }
-
- success = commitTransaction();
- return part;
- } finally {
- rollbackAndCleanup(success, null);
- }
- }
-
- private List<Partition> convertToParts(String catName, String dbName, String
tblName,
- List<MPartition> mparts, boolean isAcidTable, GetPartitionsArgs args)
- throws MetaException {
- List<Partition> parts = new ArrayList<>(mparts.size());
- for (MPartition mp : mparts) {
- parts.add(convertToPart(catName, dbName, tblName, mp, isAcidTable,
args));
- Deadline.checkTimeout();
+ this.doUseDirectSql = allowSql && isConfigEnabled &&
directSql.isCompatibleDatastore();
}
- return parts;
- }
- // TODO:pc implement max
- @Override
- public List<String> listPartitionNames(String catName, String dbName, String
tableName,
- short max) throws MetaException {
- List<String> pns = null;
- boolean success = false;
- try {
- openTransaction();
- LOG.debug("Executing getPartitionNames");
- pns = getPartitionNamesNoTxn(catName, dbName, tableName, max);
- success = commitTransaction();
- } finally {
- rollbackAndCleanup(success, null);
+ protected boolean canUseDirectSql(GetHelper<T> ctx) throws MetaException {
+ return true; // By default, assume we can user directSQL - that's kind
of the point.
}
- return pns;
- }
+ protected abstract String describeResult();
+ protected abstract T getSqlResult(GetHelper<T> ctx) throws MetaException;
+ protected abstract T getJdoResult(
+ GetHelper<T> ctx) throws MetaException, NoSuchObjectException,
InvalidObjectException, InvalidInputException;
- @Override
- public List<String> listPartitionNames(final String catName, final String
dbName, final String tblName,
- final String defaultPartName, final byte[] exprBytes,
- final String order, final int maxParts) throws MetaException,
NoSuchObjectException {
- final String defaultPartitionName =
getDefaultPartitionName(defaultPartName);
- final boolean isEmptyFilter = exprBytes.length == 1 && exprBytes[0] == -1;
- ExpressionTree tmp = null;
- if (!isEmptyFilter) {
- tmp = PartFilterExprUtil.makeExpressionTree(expressionProxy, exprBytes,
- getDefaultPartitionName(defaultPartName), conf);
- }
- final ExpressionTree exprTree = tmp;
- return new GetListHelper<String>(catName, dbName, tblName, true, true) {
- private List<String> getPartNamesPrunedByExpr(Table table, boolean
isJdoQuery) throws MetaException {
- int max = isEmptyFilter ? maxParts : -1;
- List<String> result;
- if (isJdoQuery) {
- result = getPartitionNamesViaOrm(catName, dbName, tblName,
ExpressionTree.EMPTY_TREE,
- order, max, true, table.getPartitionKeys());
- } else {
- SqlFilterForPushdown filter = new SqlFilterForPushdown(table, false);
- result = directSql.getPartitionNamesViaSql(filter,
table.getPartitionKeys(),
- defaultPartitionName, order, max);
- }
- if (!isEmptyFilter) {
- prunePartitionNamesByExpr(catName, dbName, tblName, result,
- new GetPartitionsArgs.GetPartitionsArgsBuilder()
-
.expr(exprBytes).defaultPartName(defaultPartName).max(maxParts).build());
- }
- return result;
- }
- @Override
- protected List<String> getSqlResult(GetHelper<List<String>> ctx) throws
MetaException {
- SqlFilterForPushdown filter = new SqlFilterForPushdown(ctx.getTable(),
false);
- List<String> partNames = null;
- Table table = ctx.getTable();
- if (exprTree != null) {
- if (directSql.generateSqlFilterForPushdown(table.getCatName(),
table.getDbName(), table.getTableName(),
- ctx.getTable().getPartitionKeys(), exprTree,
defaultPartitionName, filter)) {
- partNames = directSql.getPartitionNamesViaSql(filter,
table.getPartitionKeys(),
- defaultPartitionName, order, (int)maxParts);
- }
- }
- if (partNames == null) {
- partNames = getPartNamesPrunedByExpr(table, false);
- }
- return partNames;
- }
- @Override
- protected List<String> getJdoResult(
- GetHelper<List<String>> ctx) throws MetaException,
NoSuchObjectException {
- List<String> result = null;
- if (exprTree != null) {
+ public T run(boolean initTable) throws MetaException,
NoSuchObjectException {
+ try {
+ start(initTable);
+ String savePoint = isInTxn && allowJdo ? "rollback_" +
System.nanoTime() : null;
+ if (doUseDirectSql) {
try {
- result = getPartitionNamesViaOrm(catName, dbName, tblName,
exprTree, order,
- maxParts, true, ctx.getTable().getPartitionKeys());
- } catch (MetaException e) {
- result = null;
+ directSql.prepareTxn();
+ setTransactionSavePoint(savePoint);
+ this.results = getSqlResult(this);
+ LOG.debug("Using direct SQL optimization.");
+ } catch (Exception ex) {
+ handleDirectSqlError(ex, savePoint);
}
}
- if (result == null) {
- result = getPartNamesPrunedByExpr(ctx.getTable(), true);
+ // Note that this will be invoked in 2 cases:
+ // 1) DirectSQL was disabled to start with;
+ // 2) DirectSQL threw and was disabled in handleDirectSqlError.
+ if (!doUseDirectSql) {
+ this.results = getJdoResult(this);
+ LOG.debug("Not using direct SQL optimization.");
}
- return result;
+ return commit();
+ } catch (NoSuchObjectException | MetaException ex) {
+ throw ex;
+ } catch (Exception ex) {
+ LOG.error("", ex);
+ throw new MetaException(ex.getMessage());
+ } finally {
+ close();
}
- }.run(true);
- }
-
- @Override
- public List<String> listPartitionNamesByFilter(String catName, String
dbName, String tblName,
- GetPartitionsArgs args) throws MetaException, NoSuchObjectException {
-
- catName = normalizeIdentifier(catName);
- dbName = normalizeIdentifier(dbName);
- tblName = normalizeIdentifier(tblName);
-
- MTable mTable = ensureGetMTable(catName, dbName, tblName);
- List<FieldSchema> partitionKeys =
convertToFieldSchemas(mTable.getPartitionKeys());
- String filter = args.getFilter();
- final ExpressionTree tree = (filter != null && !filter.isEmpty())
- ? PartFilterExprUtil.parseFilterTree(filter) :
ExpressionTree.EMPTY_TREE;
- return new GetListHelper<String>(catName, dbName, tblName, true, true) {
- private final SqlFilterForPushdown filter = new SqlFilterForPushdown();
+ }
- @Override
- protected boolean canUseDirectSql(GetHelper<List<String>> ctx) throws
MetaException {
- return directSql.generateSqlFilterForPushdown(catName, dbName, tblName,
- partitionKeys, tree, null, filter);
+ private void start(boolean initTable) throws MetaException,
NoSuchObjectException {
+ start = doTrace ? System.nanoTime() : 0;
+ openTransaction();
+ if (initTable && (tblName != null)) {
+ table = ensureGetTable(catName, dbName, tblName);
}
+ doUseDirectSql = doUseDirectSql && canUseDirectSql(this);
+ }
- @Override
- protected List<String> getSqlResult(GetHelper<List<String>> ctx) throws
MetaException {
- return directSql.getPartitionNamesViaSql(filter, partitionKeys,
- getDefaultPartitionName(args.getDefaultPartName()), null,
args.getMax());
+ private void handleDirectSqlError(Exception ex, String savePoint) throws
MetaException, NoSuchObjectException {
+ String message = null;
+ try {
+ message = generateShorterMessage(ex);
+ } catch (Throwable t) {
+ message = ex.toString() + "; error building a better message: " +
t.getMessage();
}
+ LOG.warn(message); // Don't log the exception, people just get confused.
+ LOG.debug("Full DirectSQL callstack for debugging (not an error)", ex);
- @Override
- protected List<String> getJdoResult(GetHelper<List<String>> ctx)
- throws MetaException, NoSuchObjectException, InvalidObjectException {
- return getPartitionNamesViaOrm(catName, dbName, tblName, tree, null,
- args.getMax(), true, partitionKeys);
+ if (!allowJdo || !DatabaseProduct.isRecoverableException(ex)) {
+ throw ExceptionHandler.newMetaException(ex);
}
- }.run(false);
- }
+
+ if (!isInTxn) {
+ JDOException rollbackEx = null;
+ try {
+ rollbackTransaction();
+ } catch (JDOException jex) {
+ rollbackEx = jex;
+ }
+ if (rollbackEx != null) {
+ // Datanucleus propagates some pointless exceptions and rolls back
in the finally.
+ if (currentTransaction != null && currentTransaction.isActive()) {
+ throw rollbackEx; // Throw if the tx wasn't rolled back.
+ }
+ LOG.info("Ignoring exception, rollback succeeded: " +
rollbackEx.getMessage());
+ }
- private List<String> getPartitionNamesViaOrm(String catName, String dbName,
String tblName,
- ExpressionTree tree, String order, Integer maxParts, boolean
isValidatedFilter,
- List<FieldSchema> partitionKeys) throws MetaException {
- Map<String, Object> params = new HashMap<String, Object>();
- String jdoFilter = makeQueryFilterString(catName, dbName, tblName, tree,
- params, isValidatedFilter, partitionKeys);
- if (jdoFilter == null) {
- assert !isValidatedFilter;
- throw new MetaException("Failed to generate filter.");
- }
-
- try (QueryWrapper query = new QueryWrapper(pm.newQuery(
- "select partitionName from
org.apache.hadoop.hive.metastore.model.MPartition"))) {
- query.setFilter(jdoFilter);
- List<Object[]> orderSpecs = MetaStoreUtils.makeOrderSpecs(order);
- StringBuilder builder = new StringBuilder();
- for (Object[] spec : orderSpecs) {
- // TODO: order by casted value if the type of partition key is not
string
- builder.append("values.get(").append(spec[0]).append(")
").append(spec[1]).append(",");
- }
- if (builder.length() > 0) {
- builder.setLength(builder.length() - 1);
- query.setOrdering(builder.toString());
+ start = doTrace ? System.nanoTime() : 0;
+ openTransaction();
+ if (table != null) {
+ table = ensureGetTable(catName, dbName, tblName);
+ }
} else {
- query.setOrdering("partitionName ascending");
+ rollbackTransactionToSavePoint(savePoint);
+ start = doTrace ? System.nanoTime() : 0;
}
- if (maxParts > -1) {
- query.setRange(0, maxParts);
+ if (directSqlErrors != null) {
+ directSqlErrors.inc();
}
- String parameterDeclaration = makeParameterDeclarationStringObj(params);
- query.declareParameters(parameterDeclaration);
- Collection jdoRes = (Collection) query.executeWithMap(params);
- List<String> result = new LinkedList<String>();
- for (Object partName : jdoRes) {
- result.add((String) partName);
- }
- return result;
+ doUseDirectSql = false;
}
- }
-
- private String extractPartitionKey(FieldSchema key, List<FieldSchema> pkeys)
{
- StringBuilder buffer = new StringBuilder(256);
-
- assert pkeys.size() >= 1;
- String partKey = "/" + key.getName() + "=";
-
- // Table is partitioned by single key
- if (pkeys.size() == 1 && (pkeys.get(0).getName().matches(key.getName()))) {
- buffer.append("partitionName.substring(partitionName.indexOf(\"")
- .append(key.getName()).append("=\") +
").append(key.getName().length() + 1)
- .append(")");
-
- // First partition key - anything between key= and first /
- } else if ((pkeys.get(0).getName().matches(key.getName()))) {
-
- buffer.append("partitionName.substring(partitionName.indexOf(\"")
- .append(key.getName()).append("=\") +
").append(key.getName().length() + 1).append(", ")
- .append("partitionName.indexOf(\"/\")")
- .append(")");
-
- // Last partition key - anything between /key= and end
- } else if ((pkeys.get(pkeys.size() - 1).getName().matches(key.getName())))
{
- buffer.append("partitionName.substring(partitionName.indexOf(\"")
- .append(partKey).append("\") + ").append(partKey.length())
- .append(")");
+ private String generateShorterMessage(Exception ex) {
+ StringBuilder message = new StringBuilder(
+ "Falling back to ORM path due to direct SQL failure (this is not an
error): ");
+ Throwable t = ex;
+ StackTraceElement[] prevStack = null;
+ while (t != null) {
+ message.append(t.getMessage());
+ StackTraceElement[] stack = t.getStackTrace();
+ int uniqueFrames = stack.length - 1;
+ if (prevStack != null) {
+ int n = prevStack.length - 1;
+ while (uniqueFrames >= 0 && n >= 0 &&
stack[uniqueFrames].equals(prevStack[n])) {
+ uniqueFrames--; n--;
+ }
+ }
+ for (int i = 0; i <= uniqueFrames; ++i) {
+ StackTraceElement ste = stack[i];
+ message.append(" at ").append(ste);
+ if (ste.getMethodName().contains("getSqlResult")
+ && (ste.getFileName() == null ||
ste.getFileName().contains("ObjectStore"))) {
+ break;
+ }
+ }
+ prevStack = stack;
+ t = t.getCause();
+ if (t != null) {
+ message.append(";\n Caused by: ");
+ }
+ }
+ return message.toString();
+ }
- // Intermediate key - anything between /key= and the following /
- } else {
+ private T commit() {
+ success = commitTransaction();
+ if (doTrace) {
+ double time = ((System.nanoTime() - start) / 1000000.0);
+ String result = describeResult();
+ String retrieveType = doUseDirectSql ? "SQL" : "ORM";
- buffer.append("partitionName.substring(partitionName.indexOf(\"")
- .append(partKey).append("\") + ").append(partKey.length()).append(",
")
- .append("partitionName.indexOf(\"/\",
partitionName.indexOf(\"").append(partKey)
- .append("\") + 1))");
+ LOG.debug("{} retrieved using {} in {}ms", result, retrieveType, time);
+ }
+ return results;
}
- LOG.info("Query for Key:" + key.getName() + " is :" + buffer);
- return buffer.toString();
- }
-
- @Override
- public PartitionValuesResponse listPartitionValues(String catName, String
dbName,
- String tableName,
List<FieldSchema> cols,
- boolean applyDistinct,
String filter,
- boolean ascending,
List<FieldSchema> order,
- long maxParts) throws
MetaException {
- catName = normalizeIdentifier(catName);
- dbName = dbName.toLowerCase().trim();
- tableName = tableName.toLowerCase().trim();
- try {
- if (filter == null || filter.isEmpty()) {
- PartitionValuesResponse response =
getDistinctValuesForPartitionsNoTxn(catName, dbName,
- tableName, cols, applyDistinct, maxParts);
- LOG.info("Number of records fetched: {}",
response.getPartitionValues().size());
- return response;
- } else {
- PartitionValuesResponse response =
- extractPartitionNamesByFilter(catName, dbName, tableName, filter,
cols, ascending, maxParts);
- if (response.getPartitionValues() != null) {
- LOG.info("Number of records fetched with filter: {}",
response.getPartitionValues().size());
- }
- return response;
+ private void close() {
+ if (!success) {
+ rollbackTransaction();
}
- } catch (Exception t) {
- LOG.error("Exception in ORM", t);
- throw new MetaException("Error retrieving partition values: " + t);
+ }
+
+ public Table getTable() {
+ return table;
}
}
- private PartitionValuesResponse extractPartitionNamesByFilter(
- String catName, String dbName, String tableName, String filter,
List<FieldSchema> cols,
- boolean ascending, long maxParts)
- throws MetaException, NoSuchObjectException {
+ private abstract class GetListHelper<T> extends GetHelper<List<T>> {
+ public GetListHelper(String catName, String dbName, String tblName,
boolean allowSql,
+ boolean allowJdo) throws MetaException {
+ super(catName, dbName, tblName, null, allowSql, allowJdo);
+ }
- LOG.info("Table: {} filter: \"{}\" cols: {}",
- TableName.getQualified(catName, dbName, tableName), filter, cols);
- List<String> partitionNames = null;
- List<Partition> partitions = null;
- Table tbl = getTable(catName, dbName, tableName, null);
- try {
- // Get partitions by name - ascending or descending
- partitionNames = getPartitionNamesByFilter(catName, dbName, tableName,
filter, ascending,
- maxParts);
- } catch (MetaException e) {
- LOG.warn("Querying by partition names failed, trying out with partition
objects, filter: {}", filter);
+ public GetListHelper(String catName, String dbName, String tblName,
List<String> fields,
+ boolean allowSql, boolean allowJdo) throws MetaException {
+ super(catName, dbName, tblName, fields, allowSql, allowJdo);
}
- if (partitionNames == null) {
- partitions = getPartitionsByFilter(catName, dbName, tableName,
- new
GetPartitionsArgs.GetPartitionsArgsBuilder().filter(filter).max((short)
maxParts).build());
+ @Override
+ protected String describeResult() {
+ return results.size() + " entries";
}
+ }
- if (partitions != null) {
- partitionNames = new ArrayList<>(partitions.size());
- for (Partition partition : partitions) {
- // Check for NULL's just to be safe
- if (tbl.getPartitionKeys() != null && partition.getValues() != null) {
- partitionNames.add(Warehouse.makePartName(tbl.getPartitionKeys(),
partition.getValues()));
- }
- }
+ @VisibleForTesting
+ public abstract class GetDbHelper extends GetHelper<Database> {
+ /**
+ * GetHelper for returning db info using directSql/JDO.
+ * @param dbName The Database Name
+ * @param allowSql Whether or not we allow DirectSQL to perform this query.
+ * @param allowJdo Whether or not we allow ORM to perform this query.
+ */
+ public GetDbHelper(String catalogName, String dbName,boolean allowSql,
boolean allowJdo)
+ throws MetaException {
+ super(catalogName, dbName,null,allowSql,allowJdo);
}
- if (partitionNames == null) {
- throw new MetaException("Cannot obtain list of partitions by filter:\""
+ filter +
- "\" for " + TableName.getQualified(catName, dbName, tableName));
+ @Override
+ protected String describeResult() {
+ return "db details for db ".concat(dbName);
}
+ }
- if (!ascending) {
- partitionNames.sort(Collections.reverseOrder());
+ private abstract class GetStatHelper extends GetHelper<ColumnStatistics> {
+ public GetStatHelper(String catalogName, String dbName, String tblName,
boolean allowSql,
+ boolean allowJdo, String writeIdList) throws
MetaException {
+ super(catalogName, dbName, tblName, allowSql, allowJdo);
}
- // Return proper response
- PartitionValuesResponse response = new PartitionValuesResponse();
- response.setPartitionValues(new ArrayList<>(partitionNames.size()));
- LOG.info("Converting responses to Partition values for items: {}",
partitionNames.size());
- for (String partName : partitionNames) {
- ArrayList<String> vals = new
ArrayList<>(Collections.nCopies(tbl.getPartitionKeys().size(), null));
- PartitionValuesRow row = new PartitionValuesRow();
- Warehouse.makeValsFromName(partName, vals);
- for (String value : vals) {
- row.addToRow(value);
- }
- response.addToPartitionValues(row);
+ @Override
+ protected String describeResult() {
+ return "statistics for " + (results == null ? 0 :
results.getStatsObjSize()) + " columns";
}
- return response;
}
- private List<String> getPartitionNamesByFilter(String catName, String
dbName, String tableName,
- String filter, boolean
ascending, long maxParts)
- throws MetaException {
+ private Table ensureGetTable(String catName, String dbName, String tblName)
+ throws NoSuchObjectException, MetaException {
+ return convertToTable(ensureGetMTable(catName, dbName, tblName), conf);
+ }
- boolean success = false;
- List<String> partNames = new ArrayList<>();
- Query query = null;
- try {
- openTransaction();
- LOG.debug("Executing getPartitionNamesByFilter");
- catName = normalizeIdentifier(catName);
- dbName = dbName.toLowerCase();
- tableName = tableName.toLowerCase();
+ @Override
+ public MDatabase ensureGetMDatabase(String catName, String dbName)
Review Comment:
sure
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
