Repository: hive Updated Branches: refs/heads/hive-14535 c587404d4 -> 0f3998af3
HIVE-14990 : run all tests for MM tables and fix the issues that are found - issue with FetchOperator, drop, NPE (Sergey Shelukhin) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0f3998af Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0f3998af Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0f3998af Branch: refs/heads/hive-14535 Commit: 0f3998af3c5e9d8168b32d5085165892c2942101 Parents: c587404 Author: Sergey Shelukhin <[email protected]> Authored: Thu Oct 27 17:14:57 2016 -0700 Committer: Sergey Shelukhin <[email protected]> Committed: Thu Oct 27 17:34:00 2016 -0700 ---------------------------------------------------------------------- .../hadoop/hive/metastore/ObjectStore.java | 26 +++++++++++++++----- .../hadoop/hive/ql/exec/FetchOperator.java | 1 + 2 files changed, 21 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/0f3998af/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 3d97cbf..8ad7059 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -1161,6 +1161,12 @@ public class ObjectStore implements RawStore, Configurable { pm.deletePersistentAll(partGrants); } + // TODO# temporary; will be removed with ACID. Otherwise, need to do direct delete w/o get. + List<MTableWrite> mtw = getTableWrites(dbName, tableName, -1, -1); + if (mtw != null && mtw.size() > 0) { + pm.deletePersistentAll(mtw); + } + List<MPartitionColumnPrivilege> partColGrants = listTableAllPartitionColumnGrants(dbName, tableName); if (partColGrants != null && partColGrants.size() > 0) { @@ -8866,17 +8872,25 @@ public class ObjectStore implements RawStore, Configurable { public List<MTableWrite> getTableWrites( String dbName, String tblName, long from, long to) throws MetaException { boolean success = false; + dbName = HiveStringUtils.normalizeIdentifier(dbName); + tblName = HiveStringUtils.normalizeIdentifier(tblName); Query query = null; openTransaction(); try { - query = pm.newQuery(MTableWrite.class, - "table.tableName == t1 && table.database.name == t2 && writeId > t3 && writeId < t4"); - query.declareParameters( - "java.lang.String t1, java.lang.String t2, java.lang.Long t3, java.lang.Long t4"); + String queryStr = "table.tableName == t1 && table.database.name == t2 && writeId > t3", + argStr = "java.lang.String t1, java.lang.String t2, java.lang.Long t3"; + if (to >= 0) { + queryStr += " && writeId < t4"; + argStr += ", java.lang.Long t4"; + } + query = pm.newQuery(MTableWrite.class, queryStr); + query.declareParameters(argStr); query.setOrdering("writeId asc"); @SuppressWarnings("unchecked") - List<MTableWrite> writes = - (List<MTableWrite>) query.executeWithArray(tblName, dbName, from, to); + List<MTableWrite> writes = (List<MTableWrite>)(to >= 0 + ? query.executeWithArray(tblName, dbName, from, to) + : query.executeWithArray(tblName, dbName, from)); + pm.retrieveAll(writes); success = true; return (writes == null || writes.isEmpty()) ? null : new ArrayList<>(writes); } finally { http://git-wip-us.apache.org/repos/asf/hive/blob/0f3998af/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java index f89372c..dadae4d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java @@ -374,6 +374,7 @@ public class FetchOperator implements Serializable { Utilities.copyTableJobPropertiesToConf(currDesc.getTableDesc(), job); InputFormat inputFormat = getInputFormatFromCache(formatter, job); String inputs = processCurrPathForMmWriteIds(inputFormat); + Utilities.LOG14535.info("Setting fetch inputs to " + inputs); if (inputs == null) return null; job.set("mapred.input.dir", inputs);
