Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java Sun Aug 3 20:48:35 2014 @@ -31,15 +31,16 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Properties; import java.util.Set; -import java.lang.StringBuffer; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.ql.ErrorMsg; +import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.ql.Context; +import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.ConditionalTask; import org.apache.hadoop.hive.ql.exec.DemuxOperator; @@ -65,11 +66,10 @@ import org.apache.hadoop.hive.ql.exec.mr import org.apache.hadoop.hive.ql.exec.spark.SparkTask; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.io.RCFileInputFormat; -import org.apache.hadoop.hive.ql.io.rcfile.merge.MergeWork; +import org.apache.hadoop.hive.ql.io.merge.MergeWork; +import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMRUnionCtx; import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx; import org.apache.hadoop.hive.ql.optimizer.listbucketingpruner.ListBucketingPruner; @@ -1257,23 +1257,33 @@ public final class GenMapRedUtils { MapWork cplan; Serializable work; - if (conf.getBoolVar(ConfVars.HIVEMERGERCFILEBLOCKLEVEL) && - fsInputDesc.getTableInfo().getInputFileFormatClass().equals(RCFileInputFormat.class)) { + if ((conf.getBoolVar(ConfVars.HIVEMERGERCFILEBLOCKLEVEL) && + fsInputDesc.getTableInfo().getInputFileFormatClass().equals(RCFileInputFormat.class)) || + (conf.getBoolVar(ConfVars.HIVEMERGEORCFILESTRIPELEVEL) && + fsInputDesc.getTableInfo().getInputFileFormatClass().equals(OrcInputFormat.class))) { // Check if InputFormatClass is valid - String inputFormatClass = conf.getVar(ConfVars.HIVEMERGEINPUTFORMATBLOCKLEVEL); + final String inputFormatClass; + if (fsInputDesc.getTableInfo().getInputFileFormatClass().equals(RCFileInputFormat.class)) { + inputFormatClass = conf.getVar(ConfVars.HIVEMERGEINPUTFORMATBLOCKLEVEL); + } else { + inputFormatClass = conf.getVar(ConfVars.HIVEMERGEINPUTFORMATSTRIPELEVEL); + } try { Class c = Class.forName(inputFormatClass); - LOG.info("RCFile format- Using block level merge"); - cplan = GenMapRedUtils.createRCFileMergeTask(fsInputDesc, finalName, + if(fsInputDesc.getTableInfo().getInputFileFormatClass().equals(OrcInputFormat.class)) { + LOG.info("OrcFile format - Using stripe level merge"); + } else { + LOG.info("RCFile format- Using block level merge"); + } + cplan = GenMapRedUtils.createMergeTask(fsInputDesc, finalName, dpCtx != null && dpCtx.getNumDPCols() > 0); work = cplan; } catch (ClassNotFoundException e) { String msg = "Illegal input format class: " + inputFormatClass; throw new SemanticException(msg); } - } else { cplan = createMRWorkForMergingFiles(conf, tsMerge, fsInputDesc); if (conf.getVar(ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) { @@ -1486,20 +1496,22 @@ public final class GenMapRedUtils { } /** - * Create a block level merge task for RCFiles. + * Create a block level merge task for RCFiles or stripe level merge task for + * ORCFiles * * @param fsInputDesc * @param finalName - * @return MergeWork if table is stored as RCFile, + * @return MergeWork if table is stored as RCFile or ORCFile, * null otherwise */ - public static MapWork createRCFileMergeTask(FileSinkDesc fsInputDesc, + public static MapWork createMergeTask(FileSinkDesc fsInputDesc, Path finalName, boolean hasDynamicPartitions) throws SemanticException { Path inputDir = fsInputDesc.getFinalDirName(); TableDesc tblDesc = fsInputDesc.getTableInfo(); - if (tblDesc.getInputFileFormatClass().equals(RCFileInputFormat.class)) { + if (tblDesc.getInputFileFormatClass().equals(RCFileInputFormat.class) || + tblDesc.getInputFileFormatClass().equals(OrcInputFormat.class)) { ArrayList<Path> inputDirs = new ArrayList<Path>(1); ArrayList<String> inputDirstr = new ArrayList<String>(1); if (!hasDynamicPartitions @@ -1509,7 +1521,8 @@ public final class GenMapRedUtils { } MergeWork work = new MergeWork(inputDirs, finalName, - hasDynamicPartitions, fsInputDesc.getDynPartCtx()); + hasDynamicPartitions, fsInputDesc.getDynPartCtx(), + tblDesc.getInputFileFormatClass()); LinkedHashMap<String, ArrayList<String>> pathToAliases = new LinkedHashMap<String, ArrayList<String>>(); pathToAliases.put(inputDir.toString(), (ArrayList<String>) inputDirstr.clone()); @@ -1527,7 +1540,8 @@ public final class GenMapRedUtils { return work; } - throw new SemanticException("createRCFileMergeTask called on non-RCFile table"); + throw new SemanticException("createMergeTask called on a table with file" + + " format other than RCFile or ORCFile"); } /** @@ -1709,12 +1723,8 @@ public final class GenMapRedUtils { // generate the temporary file // it must be on the same file system as the current destination Context baseCtx = parseCtx.getContext(); - // if we are on viewfs we don't want to use /tmp as tmp dir since rename from /tmp/.. - // to final location /user/hive/warehouse/ will fail later, so instead pick tmp dir - // on same namespace as tbl dir. - Path tmpDir = dest.toUri().getScheme().equals("viewfs") ? - baseCtx.getExtTmpPathRelTo(dest.toUri()) : - baseCtx.getExternalTmpPath(dest.toUri()); + + Path tmpDir = baseCtx.getExternalTmpPath(dest); FileSinkDesc fileSinkDesc = fsOp.getConf(); // Change all the linked file sink descriptors
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeJoinProc.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeJoinProc.java?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeJoinProc.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeJoinProc.java Sun Aug 3 20:48:35 2014 @@ -18,17 +18,13 @@ package org.apache.hadoop.hive.ql.optimizer; -import java.util.HashMap; -import java.util.Map; import java.util.Stack; import org.apache.hadoop.hive.ql.exec.JoinOperator; -import org.apache.hadoop.hive.ql.exec.MapJoinOperator; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.parse.ParseContext; -import org.apache.hadoop.hive.ql.parse.QBJoinTree; import org.apache.hadoop.hive.ql.parse.SemanticException; public class SortedMergeJoinProc extends AbstractSMBJoinProc implements NodeProcessor { @@ -46,12 +42,6 @@ public class SortedMergeJoinProc extends JoinOperator joinOp = (JoinOperator) nd; SortBucketJoinProcCtx smbJoinContext = (SortBucketJoinProcCtx) procCtx; - Map<MapJoinOperator, QBJoinTree> mapJoinMap = pGraphContext.getMapJoinContext(); - if (mapJoinMap == null) { - mapJoinMap = new HashMap<MapJoinOperator, QBJoinTree>(); - pGraphContext.setMapJoinContext(mapJoinMap); - } - boolean convert = canConvertJoinToSMBJoin( joinOp, smbJoinContext, pGraphContext); Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java Sun Aug 3 20:48:35 2014 @@ -401,7 +401,6 @@ public class StatsRulesProcFactory { long numRows = stats.getNumRows(); - // evaluate similar to "col = constant" expr if (pred instanceof ExprNodeGenericFuncDesc) { ExprNodeGenericFuncDesc genFunc = (ExprNodeGenericFuncDesc) pred; @@ -413,9 +412,7 @@ public class StatsRulesProcFactory { String tabAlias = colDesc.getTabAlias(); ColStatistics cs = stats.getColumnStatisticsForColumn(tabAlias, colName); if (cs != null) { - long dvs = cs.getCountDistint(); - numRows = dvs == 0 ? numRows / 2 : numRows / dvs; - return numRows; + return cs.getNumNulls(); } } } Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTablePartMergeFilesDesc.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTablePartMergeFilesDesc.java?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTablePartMergeFilesDesc.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTablePartMergeFilesDesc.java Sun Aug 3 20:48:35 2014 @@ -25,6 +25,7 @@ import java.util.List; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.ListBucketingCtx; +import org.apache.hadoop.mapred.InputFormat; @Explain(displayName = "Alter Table Partition Merge Files") public class AlterTablePartMergeFilesDesc { @@ -35,6 +36,7 @@ public class AlterTablePartMergeFilesDes private List<Path> inputDir = new ArrayList<Path>(); private Path outputDir = null; + private Class<? extends InputFormat> inputFormatClass; public AlterTablePartMergeFilesDesc(String tableName, HashMap<String, String> partSpec) { @@ -90,4 +92,12 @@ public class AlterTablePartMergeFilesDes this.lbCtx = lbCtx; } + public Class<? extends InputFormat> getInputFormatClass() { + return inputFormatClass; + } + + public void setInputFormatClass(Class<? extends InputFormat> inputFormatClass) { + this.inputFormatClass = inputFormatClass; + } + } Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Sun Aug 3 20:48:35 2014 @@ -69,6 +69,7 @@ import org.apache.hadoop.hive.ql.index.H import org.apache.hadoop.hive.ql.index.HiveIndexHandler; import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.ql.io.RCFileInputFormat; +import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager; import org.apache.hadoop.hive.ql.lockmgr.LockException; @@ -969,7 +970,7 @@ public class DDLSemanticAnalyzer extends TableDesc tblDesc = Utilities.getTableDesc(table); // Write the output to temporary directory and move it to the final location at the end // so the operation is atomic. - Path queryTmpdir = ctx.getExternalTmpPath(newTblPartLoc.toUri()); + Path queryTmpdir = ctx.getExternalTmpPath(newTblPartLoc); truncateTblDesc.setOutputDir(queryTmpdir); LoadTableDesc ltd = new LoadTableDesc(queryTmpdir, tblDesc, partSpec == null ? new HashMap<String, String>() : partSpec); @@ -1520,11 +1521,13 @@ public class DDLSemanticAnalyzer extends tblObj.getSkewedColValueLocationMaps(), tblObj.isStoredAsSubDirectories(), conf); } - // throw a HiveException for non-rcfile. - if (!inputFormatClass.equals(RCFileInputFormat.class)) { + // throw a HiveException for other than rcfile and orcfile. + if (!((inputFormatClass.equals(RCFileInputFormat.class) || + (inputFormatClass.equals(OrcInputFormat.class))))) { throw new SemanticException( - "Only RCFileFormat is supportted right now."); + "Only RCFile and ORCFile Formats are supportted right now."); } + mergeDesc.setInputFormatClass(inputFormatClass); // throw a HiveException if the table/partition is bucketized if (bucketCols != null && bucketCols.size() > 0) { @@ -1549,7 +1552,7 @@ public class DDLSemanticAnalyzer extends ddlWork.setNeedLock(true); Task<? extends Serializable> mergeTask = TaskFactory.get(ddlWork, conf); TableDesc tblDesc = Utilities.getTableDesc(tblObj); - Path queryTmpdir = ctx.getExternalTmpPath(newTblPartLoc.toUri()); + Path queryTmpdir = ctx.getExternalTmpPath(newTblPartLoc); mergeDesc.setOutputDir(queryTmpdir); LoadTableDesc ltd = new LoadTableDesc(queryTmpdir, tblDesc, partSpec == null ? new HashMap<String, String>() : partSpec); Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java Sun Aug 3 20:48:35 2014 @@ -313,7 +313,8 @@ public class GenTezUtils { if (chDir) { // Merge the files in the destination table/partitions by creating Map-only merge job - // If underlying data is RCFile a RCFileBlockMerge task would be created. + // If underlying data is RCFile or OrcFile, RCFileBlockMerge task or + // OrcFileStripeMerge task would be created. LOG.info("using CombineHiveInputformat for the merge job"); GenMapRedUtils.createMRWorkForMergingFiles(fileSink, finalName, context.dependencyTask, context.moveTask, Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g Sun Aug 3 20:48:35 2014 @@ -928,7 +928,7 @@ alterStatement | KW_INDEX! alterIndexStatementSuffix | - KW_DATABASE! alterDatabaseStatementSuffix + (KW_DATABASE|KW_SCHEMA)! alterDatabaseStatementSuffix ) ; @@ -1299,7 +1299,7 @@ descStatement @after { popMsg(state); } : (KW_DESCRIBE|KW_DESC) (descOptions=KW_FORMATTED|descOptions=KW_EXTENDED|descOptions=KW_PRETTY)? (parttype=descPartTypeExpr) -> ^(TOK_DESCTABLE $parttype $descOptions?) | (KW_DESCRIBE|KW_DESC) KW_FUNCTION KW_EXTENDED? (name=descFuncNames) -> ^(TOK_DESCFUNCTION $name KW_EXTENDED?) - | (KW_DESCRIBE|KW_DESC) KW_DATABASE KW_EXTENDED? (dbName=identifier) -> ^(TOK_DESCDATABASE $dbName KW_EXTENDED?) + | (KW_DESCRIBE|KW_DESC) (KW_DATABASE|KW_SCHEMA) KW_EXTENDED? (dbName=identifier) -> ^(TOK_DESCDATABASE $dbName KW_EXTENDED?) ; analyzeStatement @@ -1324,7 +1324,7 @@ showStatement -> ^(TOK_SHOW_TABLESTATUS showStmtIdentifier $db_name? partitionSpec?) | KW_SHOW KW_TBLPROPERTIES tblName=identifier (LPAREN prptyName=StringLiteral RPAREN)? -> ^(TOK_SHOW_TBLPROPERTIES $tblName $prptyName?) | KW_SHOW KW_LOCKS (parttype=partTypeExpr)? (isExtended=KW_EXTENDED)? -> ^(TOK_SHOWLOCKS $parttype? $isExtended?) - | KW_SHOW KW_LOCKS KW_DATABASE (dbName=Identifier) (isExtended=KW_EXTENDED)? -> ^(TOK_SHOWDBLOCKS $dbName $isExtended?) + | KW_SHOW KW_LOCKS (KW_DATABASE|KW_SCHEMA) (dbName=Identifier) (isExtended=KW_EXTENDED)? -> ^(TOK_SHOWDBLOCKS $dbName $isExtended?) | KW_SHOW (showOptions=KW_FORMATTED)? (KW_INDEX|KW_INDEXES) KW_ON showStmtIdentifier ((KW_FROM|KW_IN) db_name=identifier)? -> ^(TOK_SHOWINDEXES showStmtIdentifier $showOptions? $db_name?) | KW_SHOW KW_COMPACTIONS -> ^(TOK_SHOW_COMPACTIONS) @@ -1341,7 +1341,7 @@ lockStatement lockDatabase @init { pushMsg("lock database statement", state); } @after { popMsg(state); } - : KW_LOCK KW_DATABASE (dbName=Identifier) lockMode -> ^(TOK_LOCKDB $dbName lockMode) + : KW_LOCK (KW_DATABASE|KW_SCHEMA) (dbName=Identifier) lockMode -> ^(TOK_LOCKDB $dbName lockMode) ; lockMode @@ -1359,7 +1359,7 @@ unlockStatement unlockDatabase @init { pushMsg("unlock database statement", state); } @after { popMsg(state); } - : KW_UNLOCK KW_DATABASE (dbName=Identifier) -> ^(TOK_UNLOCKDB $dbName) + : KW_UNLOCK (KW_DATABASE|KW_SCHEMA) (dbName=Identifier) -> ^(TOK_UNLOCKDB $dbName) ; createRoleStatement @@ -1471,7 +1471,7 @@ privilegeObject privObjectType @init {pushMsg("privilege object type type", state);} @after {popMsg(state);} - : KW_DATABASE -> ^(TOK_DB_TYPE) + : (KW_DATABASE|KW_SCHEMA) -> ^(TOK_DB_TYPE) | KW_TABLE? -> ^(TOK_TABLE_TYPE) ; Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java Sun Aug 3 20:48:35 2014 @@ -276,7 +276,7 @@ public class ImportSemanticAnalyzer exte private Task<?> loadTable(URI fromURI, Table table) { Path dataPath = new Path(fromURI.toString(), "data"); - Path tmpPath = ctx.getExternalTmpPath(fromURI); + Path tmpPath = ctx.getExternalTmpPath(new Path(fromURI)); Task<?> copyTask = TaskFactory.get(new CopyWork(dataPath, tmpPath, false), conf); LoadTableDesc loadTableWork = new LoadTableDesc(tmpPath, @@ -321,7 +321,7 @@ public class ImportSemanticAnalyzer exte LOG.debug("adding dependent CopyWork/AddPart/MoveWork for partition " + partSpecToString(partSpec.getPartSpec()) + " with source location: " + srcLocation); - Path tmpPath = ctx.getExternalTmpPath(fromURI); + Path tmpPath = ctx.getExternalTmpPath(new Path(fromURI)); Task<?> copyTask = TaskFactory.get(new CopyWork(new Path(srcLocation), tmpPath, false), conf); Task<?> addPartTask = TaskFactory.get(new DDLWork(getInputs(), Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Sun Aug 3 20:48:35 2014 @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.ql.parse; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVESTATSDBCLASS; -import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DATABASE_WAREHOUSE_SUFFIX; import java.io.IOException; import java.io.Serializable; @@ -31,10 +30,10 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.UUID; import java.util.Map.Entry; import java.util.Set; import java.util.TreeSet; +import java.util.UUID; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; @@ -1426,7 +1425,7 @@ public class SemanticAnalyzer extends Ba } try { fname = ctx.getExternalTmpPath( - FileUtils.makeQualified(location, conf).toUri()).toString(); + FileUtils.makeQualified(location, conf)).toString(); } catch (Exception e) { throw new SemanticException(generateErrorMessage(ast, "Error creating temporary folder on: " + location.toString()), e); @@ -2328,13 +2327,13 @@ public class SemanticAnalyzer extends Ba * for inner joins push a 'is not null predicate' to the join sources for * every non nullSafe predicate. */ - private Operator genNotNullFilterForJoinSourcePlan(QB qb, Operator input, + private Operator genNotNullFilterForJoinSourcePlan(QB qb, Operator input, QBJoinTree joinTree, ExprNodeDesc[] joinKeys) throws SemanticException { if (qb == null || joinTree == null) { return input; } - + if (!joinTree.getNoOuterJoin()) { return input; } @@ -5662,12 +5661,7 @@ public class SemanticAnalyzer extends Ba if (isNonNativeTable) { queryTmpdir = dest_path; } else { - // if we are on viewfs we don't want to use /tmp as tmp dir since rename from /tmp/.. - // to final /user/hive/warehouse/ will fail later, so instead pick tmp dir - // on same namespace as tbl dir. - queryTmpdir = dest_path.toUri().getScheme().equals("viewfs") ? - ctx.getExtTmpPathRelTo(dest_path.getParent().toUri()) : - ctx.getExternalTmpPath(dest_path.toUri()); + queryTmpdir = ctx.getExternalTmpPath(dest_path); } if (dpCtx != null) { // set the root of the temporary path where dynamic partition columns will populate @@ -5780,12 +5774,7 @@ public class SemanticAnalyzer extends Ba dest_path = new Path(tabPath.toUri().getScheme(), tabPath.toUri() .getAuthority(), partPath.toUri().getPath()); - // if we are on viewfs we don't want to use /tmp as tmp dir since rename from /tmp/.. - // to final /user/hive/warehouse/ will fail later, so instead pick tmp dir - // on same namespace as tbl dir. - queryTmpdir = dest_path.toUri().getScheme().equals("viewfs") ? - ctx.getExtTmpPathRelTo(dest_path.getParent().toUri()) : - ctx.getExternalTmpPath(dest_path.toUri()); + queryTmpdir = ctx.getExternalTmpPath(dest_path); table_desc = Utilities.getTableDesc(dest_tab); // Add sorting/bucketing if needed @@ -5842,7 +5831,7 @@ public class SemanticAnalyzer extends Ba try { Path qPath = FileUtils.makeQualified(dest_path, conf); - queryTmpdir = ctx.getExternalTmpPath(qPath.toUri()); + queryTmpdir = ctx.getExternalTmpPath(qPath); } catch (Exception e) { throw new SemanticException("Error creating temporary folder on: " + dest_path, e); @@ -6003,7 +5992,7 @@ public class SemanticAnalyzer extends Ba // it should be the same as the MoveWork's sourceDir. fileSinkDesc.setStatsAggPrefix(fileSinkDesc.getDirName().toString()); if (HiveConf.getVar(conf, HIVESTATSDBCLASS).equalsIgnoreCase(StatDB.fs.name())) { - String statsTmpLoc = ctx.getExternalTmpPath(queryTmpdir.toUri()).toString(); + String statsTmpLoc = ctx.getExternalTmpPath(queryTmpdir).toString(); LOG.info("Set stats collection dir : " + statsTmpLoc); conf.set(StatsSetupConst.STATS_TMP_LOC, statsTmpLoc); } @@ -9014,7 +9003,7 @@ public class SemanticAnalyzer extends Ba tsDesc.setGatherStats(false); } else { if (HiveConf.getVar(conf, HIVESTATSDBCLASS).equalsIgnoreCase(StatDB.fs.name())) { - String statsTmpLoc = ctx.getExternalTmpPath(tab.getPath().toUri()).toString(); + String statsTmpLoc = ctx.getExternalTmpPath(tab.getPath()).toString(); LOG.info("Set stats collection dir : " + statsTmpLoc); conf.set(StatsSetupConst.STATS_TMP_LOC, statsTmpLoc); } @@ -9501,7 +9490,11 @@ public class SemanticAnalyzer extends Ba // Generate column access stats if required - wait until column pruning takes place // during optimization - if (HiveConf.getBoolVar(this.conf, HiveConf.ConfVars.HIVE_STATS_COLLECT_SCANCOLS) == true) { + boolean isColumnInfoNeedForAuth = SessionState.get().isAuthorizationModeV2() + && HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED); + + if (isColumnInfoNeedForAuth + || HiveConf.getBoolVar(this.conf, HiveConf.ConfVars.HIVE_STATS_COLLECT_SCANCOLS) == true) { ColumnAccessAnalyzer columnAccessAnalyzer = new ColumnAccessAnalyzer(pCtx); setColumnAccessInfo(columnAccessAnalyzer.analyzeColumnAccess()); } @@ -9548,7 +9541,7 @@ public class SemanticAnalyzer extends Ba if (((TableScanDesc)topOp.getConf()).getIsMetadataOnly()) { continue; } - PrunedPartitionList parts = pCtx.getOpToPartList().get((TableScanOperator) topOp); + PrunedPartitionList parts = pCtx.getOpToPartList().get(topOp); if (parts.getPartitions().size() > scanLimit) { throw new SemanticException(ErrorMsg.PARTITION_SCAN_LIMIT_EXCEEDED, "" + parts.getPartitions().size(), "" + parts.getSourceTable().getTableName(), "" @@ -10169,7 +10162,7 @@ public class SemanticAnalyzer extends Ba String dbName = qualified.length == 1 ? SessionState.get().getCurrentDatabase() : qualified[0]; Database database = getDatabase(dbName); outputs.add(new WriteEntity(database, WriteEntity.WriteType.DDL_SHARED)); - + if (isTemporary) { if (partCols.size() > 0) { throw new SemanticException("Partition columns are not supported on temporary tables"); Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java Sun Aug 3 20:48:35 2014 @@ -36,6 +36,17 @@ public class ExprNodeConstantDesc extend private static final long serialVersionUID = 1L; final protected transient static char[] hexArray = "0123456789ABCDEF".toCharArray(); private Object value; + // If this constant was created while doing constant folding, foldedFromCol holds the name of + // original column from which it was folded. + private transient String foldedFromCol; + + public String getFoldedFromCol() { + return foldedFromCol; + } + + public void setFoldedFromCol(String foldedFromCol) { + this.foldedFromCol = foldedFromCol; + } public ExprNodeConstantDesc() { } Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationUtils.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationUtils.java?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationUtils.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationUtils.java Sun Aug 3 20:48:35 2014 @@ -17,6 +17,10 @@ */ package org.apache.hadoop.hive.ql.security.authorization; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate; import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege; import org.apache.hadoop.hive.metastore.api.HiveObjectRef; @@ -24,8 +28,8 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo; import org.apache.hadoop.hive.ql.ErrorMsg; -import org.apache.hadoop.hive.ql.hooks.Entity; import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.hooks.Entity; import org.apache.hadoop.hive.ql.hooks.Entity.Type; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity.WriteType; @@ -42,9 +46,6 @@ import org.apache.hadoop.hive.ql.securit import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivilegeObjectType; import org.apache.hadoop.hive.ql.session.SessionState; -import java.util.ArrayList; -import java.util.List; - /** * Utility code shared by hive internal code and sql standard authorization plugin implementation */ @@ -173,7 +174,7 @@ public class AuthorizationUtils { } public static HivePrivilegeObject getHivePrivilegeObject( - PrivilegeObjectDesc privSubjectDesc, List<String> columns) throws HiveException { + PrivilegeObjectDesc privSubjectDesc, Set<String> columns) throws HiveException { // null means ALL for show grants, GLOBAL for grant/revoke HivePrivilegeObjectType objectType = null; Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/DefaultHiveAuthorizationProvider.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/DefaultHiveAuthorizationProvider.java?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/DefaultHiveAuthorizationProvider.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/DefaultHiveAuthorizationProvider.java Sun Aug 3 20:48:35 2014 @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.ql.security.authorization; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -27,7 +26,7 @@ public class DefaultHiveAuthorizationPro BitSetCheckedAuthorizationProvider { public void init(Configuration conf) throws HiveException { - hive_db = new HiveProxy(Hive.get(new HiveConf(conf, HiveAuthorizationProvider.class))); + hive_db = new HiveProxy(Hive.get(conf, HiveAuthorizationProvider.class)); } } Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java Sun Aug 3 20:48:35 2014 @@ -35,7 +35,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.Database; @@ -83,7 +82,7 @@ public class StorageBasedAuthorizationPr // till we explicitly initialize it as being from the client side. So, we have a // chicken-and-egg problem. So, we now track whether or not we're running from client-side // in the SBAP itself. - hive_db = new HiveProxy(Hive.get(new HiveConf(getConf(), StorageBasedAuthorizationProvider.class))); + hive_db = new HiveProxy(Hive.get(getConf(), StorageBasedAuthorizationProvider.class)); this.wh = new Warehouse(getConf()); if (this.wh == null){ // If wh is still null after just having initialized it, bail out - something's very wrong. @@ -117,7 +116,7 @@ public class StorageBasedAuthorizationPr // Update to previous comment: there does seem to be one place that uses this // and that is to authorize "show databases" in hcat commandline, which is used - // by webhcat. And user-level auth seems to be a resonable default in this case. + // by webhcat. And user-level auth seems to be a reasonable default in this case. // The now deprecated HdfsAuthorizationProvider in hcatalog approached this in // another way, and that was to see if the user had said above appropriate requested // privileges for the hive root warehouse directory. That seems to be the best Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java Sun Aug 3 20:48:35 2014 @@ -17,9 +17,12 @@ */ package org.apache.hadoop.hive.ql.security.authorization.plugin; -import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; +import java.util.Set; import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate; import org.apache.hadoop.hive.common.classification.InterfaceStability.Unstable; @@ -35,7 +38,9 @@ public class HivePrivilegeObject impleme public int compareTo(HivePrivilegeObject o) { int compare = type.compareTo(o.type); if (compare == 0) { - compare = dbname.compareTo(o.dbname); + compare = dbname != null ? + (o.dbname != null ? dbname.compareTo(o.dbname) : 1) : + (o.dbname != null ? -1 : 0); } if (compare == 0) { compare = objectName != null ? @@ -55,9 +60,18 @@ public class HivePrivilegeObject impleme return compare; } - private int compare(List<String> o1, List<String> o2) { - for (int i = 0; i < Math.min(o1.size(), o2.size()); i++) { - int compare = o1.get(i).compareTo(o2.get(i)); + private int compare(Collection<String> o1, Collection<String> o2) { + Iterator<String> it1 = o1.iterator(); + Iterator<String> it2 = o2.iterator(); + while (it1.hasNext()) { + if (!it2.hasNext()) { + break; + } + String s1 = it1.next(); + String s2 = it2.next(); + int compare = s1 != null ? + (s2 != null ? s1.compareTo(s2) : 1) : + (s2 != null ? -1 : 0); if (compare != 0) { return compare; } @@ -77,7 +91,7 @@ public class HivePrivilegeObject impleme private final String objectName; private final List<String> commandParams; private final List<String> partKeys; - private final List<String> columns; + private Set<String> columns; private final HivePrivObjectActionType actionType; public HivePrivilegeObject(HivePrivilegeObjectType type, String dbname, String objectName) { @@ -92,9 +106,8 @@ public class HivePrivilegeObject impleme public HivePrivilegeObject(HivePrivilegeObjectType type, String dbname, String objectName, List<String> partKeys, String column) { this(type, dbname, objectName, partKeys, - column == null ? null : new ArrayList<String>(Arrays.asList(column)), + column == null ? null : new HashSet<String>(Arrays.asList(column)), HivePrivObjectActionType.OTHER, null); - } /** @@ -108,12 +121,12 @@ public class HivePrivilegeObject impleme } public HivePrivilegeObject(HivePrivilegeObjectType type, String dbname, String objectName, - List<String> partKeys, List<String> columns, List<String> commandParams) { + List<String> partKeys, Set<String> columns, List<String> commandParams) { this(type, dbname, objectName, partKeys, columns, HivePrivObjectActionType.OTHER, commandParams); } public HivePrivilegeObject(HivePrivilegeObjectType type, String dbname, String objectName, - List<String> partKeys, List<String> columns, HivePrivObjectActionType actionType, + List<String> partKeys, Set<String> columns, HivePrivObjectActionType actionType, List<String> commandParams) { this.type = type; this.dbname = dbname; @@ -151,7 +164,13 @@ public class HivePrivilegeObject impleme return partKeys; } - public List<String> getColumns() { + /** + * Applicable columns in this object + * In case of DML read operations, this is the set of columns being used. + * Column information is not set for DDL operations and for tables being written into + * @return list of applicable columns + */ + public Set<String> getColumns() { return columns; } @@ -200,4 +219,8 @@ public class HivePrivilegeObject impleme return (dbname == null ? "" : dbname + ".") + objectName; } + public void setColumns(Set<String> columnms) { + this.columns = columnms; + } + } Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveRoleGrant.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveRoleGrant.java?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveRoleGrant.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveRoleGrant.java Sun Aug 3 20:48:35 2014 @@ -123,5 +123,7 @@ public class HiveRoleGrant implements Co } - + public String toString() { + return roleName + "[" + principalName + ":" + principalType + (grantOption ? ":WITH GRANT]" : "]"); + } } Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveV1Authorizer.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveV1Authorizer.java?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveV1Authorizer.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveV1Authorizer.java Sun Aug 3 20:48:35 2014 @@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.securi import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.Warehouse; @@ -318,7 +319,7 @@ public class HiveV1Authorizer implements privs.addAll(hive.showPrivilegeGrant(HiveObjectType.DATABASE, name, type, dbObj.getName(), null, null, null)); } else { - List<String> columns = privObj.getColumns(); + Set<String> columns = privObj.getColumns(); if (columns != null && !columns.isEmpty()) { // show column level privileges for (String columnName : columns) { Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessController.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessController.java?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessController.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessController.java Sun Aug 3 20:48:35 2014 @@ -105,6 +105,7 @@ public class SQLStdHiveAccessController } this.currentUserName = newUserName; this.currentRoles = getRolesFromMS(); + LOG.info("Current user : " + currentUserName + ", Current Roles : " + currentRoles); } private List<HiveRoleGrant> getRolesFromMS() throws HiveAuthzPluginException { @@ -532,6 +533,7 @@ public class SQLStdHiveAccessController currentRoles.add(adminRole); return; } + LOG.info("Current user : " + currentUserName + ", Current Roles : " + currentRoles); // If we are here it means, user is requesting a role he doesn't belong to. throw new HiveAccessControlException(currentUserName +" doesn't belong to role " +roleName); Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java Sun Aug 3 20:48:35 2014 @@ -429,6 +429,8 @@ public class StatsUtils { cs.setAvgColLen(JavaDataModel.get().lengthOfTimestamp()); } else if (colType.startsWith(serdeConstants.DECIMAL_TYPE_NAME)) { cs.setAvgColLen(JavaDataModel.get().lengthOfDecimal()); + cs.setCountDistint(csd.getDecimalStats().getNumDVs()); + cs.setNumNulls(csd.getDecimalStats().getNumNulls()); } else if (colType.equalsIgnoreCase(serdeConstants.DATE_TYPE_NAME)) { cs.setAvgColLen(JavaDataModel.get().lengthOfDate()); } else { Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java (original) +++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java Sun Aug 3 20:48:35 2014 @@ -126,12 +126,13 @@ public class TestDbTxnManager { public void testSingleWriteTable() throws Exception { WriteEntity we = addTableOutput(WriteEntity.WriteType.INSERT); QueryPlan qp = new MockQueryPlan(this); + txnMgr.openTxn("fred"); txnMgr.acquireLocks(qp, ctx, "fred"); List<HiveLock> locks = ctx.getHiveLocks(); Assert.assertEquals(1, locks.size()); Assert.assertEquals(1, TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId)); - txnMgr.getLockManager().unlock(locks.get(0)); + txnMgr.commitTxn(); locks = txnMgr.getLockManager().getLocks(false, false); Assert.assertEquals(0, locks.size()); } @@ -144,12 +145,13 @@ public class TestDbTxnManager { addPartitionInput(t); WriteEntity we = addTableOutput(WriteEntity.WriteType.INSERT); QueryPlan qp = new MockQueryPlan(this); + txnMgr.openTxn("fred"); txnMgr.acquireLocks(qp, ctx, "fred"); List<HiveLock> locks = ctx.getHiveLocks(); Assert.assertEquals(1, locks.size()); Assert.assertEquals(4, TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId)); - txnMgr.getLockManager().unlock(locks.get(0)); + txnMgr.commitTxn(); locks = txnMgr.getLockManager().getLocks(false, false); Assert.assertEquals(0, locks.size()); } @@ -158,12 +160,13 @@ public class TestDbTxnManager { public void testUpdate() throws Exception { WriteEntity we = addTableOutput(WriteEntity.WriteType.UPDATE); QueryPlan qp = new MockQueryPlan(this); + txnMgr.openTxn("fred"); txnMgr.acquireLocks(qp, ctx, "fred"); List<HiveLock> locks = ctx.getHiveLocks(); Assert.assertEquals(1, locks.size()); Assert.assertEquals(1, TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId)); - txnMgr.getLockManager().unlock(locks.get(0)); + txnMgr.commitTxn(); locks = txnMgr.getLockManager().getLocks(false, false); Assert.assertEquals(0, locks.size()); } @@ -172,12 +175,28 @@ public class TestDbTxnManager { public void testDelete() throws Exception { WriteEntity we = addTableOutput(WriteEntity.WriteType.DELETE); QueryPlan qp = new MockQueryPlan(this); + txnMgr.openTxn("fred"); txnMgr.acquireLocks(qp, ctx, "fred"); List<HiveLock> locks = ctx.getHiveLocks(); Assert.assertEquals(1, locks.size()); Assert.assertEquals(1, TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId)); - txnMgr.getLockManager().unlock(locks.get(0)); + txnMgr.commitTxn(); + locks = txnMgr.getLockManager().getLocks(false, false); + Assert.assertEquals(0, locks.size()); + } + + @Test + public void testRollback() throws Exception { + WriteEntity we = addTableOutput(WriteEntity.WriteType.DELETE); + QueryPlan qp = new MockQueryPlan(this); + txnMgr.openTxn("fred"); + txnMgr.acquireLocks(qp, ctx, "fred"); + List<HiveLock> locks = ctx.getHiveLocks(); + Assert.assertEquals(1, locks.size()); + Assert.assertEquals(1, + TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId)); + txnMgr.rollbackTxn(); locks = txnMgr.getLockManager().getLocks(false, false); Assert.assertEquals(0, locks.size()); } Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java (original) +++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java Sun Aug 3 20:48:35 2014 @@ -509,6 +509,7 @@ public class TestHive extends TestCase { public void testHiveRefreshOnConfChange() throws Throwable{ Hive prevHiveObj = Hive.get(); + prevHiveObj.getDatabaseCurrent(); Hive newHiveObj; //if HiveConf has not changed, same object should be returned @@ -522,6 +523,7 @@ public class TestHive extends TestCase { //if HiveConf has changed, new object should be returned prevHiveObj = Hive.get(); + prevHiveObj.getDatabaseCurrent(); //change value of a metavar config param in new hive conf newHconf = new HiveConf(hiveConf); newHconf.setIntVar(ConfVars.METASTORETHRIFTCONNECTIONRETRIES, Modified: hive/branches/spark/ql/src/test/queries/clientnegative/authorization_cannot_create_all_role.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientnegative/authorization_cannot_create_all_role.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientnegative/authorization_cannot_create_all_role.q (original) +++ hive/branches/spark/ql/src/test/queries/clientnegative/authorization_cannot_create_all_role.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set user.name=hive_admin_user; Modified: hive/branches/spark/ql/src/test/queries/clientnegative/authorization_cannot_create_default_role.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientnegative/authorization_cannot_create_default_role.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientnegative/authorization_cannot_create_default_role.q (original) +++ hive/branches/spark/ql/src/test/queries/clientnegative/authorization_cannot_create_default_role.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set user.name=hive_admin_user; Modified: hive/branches/spark/ql/src/test/queries/clientnegative/authorization_cannot_create_none_role.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientnegative/authorization_cannot_create_none_role.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientnegative/authorization_cannot_create_none_role.q (original) +++ hive/branches/spark/ql/src/test/queries/clientnegative/authorization_cannot_create_none_role.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set user.name=hive_admin_user; Modified: hive/branches/spark/ql/src/test/queries/clientnegative/authorization_caseinsensitivity.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientnegative/authorization_caseinsensitivity.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientnegative/authorization_caseinsensitivity.q (original) +++ hive/branches/spark/ql/src/test/queries/clientnegative/authorization_caseinsensitivity.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set user.name=hive_admin_user; Modified: hive/branches/spark/ql/src/test/queries/clientnegative/authorization_drop_db_cascade.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientnegative/authorization_drop_db_cascade.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientnegative/authorization_drop_db_cascade.q (original) +++ hive/branches/spark/ql/src/test/queries/clientnegative/authorization_drop_db_cascade.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set hive.security.authorization.enabled=true; Modified: hive/branches/spark/ql/src/test/queries/clientnegative/authorization_drop_db_empty.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientnegative/authorization_drop_db_empty.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientnegative/authorization_drop_db_empty.q (original) +++ hive/branches/spark/ql/src/test/queries/clientnegative/authorization_drop_db_empty.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set hive.security.authorization.enabled=true; Modified: hive/branches/spark/ql/src/test/queries/clientnegative/authorization_drop_role_no_admin.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientnegative/authorization_drop_role_no_admin.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientnegative/authorization_drop_role_no_admin.q (original) +++ hive/branches/spark/ql/src/test/queries/clientnegative/authorization_drop_role_no_admin.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set user.name=hive_admin_user; Modified: hive/branches/spark/ql/src/test/queries/clientnegative/authorization_priv_current_role_neg.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientnegative/authorization_priv_current_role_neg.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientnegative/authorization_priv_current_role_neg.q (original) +++ hive/branches/spark/ql/src/test/queries/clientnegative/authorization_priv_current_role_neg.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set user.name=hive_admin_user; Modified: hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_cycles1.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_cycles1.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_cycles1.q (original) +++ hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_cycles1.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set user.name=hive_admin_user; Modified: hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_cycles2.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_cycles2.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_cycles2.q (original) +++ hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_cycles2.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; Modified: hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_grant.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_grant.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_grant.q (original) +++ hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_grant.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set user.name=hive_admin_user; Modified: hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_grant2.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_grant2.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_grant2.q (original) +++ hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_grant2.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set user.name=hive_admin_user; Modified: hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_grant_nosuchrole.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_grant_nosuchrole.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_grant_nosuchrole.q (original) +++ hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_grant_nosuchrole.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set user.name=hive_admin_user; Modified: hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_grant_otherrole.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_grant_otherrole.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_grant_otherrole.q (original) +++ hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_grant_otherrole.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; Modified: hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_grant_otheruser.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_grant_otheruser.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_grant_otheruser.q (original) +++ hive/branches/spark/ql/src/test/queries/clientnegative/authorization_role_grant_otheruser.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; Modified: hive/branches/spark/ql/src/test/queries/clientnegative/authorization_rolehierarchy_privs.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientnegative/authorization_rolehierarchy_privs.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientnegative/authorization_rolehierarchy_privs.q (original) +++ hive/branches/spark/ql/src/test/queries/clientnegative/authorization_rolehierarchy_privs.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set hive.security.authorization.enabled=true; Modified: hive/branches/spark/ql/src/test/queries/clientnegative/authorization_set_role_neg2.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientnegative/authorization_set_role_neg2.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientnegative/authorization_set_role_neg2.q (original) +++ hive/branches/spark/ql/src/test/queries/clientnegative/authorization_set_role_neg2.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set user.name=hive_admin_user; Modified: hive/branches/spark/ql/src/test/queries/clientnegative/authorization_show_grant_otherrole.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientnegative/authorization_show_grant_otherrole.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientnegative/authorization_show_grant_otherrole.q (original) +++ hive/branches/spark/ql/src/test/queries/clientnegative/authorization_show_grant_otherrole.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set hive.security.authorization.enabled=true; Modified: hive/branches/spark/ql/src/test/queries/clientnegative/authorization_show_grant_otheruser_all.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientnegative/authorization_show_grant_otheruser_all.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientnegative/authorization_show_grant_otheruser_all.q (original) +++ hive/branches/spark/ql/src/test/queries/clientnegative/authorization_show_grant_otheruser_all.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set hive.security.authorization.enabled=true; Modified: hive/branches/spark/ql/src/test/queries/clientnegative/authorization_show_grant_otheruser_alltabs.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientnegative/authorization_show_grant_otheruser_alltabs.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientnegative/authorization_show_grant_otheruser_alltabs.q (original) +++ hive/branches/spark/ql/src/test/queries/clientnegative/authorization_show_grant_otheruser_alltabs.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set hive.security.authorization.enabled=true; Modified: hive/branches/spark/ql/src/test/queries/clientnegative/authorization_show_grant_otheruser_wtab.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientnegative/authorization_show_grant_otheruser_wtab.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientnegative/authorization_show_grant_otheruser_wtab.q (original) +++ hive/branches/spark/ql/src/test/queries/clientnegative/authorization_show_grant_otheruser_wtab.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set hive.security.authorization.enabled=true; Modified: hive/branches/spark/ql/src/test/queries/clientpositive/authorization_1_sql_std.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/authorization_1_sql_std.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientpositive/authorization_1_sql_std.q (original) +++ hive/branches/spark/ql/src/test/queries/clientpositive/authorization_1_sql_std.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set user.name=hive_admin_user; Modified: hive/branches/spark/ql/src/test/queries/clientpositive/authorization_9.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/authorization_9.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientpositive/authorization_9.q (original) +++ hive/branches/spark/ql/src/test/queries/clientpositive/authorization_9.q Sun Aug 3 20:48:35 2014 @@ -2,6 +2,7 @@ create table dummy (key string, value string); +grant select to user hive_test_user; grant select on database default to user hive_test_user; grant select on table dummy to user hive_test_user; grant select (key, value) on table dummy to user hive_test_user; @@ -10,16 +11,21 @@ show grant user hive_test_user on databa show grant user hive_test_user on table dummy; show grant user hive_test_user on all; +grant select to user hive_test_user2; grant select on database default to user hive_test_user2; grant select on table dummy to user hive_test_user2; grant select (key, value) on table dummy to user hive_test_user2; show grant on all; +show grant user hive_test_user on all; +show grant user hive_test_user2 on all; +revoke select from user hive_test_user; revoke select on database default from user hive_test_user; revoke select on table dummy from user hive_test_user; revoke select (key, value) on table dummy from user hive_test_user; +revoke select from user hive_test_user2; revoke select on database default from user hive_test_user2; revoke select on table dummy from user hive_test_user2; revoke select (key, value) on table dummy from user hive_test_user2; Modified: hive/branches/spark/ql/src/test/queries/clientpositive/authorization_admin_almighty1.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/authorization_admin_almighty1.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientpositive/authorization_admin_almighty1.q (original) +++ hive/branches/spark/ql/src/test/queries/clientpositive/authorization_admin_almighty1.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set user.name=hive_test_user; Modified: hive/branches/spark/ql/src/test/queries/clientpositive/authorization_admin_almighty2.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/authorization_admin_almighty2.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientpositive/authorization_admin_almighty2.q (original) +++ hive/branches/spark/ql/src/test/queries/clientpositive/authorization_admin_almighty2.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set user.name=hive_admin_user; Modified: hive/branches/spark/ql/src/test/queries/clientpositive/authorization_create_func1.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/authorization_create_func1.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientpositive/authorization_create_func1.q (original) +++ hive/branches/spark/ql/src/test/queries/clientpositive/authorization_create_func1.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set hive.security.authorization.enabled=true; Modified: hive/branches/spark/ql/src/test/queries/clientpositive/authorization_create_macro1.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/authorization_create_macro1.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientpositive/authorization_create_macro1.q (original) +++ hive/branches/spark/ql/src/test/queries/clientpositive/authorization_create_macro1.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set hive.security.authorization.enabled=true; Modified: hive/branches/spark/ql/src/test/queries/clientpositive/authorization_insert.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/authorization_insert.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientpositive/authorization_insert.q (original) +++ hive/branches/spark/ql/src/test/queries/clientpositive/authorization_insert.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; Modified: hive/branches/spark/ql/src/test/queries/clientpositive/authorization_owner_actions_db.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/authorization_owner_actions_db.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientpositive/authorization_owner_actions_db.q (original) +++ hive/branches/spark/ql/src/test/queries/clientpositive/authorization_owner_actions_db.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set hive.security.authorization.enabled=true; Modified: hive/branches/spark/ql/src/test/queries/clientpositive/authorization_role_grant1.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/authorization_role_grant1.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientpositive/authorization_role_grant1.q (original) +++ hive/branches/spark/ql/src/test/queries/clientpositive/authorization_role_grant1.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set user.name=hive_admin_user; Modified: hive/branches/spark/ql/src/test/queries/clientpositive/authorization_role_grant2.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/authorization_role_grant2.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientpositive/authorization_role_grant2.q (original) +++ hive/branches/spark/ql/src/test/queries/clientpositive/authorization_role_grant2.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; Modified: hive/branches/spark/ql/src/test/queries/clientpositive/authorization_set_show_current_role.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/authorization_set_show_current_role.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientpositive/authorization_set_show_current_role.q (original) +++ hive/branches/spark/ql/src/test/queries/clientpositive/authorization_set_show_current_role.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set user.name=hive_admin_user; Modified: hive/branches/spark/ql/src/test/queries/clientpositive/authorization_show_grant.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/authorization_show_grant.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientpositive/authorization_show_grant.q (original) +++ hive/branches/spark/ql/src/test/queries/clientpositive/authorization_show_grant.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set hive.security.authorization.enabled=true; Modified: hive/branches/spark/ql/src/test/queries/clientpositive/authorization_view_sqlstd.q URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/authorization_view_sqlstd.q?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/queries/clientpositive/authorization_view_sqlstd.q (original) +++ hive/branches/spark/ql/src/test/queries/clientpositive/authorization_view_sqlstd.q Sun Aug 3 20:48:35 2014 @@ -1,4 +1,3 @@ -set hive.users.in.admin.role=hive_admin_user; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; set hive.security.authorization.enabled=true; Modified: hive/branches/spark/ql/src/test/results/clientpositive/annotate_stats_filter.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/annotate_stats_filter.q.out?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== Files hive/branches/spark/ql/src/test/results/clientpositive/annotate_stats_filter.q.out (original) and hive/branches/spark/ql/src/test/results/clientpositive/annotate_stats_filter.q.out Sun Aug 3 20:48:35 2014 differ Modified: hive/branches/spark/ql/src/test/results/clientpositive/annotate_stats_join.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/annotate_stats_join.q.out?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== Files hive/branches/spark/ql/src/test/results/clientpositive/annotate_stats_join.q.out (original) and hive/branches/spark/ql/src/test/results/clientpositive/annotate_stats_join.q.out Sun Aug 3 20:48:35 2014 differ Modified: hive/branches/spark/ql/src/test/results/clientpositive/authorization_9.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/authorization_9.q.out?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/clientpositive/authorization_9.q.out (original) +++ hive/branches/spark/ql/src/test/results/clientpositive/authorization_9.q.out Sun Aug 3 20:48:35 2014 @@ -9,6 +9,10 @@ create table dummy (key string, value st POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@dummy +PREHOOK: query: grant select to user hive_test_user +PREHOOK: type: GRANT_PRIVILEGE +POSTHOOK: query: grant select to user hive_test_user +POSTHOOK: type: GRANT_PRIVILEGE PREHOOK: query: grant select on database default to user hive_test_user PREHOOK: type: GRANT_PRIVILEGE POSTHOOK: query: grant select on database default to user hive_test_user @@ -39,10 +43,15 @@ PREHOOK: query: show grant user hive_tes PREHOOK: type: SHOW_GRANT POSTHOOK: query: show grant user hive_test_user on all POSTHOOK: type: SHOW_GRANT + hive_test_user USER SELECT false -1 hive_test_user default hive_test_user USER SELECT false -1 hive_test_user default dummy hive_test_user USER SELECT false -1 hive_test_user default dummy [key] hive_test_user USER SELECT false -1 hive_test_user default dummy [value] hive_test_user USER SELECT false -1 hive_test_user +PREHOOK: query: grant select to user hive_test_user2 +PREHOOK: type: GRANT_PRIVILEGE +POSTHOOK: query: grant select to user hive_test_user2 +POSTHOOK: type: GRANT_PRIVILEGE PREHOOK: query: grant select on database default to user hive_test_user2 PREHOOK: type: GRANT_PRIVILEGE POSTHOOK: query: grant select on database default to user hive_test_user2 @@ -64,6 +73,8 @@ PREHOOK: type: SHOW_GRANT POSTHOOK: query: show grant on all POSTHOOK: type: SHOW_GRANT admin ROLE ALL true -1 admin + hive_test_user USER SELECT false -1 hive_test_user + hive_test_user2 USER SELECT false -1 hive_test_user default hive_test_user USER SELECT false -1 hive_test_user default hive_test_user2 USER SELECT false -1 hive_test_user default dummy hive_test_user USER SELECT false -1 hive_test_user @@ -72,6 +83,28 @@ default dummy [key] hive_test_user USER default dummy [key] hive_test_user2 USER SELECT false -1 hive_test_user default dummy [value] hive_test_user USER SELECT false -1 hive_test_user default dummy [value] hive_test_user2 USER SELECT false -1 hive_test_user +PREHOOK: query: show grant user hive_test_user on all +PREHOOK: type: SHOW_GRANT +POSTHOOK: query: show grant user hive_test_user on all +POSTHOOK: type: SHOW_GRANT + hive_test_user USER SELECT false -1 hive_test_user +default hive_test_user USER SELECT false -1 hive_test_user +default dummy hive_test_user USER SELECT false -1 hive_test_user +default dummy [key] hive_test_user USER SELECT false -1 hive_test_user +default dummy [value] hive_test_user USER SELECT false -1 hive_test_user +PREHOOK: query: show grant user hive_test_user2 on all +PREHOOK: type: SHOW_GRANT +POSTHOOK: query: show grant user hive_test_user2 on all +POSTHOOK: type: SHOW_GRANT + hive_test_user2 USER SELECT false -1 hive_test_user +default hive_test_user2 USER SELECT false -1 hive_test_user +default dummy hive_test_user2 USER SELECT false -1 hive_test_user +default dummy [key] hive_test_user2 USER SELECT false -1 hive_test_user +default dummy [value] hive_test_user2 USER SELECT false -1 hive_test_user +PREHOOK: query: revoke select from user hive_test_user +PREHOOK: type: REVOKE_PRIVILEGE +POSTHOOK: query: revoke select from user hive_test_user +POSTHOOK: type: REVOKE_PRIVILEGE PREHOOK: query: revoke select on database default from user hive_test_user PREHOOK: type: REVOKE_PRIVILEGE POSTHOOK: query: revoke select on database default from user hive_test_user @@ -88,6 +121,10 @@ PREHOOK: Output: default@dummy POSTHOOK: query: revoke select (key, value) on table dummy from user hive_test_user POSTHOOK: type: REVOKE_PRIVILEGE POSTHOOK: Output: default@dummy +PREHOOK: query: revoke select from user hive_test_user2 +PREHOOK: type: REVOKE_PRIVILEGE +POSTHOOK: query: revoke select from user hive_test_user2 +POSTHOOK: type: REVOKE_PRIVILEGE PREHOOK: query: revoke select on database default from user hive_test_user2 PREHOOK: type: REVOKE_PRIVILEGE POSTHOOK: query: revoke select on database default from user hive_test_user2 Modified: hive/branches/spark/ql/src/test/results/clientpositive/infer_bucket_sort_dyn_part.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/infer_bucket_sort_dyn_part.q.out?rev=1615452&r1=1615451&r2=1615452&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/clientpositive/infer_bucket_sort_dyn_part.q.out (original) +++ hive/branches/spark/ql/src/test/results/clientpositive/infer_bucket_sort_dyn_part.q.out Sun Aug 3 20:48:35 2014 @@ -563,10 +563,14 @@ STAGE PLANS: Stats-Aggr Operator Stage: Stage-4 - Block level merge + Merge Work + merge level: block + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat Stage: Stage-6 - Block level merge + Merge Work + merge level: block + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat Stage: Stage-7 Move Operator
