Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java Tue Nov 18 00:48:40 2014 @@ -57,7 +57,9 @@ import org.apache.hadoop.hive.ql.udf.gen import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr; import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; @@ -420,9 +422,10 @@ public class PartitionPruner implements String defaultPartitionName = conf.getVar(HiveConf.ConfVars.DEFAULTPARTITIONNAME); List<String> partCols = extractPartColNames(tab); + List<PrimitiveTypeInfo> partColTypeInfos = extractPartColTypes(tab); boolean hasUnknownPartitions = prunePartitionNames( - partCols, prunerExpr, defaultPartitionName, partNames); + partCols, partColTypeInfos, prunerExpr, defaultPartitionName, partNames); perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.PRUNE_LISTING); perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.PARTITION_RETRIEVING); @@ -442,19 +445,30 @@ public class PartitionPruner implements return partCols; } + private static List<PrimitiveTypeInfo> extractPartColTypes(Table tab) { + List<FieldSchema> pCols = tab.getPartCols(); + List<PrimitiveTypeInfo> partColTypeInfos = new ArrayList<PrimitiveTypeInfo>(pCols.size()); + for (FieldSchema pCol : pCols) { + partColTypeInfos.add(TypeInfoFactory.getPrimitiveTypeInfo(pCol.getType())); + } + return partColTypeInfos; + } + /** * Prunes partition names to see if they match the prune expression. - * @param columnNames name of partition columns + * @param partColumnNames name of partition columns + * @param partColumnTypeInfos types of partition columns * @param prunerExpr The expression to match. * @param defaultPartitionName name of default partition * @param partNames Partition names to filter. The list is modified in place. * @return Whether the list has any partitions for which the expression may or may not match. */ - public static boolean prunePartitionNames(List<String> columnNames, ExprNodeGenericFuncDesc prunerExpr, + public static boolean prunePartitionNames(List<String> partColumnNames, + List<PrimitiveTypeInfo> partColumnTypeInfos, ExprNodeGenericFuncDesc prunerExpr, String defaultPartitionName, List<String> partNames) throws HiveException, MetaException { // Prepare the expression to filter on the columns. ObjectPair<PrimitiveObjectInspector, ExprNodeEvaluator> handle = - PartExprEvalUtils.prepareExpr(prunerExpr, columnNames); + PartExprEvalUtils.prepareExpr(prunerExpr, partColumnNames, partColumnTypeInfos); // Filter the name list. Removing elements one by one can be slow on e.g. ArrayList, // so let's create a new list and copy it if we don't have a linked list @@ -462,8 +476,8 @@ public class PartitionPruner implements List<String> partNamesSeq = inPlace ? partNames : new LinkedList<String>(partNames); // Array for the values to pass to evaluator. - ArrayList<String> values = new ArrayList<String>(columnNames.size()); - for (int i = 0; i < columnNames.size(); ++i) { + ArrayList<String> values = new ArrayList<String>(partColumnNames.size()); + for (int i = 0; i < partColumnNames.size(); ++i) { values.add(null); } @@ -473,8 +487,17 @@ public class PartitionPruner implements String partName = partIter.next(); Warehouse.makeValsFromName(partName, values); + ArrayList<Object> convertedValues = new ArrayList<Object>(values.size()); + for(int i=0; i<values.size(); i++) { + Object o = ObjectInspectorConverters.getConverter( + PrimitiveObjectInspectorFactory.javaStringObjectInspector, + PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(partColumnTypeInfos.get(i))) + .convert(values.get(i)); + convertedValues.add(o); + } + // Evaluate the expression tree. - Boolean isNeeded = (Boolean)PartExprEvalUtils.evaluateExprOnPart(handle, values); + Boolean isNeeded = (Boolean)PartExprEvalUtils.evaluateExprOnPart(handle, convertedValues); boolean isUnknown = (isNeeded == null); if (!isUnknown && !isNeeded) { partIter.remove();
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Tue Nov 18 00:48:40 2014 @@ -22,6 +22,8 @@ import static org.apache.hadoop.hive.ql. import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_DATABASEPROPERTIES; import java.io.Serializable; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; @@ -229,7 +231,7 @@ public class DDLSemanticAnalyzer extends reservedPartitionValues.add(HiveConf.getVar(conf, ConfVars.METASTORE_INT_ORIGINAL)); reservedPartitionValues.add(HiveConf.getVar(conf, ConfVars.METASTORE_INT_ARCHIVED)); reservedPartitionValues.add(HiveConf.getVar(conf, ConfVars.METASTORE_INT_EXTRACTED)); - hiveAuthorizationTaskFactory = new HiveAuthorizationTaskFactoryImpl(conf, db); + hiveAuthorizationTaskFactory = createAuthorizationTaskFactory(conf, db); } @Override @@ -3419,4 +3421,30 @@ public class DDLSemanticAnalyzer extends throw new SemanticException(e); } } + + private HiveAuthorizationTaskFactory createAuthorizationTaskFactory(HiveConf conf, Hive db) { + Class<? extends HiveAuthorizationTaskFactory> authProviderClass = conf. + getClass(HiveConf.ConfVars.HIVE_AUTHORIZATION_TASK_FACTORY.varname, + HiveAuthorizationTaskFactoryImpl.class, + HiveAuthorizationTaskFactory.class); + String msg = "Unable to create instance of " + authProviderClass.getName() + ": "; + try { + Constructor<? extends HiveAuthorizationTaskFactory> constructor = + authProviderClass.getConstructor(HiveConf.class, Hive.class); + return constructor.newInstance(conf, db); + } catch (NoSuchMethodException e) { + throw new IllegalStateException(msg + e.getMessage(), e); + } catch (SecurityException e) { + throw new IllegalStateException(msg + e.getMessage(), e); + } catch (InstantiationException e) { + throw new IllegalStateException(msg + e.getMessage(), e); + } catch (IllegalAccessException e) { + throw new IllegalStateException(msg + e.getMessage(), e); + } catch (IllegalArgumentException e) { + throw new IllegalStateException(msg + e.getMessage(), e); + } catch (InvocationTargetException e) { + throw new IllegalStateException(msg + e.getMessage(), e); + } + } + } Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g Tue Nov 18 00:48:40 2014 @@ -283,6 +283,8 @@ KW_ROLE: 'ROLE'; KW_ROLES: 'ROLES'; KW_INNER: 'INNER'; KW_EXCHANGE: 'EXCHANGE'; +KW_URI: 'URI'; +KW_SERVER : 'SERVER'; KW_ADMIN: 'ADMIN'; KW_OWNER: 'OWNER'; KW_PRINCIPALS: 'PRINCIPALS'; Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g Tue Nov 18 00:48:40 2014 @@ -109,6 +109,7 @@ TOK_DATE; TOK_DATELITERAL; TOK_DATETIME; TOK_TIMESTAMP; +TOK_TIMESTAMPLITERAL; TOK_STRING; TOK_CHAR; TOK_VARCHAR; @@ -340,6 +341,8 @@ TOK_VIRTUAL_TABLE; TOK_VIRTUAL_TABREF; TOK_ANONYMOUS; TOK_COL_NAME; +TOK_URI_TYPE; +TOK_SERVER_TYPE; } @@ -1485,11 +1488,15 @@ privilegeObject privObject : (KW_DATABASE|KW_SCHEMA) identifier -> ^(TOK_DB_TYPE identifier) | KW_TABLE? tableName partitionSpec? -> ^(TOK_TABLE_TYPE tableName partitionSpec?) + | KW_URI (path=StringLiteral) -> ^(TOK_URI_TYPE $path) + | KW_SERVER identifier -> ^(TOK_SERVER_TYPE identifier) ; privObjectCols : (KW_DATABASE|KW_SCHEMA) identifier -> ^(TOK_DB_TYPE identifier) | KW_TABLE? tableName (LPAREN cols=columnNameList RPAREN)? partitionSpec? -> ^(TOK_TABLE_TYPE tableName $cols? partitionSpec?) + | KW_URI (path=StringLiteral) -> ^(TOK_URI_TYPE $path) + | KW_SERVER identifier -> ^(TOK_SERVER_TYPE identifier) ; privilegeList Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g Tue Nov 18 00:48:40 2014 @@ -218,6 +218,7 @@ constant : Number | dateLiteral + | timestampLiteral | StringLiteral | stringLiteralSequence | BigintLiteral @@ -250,6 +251,14 @@ dateLiteral } ; +timestampLiteral + : + KW_TIMESTAMP StringLiteral -> + { + adaptor.create(TOK_TIMESTAMPLITERAL, $StringLiteral.text) + } + ; + expression @init { gParent.pushMsg("expression specification", state); } @after { gParent.popMsg(state); } @@ -260,7 +269,6 @@ expression atomExpression : KW_NULL -> TOK_NULL - | dateLiteral | constant | castExpression | caseExpression @@ -545,5 +553,5 @@ principalIdentifier nonReserved : - KW_TRUE | KW_FALSE | KW_LIKE | KW_EXISTS | KW_ASC | KW_DESC | KW_ORDER | KW_GROUP | KW_BY | KW_AS | KW_INSERT | KW_OVERWRITE | KW_OUTER | KW_LEFT | KW_RIGHT | KW_FULL | KW_PARTITION | KW_PARTITIONS | KW_TABLE | KW_TABLES | KW_COLUMNS | KW_INDEX | KW_INDEXES | KW_REBUILD | KW_FUNCTIONS | KW_SHOW | KW_MSCK | KW_REPAIR | KW_DIRECTORY | KW_LOCAL | KW_USING | KW_CLUSTER | KW_DISTRIBUTE | KW_SORT | KW_UNION | KW_LOAD | KW_EXPORT | KW_IMPORT | KW_DATA | KW_INPATH | KW_IS | KW_NULL | KW_CREATE | KW_EXTERNAL | KW_ALTER | KW_CHANGE | KW_FIRST | KW_AFTER | KW_DESCRIBE | KW_DROP | KW_RENAME | KW_IGNORE | KW_PROTECTION | KW_TO | KW_COMMENT | KW_BOOLEAN | KW_TINYINT | KW_SMALLINT | KW_INT | KW_BIGINT | KW_FLOAT | KW_DOUBLE | KW_DATE | KW_DATETIME | KW_TIMESTAMP | KW_DECIMAL | KW_STRING | KW_ARRAY | KW_STRUCT | KW_UNIONTYPE | KW_PARTITIONED | KW_CLUSTERED | KW_SORTED | KW_INTO | KW_BUCKETS | KW_ROW | KW_ROWS | KW_FORMAT | KW_DELIMITED | KW_FIELDS | KW_TERMINATED | KW_ESCAPED | KW_COLLECTION | KW_ITEMS | KW_KEYS | KW_KEY_TYPE | KW_LINES | KW_STORED | KW_FILEFORMAT | KW_INPUTFORMAT | KW_OUTPUTFORMAT | KW_INPUTDRIVER | KW_OUTPUTDRIVER | KW_OFFLINE | KW_ENABLE | KW_DISABLE | KW_READONLY | KW_NO_DROP | KW_LOCATION | KW_BUCKET | KW_OUT | KW_OF | KW_PERCENT | KW_ADD | KW_REPLACE | KW_RLIKE | KW_REGEXP | KW_TEMPORARY | KW_EXPLAIN | KW_FORMATTED | KW_PRETTY | KW_DEPENDENCY | KW_LOGICAL | KW_SERDE | KW_WITH | KW_DEFERRED | KW_SERDEPROPERTIES | KW_DBPROPERTIES | KW_LIMIT | KW_SET | KW_UNSET | KW_TBLPROPERTIES | KW_IDXPROPERTIES | KW_VALUE_TYPE | KW_ELEM_TYPE | KW_MAPJOIN | KW_STREAMTABLE | KW_HOLD_DDLTIME | KW_CLUSTERSTATUS | KW_UTC | KW_UTCTIMESTAMP | KW_LONG | KW_DELETE | KW_PLUS | KW_MINUS | KW_FETCH | KW_INTERSECT | KW_VIEW | KW_IN | KW_DATABASES | KW_MATERIALIZED | KW_SCHEMA | KW_SCHEMAS | KW_GRANT | KW_REVOKE | KW_SSL | KW_UNDO | KW_LOCK | KW_LOCKS | KW_UNLOCK | KW_SHARED | KW_EXCLUSIVE | KW_PROCEDURE | KW_UNSIGNED | KW_WHILE | KW_READ | KW_READS | KW_PURGE | KW_RANGE | KW_AN ALYZE | KW_BEFORE | KW_BETWEEN | KW_BOTH | KW_BINARY | KW_CONTINUE | KW_CURSOR | KW_TRIGGER | KW_RECORDREADER | KW_RECORDWRITER | KW_SEMI | KW_LATERAL | KW_TOUCH | KW_ARCHIVE | KW_UNARCHIVE | KW_COMPUTE | KW_STATISTICS | KW_USE | KW_OPTION | KW_CONCATENATE | KW_SHOW_DATABASE | KW_UPDATE | KW_RESTRICT | KW_CASCADE | KW_SKEWED | KW_ROLLUP | KW_CUBE | KW_DIRECTORIES | KW_FOR | KW_GROUPING | KW_SETS | KW_TRUNCATE | KW_NOSCAN | KW_USER | KW_ROLE | KW_ROLES | KW_INNER | KW_DEFINED | KW_ADMIN | KW_JAR | KW_FILE | KW_OWNER | KW_PRINCIPALS | KW_ALL | KW_DEFAULT | KW_NONE | KW_COMPACT | KW_COMPACTIONS | KW_TRANSACTIONS | KW_REWRITE | KW_AUTHORIZATION | KW_VALUES + KW_TRUE | KW_FALSE | KW_LIKE | KW_EXISTS | KW_ASC | KW_DESC | KW_ORDER | KW_GROUP | KW_BY | KW_AS | KW_INSERT | KW_OVERWRITE | KW_OUTER | KW_LEFT | KW_RIGHT | KW_FULL | KW_PARTITION | KW_PARTITIONS | KW_TABLE | KW_TABLES | KW_COLUMNS | KW_INDEX | KW_INDEXES | KW_REBUILD | KW_FUNCTIONS | KW_SHOW | KW_MSCK | KW_REPAIR | KW_DIRECTORY | KW_LOCAL | KW_USING | KW_CLUSTER | KW_DISTRIBUTE | KW_SORT | KW_UNION | KW_LOAD | KW_EXPORT | KW_IMPORT | KW_DATA | KW_INPATH | KW_IS | KW_NULL | KW_CREATE | KW_EXTERNAL | KW_ALTER | KW_CHANGE | KW_FIRST | KW_AFTER | KW_DESCRIBE | KW_DROP | KW_RENAME | KW_IGNORE | KW_PROTECTION | KW_TO | KW_COMMENT | KW_BOOLEAN | KW_TINYINT | KW_SMALLINT | KW_INT | KW_BIGINT | KW_FLOAT | KW_DOUBLE | KW_DATE | KW_DATETIME | KW_TIMESTAMP | KW_DECIMAL | KW_STRING | KW_ARRAY | KW_STRUCT | KW_UNIONTYPE | KW_PARTITIONED | KW_CLUSTERED | KW_SORTED | KW_INTO | KW_BUCKETS | KW_ROW | KW_ROWS | KW_FORMAT | KW_DELIMITED | KW_FIELDS | KW_TERMINATED | KW_ESCAPED | KW_COLLECTION | KW_ITEMS | KW_KEYS | KW_KEY_TYPE | KW_LINES | KW_STORED | KW_FILEFORMAT | KW_INPUTFORMAT | KW_OUTPUTFORMAT | KW_INPUTDRIVER | KW_OUTPUTDRIVER | KW_OFFLINE | KW_ENABLE | KW_DISABLE | KW_READONLY | KW_NO_DROP | KW_LOCATION | KW_BUCKET | KW_OUT | KW_OF | KW_PERCENT | KW_ADD | KW_REPLACE | KW_RLIKE | KW_REGEXP | KW_TEMPORARY | KW_EXPLAIN | KW_FORMATTED | KW_PRETTY | KW_DEPENDENCY | KW_LOGICAL | KW_SERDE | KW_WITH | KW_DEFERRED | KW_SERDEPROPERTIES | KW_DBPROPERTIES | KW_LIMIT | KW_SET | KW_UNSET | KW_TBLPROPERTIES | KW_IDXPROPERTIES | KW_VALUE_TYPE | KW_ELEM_TYPE | KW_MAPJOIN | KW_STREAMTABLE | KW_HOLD_DDLTIME | KW_CLUSTERSTATUS | KW_UTC | KW_UTCTIMESTAMP | KW_LONG | KW_DELETE | KW_PLUS | KW_MINUS | KW_FETCH | KW_INTERSECT | KW_VIEW | KW_IN | KW_DATABASES | KW_MATERIALIZED | KW_SCHEMA | KW_SCHEMAS | KW_GRANT | KW_REVOKE | KW_SSL | KW_UNDO | KW_LOCK | KW_LOCKS | KW_UNLOCK | KW_SHARED | KW_EXCLUSIVE | KW_PROCEDURE | KW_UNSIGNED | KW_WHILE | KW_READ | KW_READS | KW_PURGE | KW_RANGE | KW_AN ALYZE | KW_BEFORE | KW_BETWEEN | KW_BOTH | KW_BINARY | KW_CONTINUE | KW_CURSOR | KW_TRIGGER | KW_RECORDREADER | KW_RECORDWRITER | KW_SEMI | KW_LATERAL | KW_TOUCH | KW_ARCHIVE | KW_UNARCHIVE | KW_COMPUTE | KW_STATISTICS | KW_USE | KW_OPTION | KW_CONCATENATE | KW_SHOW_DATABASE | KW_UPDATE | KW_RESTRICT | KW_CASCADE | KW_SKEWED | KW_ROLLUP | KW_CUBE | KW_DIRECTORIES | KW_FOR | KW_GROUPING | KW_SETS | KW_TRUNCATE | KW_NOSCAN | KW_USER | KW_ROLE | KW_ROLES | KW_INNER | KW_DEFINED | KW_ADMIN | KW_JAR | KW_FILE | KW_OWNER | KW_PRINCIPALS | KW_ALL | KW_DEFAULT | KW_NONE | KW_COMPACT | KW_COMPACTIONS | KW_TRANSACTIONS | KW_REWRITE | KW_AUTHORIZATION | KW_VALUES | KW_URI | KW_SERVER ; Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Tue Nov 18 00:48:40 2014 @@ -51,6 +51,7 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Order; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryProperties; import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator; @@ -318,9 +319,6 @@ public class SemanticAnalyzer extends Ba private static final String VALUES_TMP_TABLE_NAME_PREFIX = "Values__Tmp__Table__"; - @VisibleForTesting - static final String ACID_TABLE_PROPERTY = "transactional"; - private HashMap<TableScanOperator, ExprNodeDesc> opToPartPruner; private HashMap<TableScanOperator, PrunedPartitionList> opToPartList; private HashMap<String, Operator<? extends OperatorDesc>> topOps; @@ -2764,11 +2762,14 @@ public class SemanticAnalyzer extends Ba // TODO: make aliases unique, otherwise needless rewriting takes place private Integer genColListRegex(String colRegex, String tabAlias, ASTNode sel, ArrayList<ExprNodeDesc> col_list, HashSet<ColumnInfo> excludeCols, RowResolver input, - Integer pos, RowResolver output, List<String> aliases, boolean ensureUniqueCols) - throws SemanticException { + RowResolver colSrcRR, Integer pos, RowResolver output, List<String> aliases, + boolean ensureUniqueCols) throws SemanticException { + if (colSrcRR == null) { + colSrcRR = input; + } // The table alias should exist - if (tabAlias != null && !input.hasTableAlias(tabAlias)) { + if (tabAlias != null && !colSrcRR.hasTableAlias(tabAlias)) { throw new SemanticException(ErrorMsg.INVALID_TABLE_ALIAS.getMsg(sel)); } @@ -2797,7 +2798,7 @@ public class SemanticAnalyzer extends Ba // For expr "*", aliases should be iterated in the order they are specified // in the query. for (String alias : aliases) { - HashMap<String, ColumnInfo> fMap = input.getFieldMap(alias); + HashMap<String, ColumnInfo> fMap = colSrcRR.getFieldMap(alias); if (fMap == null) { continue; } @@ -2808,8 +2809,11 @@ public class SemanticAnalyzer extends Ba if (excludeCols != null && excludeCols.contains(colInfo)) { continue; // This was added during plan generation. } + // First, look up the column from the source against which * is to be resolved. + // We'd later translated this into the column from proper input, if it's valid. + // TODO: excludeCols may be possible to remove using the same technique. String name = colInfo.getInternalName(); - String[] tmp = input.reverseLookup(name); + String[] tmp = colSrcRR.reverseLookup(name); // Skip the colinfos which are not for this particular alias if (tabAlias != null && !tmp[0].equalsIgnoreCase(tabAlias)) { @@ -2825,6 +2829,27 @@ public class SemanticAnalyzer extends Ba continue; } + // If input (GBY) is different than the source of columns, find the same column in input. + // TODO: This is fraught with peril. + if (input != colSrcRR) { + colInfo = input.get(tabAlias, tmp[1]); + if (colInfo == null) { + LOG.error("Cannot find colInfo for " + tabAlias + "." + tmp[1] + + ", derived from [" + colSrcRR + "], in [" + input + "]"); + throw new SemanticException(ErrorMsg.NON_KEY_EXPR_IN_GROUPBY, tmp[1]); + } + String oldCol = null; + if (LOG.isDebugEnabled()) { + oldCol = name + " => " + (tmp == null ? "null" : (tmp[0] + "." + tmp[1])); + } + name = colInfo.getInternalName(); + tmp = input.reverseLookup(name); + if (LOG.isDebugEnabled()) { + String newCol = name + " => " + (tmp == null ? "null" : (tmp[0] + "." + tmp[1])); + LOG.debug("Translated [" + oldCol + "] to [" + newCol + "]"); + } + } + ColumnInfo oColInfo = inputColsProcessed.get(colInfo); if (oColInfo == null) { ExprNodeColumnDesc expr = new ExprNodeColumnDesc(colInfo.getType(), @@ -3423,11 +3448,10 @@ public class SemanticAnalyzer extends Ba } - private Operator<?> genSelectPlan(String dest, QB qb, Operator<?> input) - throws SemanticException { + private Operator<?> genSelectPlan(String dest, QB qb, Operator<?> input, + Operator<?> inputForSelectStar) throws SemanticException { ASTNode selExprList = qb.getParseInfo().getSelForClause(dest); - - Operator<?> op = genSelectPlan(selExprList, qb, input, false); + Operator<?> op = genSelectPlan(selExprList, qb, input, inputForSelectStar, false); if (LOG.isDebugEnabled()) { LOG.debug("Created Select Plan for clause: " + dest); @@ -3437,8 +3461,8 @@ public class SemanticAnalyzer extends Ba } @SuppressWarnings("nls") - private Operator<?> genSelectPlan(ASTNode selExprList, QB qb, - Operator<?> input, boolean outerLV) throws SemanticException { + private Operator<?> genSelectPlan(ASTNode selExprList, QB qb, Operator<?> input, + Operator<?> inputForSelectStar, boolean outerLV) throws SemanticException { if (LOG.isDebugEnabled()) { LOG.debug("tree: " + selExprList.toStringTree()); @@ -3449,6 +3473,10 @@ public class SemanticAnalyzer extends Ba ASTNode trfm = null; Integer pos = Integer.valueOf(0); RowResolver inputRR = opParseCtx.get(input).getRowResolver(); + RowResolver starRR = null; + if (inputForSelectStar != null && inputForSelectStar != input) { + starRR = opParseCtx.get(inputForSelectStar).getRowResolver(); + } // SELECT * or SELECT TRANSFORM(*) boolean selectStar = false; int posn = 0; @@ -3494,7 +3522,7 @@ public class SemanticAnalyzer extends Ba } if (isUDTF && (selectStar = udtfExprType == HiveParser.TOK_FUNCTIONSTAR)) { genColListRegex(".*", null, (ASTNode) udtfExpr.getChild(0), - col_list, null, inputRR, pos, out_rwsch, qb.getAliases(), false); + col_list, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false); } } @@ -3547,7 +3575,7 @@ public class SemanticAnalyzer extends Ba } if (LOG.isDebugEnabled()) { - LOG.debug("genSelectPlan: input = " + inputRR.toString()); + LOG.debug("genSelectPlan: input = " + inputRR + " starRr = " + starRR); } // For UDTF's, skip the function name to get the expressions @@ -3616,7 +3644,7 @@ public class SemanticAnalyzer extends Ba if (expr.getType() == HiveParser.TOK_ALLCOLREF) { pos = genColListRegex(".*", expr.getChildCount() == 0 ? null : getUnescapedName((ASTNode) expr.getChild(0)).toLowerCase(), - expr, col_list, null, inputRR, pos, out_rwsch, qb.getAliases(), false); + expr, col_list, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false); selectStar = true; } else if (expr.getType() == HiveParser.TOK_TABLE_OR_COL && !hasAsClause && !inputRR.getIsExprResolver() @@ -3625,7 +3653,7 @@ public class SemanticAnalyzer extends Ba // This can only happen without AS clause // We don't allow this for ExprResolver - the Group By case pos = genColListRegex(unescapeIdentifier(expr.getChild(0).getText()), - null, expr, col_list, null, inputRR, pos, out_rwsch, qb.getAliases(), false); + null, expr, col_list, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false); } else if (expr.getType() == HiveParser.DOT && expr.getChild(0).getType() == HiveParser.TOK_TABLE_OR_COL && inputRR.hasTableAlias(unescapeIdentifier(expr.getChild(0) @@ -3637,7 +3665,7 @@ public class SemanticAnalyzer extends Ba // We don't allow this for ExprResolver - the Group By case pos = genColListRegex(unescapeIdentifier(expr.getChild(1).getText()), unescapeIdentifier(expr.getChild(0).getChild(0).getText().toLowerCase()), - expr, col_list, null, inputRR, pos, out_rwsch, qb.getAliases(), false); + expr, col_list, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false); } else { // Case when this is an expression TypeCheckCtx tcCtx = new TypeCheckCtx(inputRR); @@ -5165,7 +5193,8 @@ public class SemanticAnalyzer extends Ba Operator groupByOperatorInfo = genGroupByPlanGroupByOperator(parseInfo, dest, curr, reduceSinkOperatorInfo, GroupByDesc.Mode.COMPLETE, null); - curr = genPostGroupByBodyPlan(groupByOperatorInfo, dest, qb, aliasToOpInfo); + // TODO: should we pass curr instead of null? + curr = genPostGroupByBodyPlan(groupByOperatorInfo, dest, qb, aliasToOpInfo, null); } return curr; @@ -8782,7 +8811,7 @@ public class SemanticAnalyzer extends Ba for (String dest : ks) { curr = input; curr = genGroupByPlan2MRMultiGroupBy(dest, qb, curr); - curr = genSelectPlan(dest, qb, curr); + curr = genSelectPlan(dest, qb, curr, null); // TODO: we may need to pass "input" here instead of null Integer limit = qbp.getDestLimit(dest); if (limit != null) { curr = genLimitMapRedPlan(dest, qb, curr, limit.intValue(), true); @@ -8839,6 +8868,8 @@ public class SemanticAnalyzer extends Ba ASTNode whereExpr = qb.getParseInfo().getWhrForClause(dest); curr = genFilterPlan((ASTNode) whereExpr.getChild(0), qb, curr, aliasToOpInfo, false); } + // Preserve operator before the GBY - we'll use it to resolve '*' + Operator<?> gbySource = curr; if (qbp.getAggregationExprsForClause(dest).size() != 0 || getGroupByForClause(qbp, dest).size() > 0) { @@ -8863,8 +8894,12 @@ public class SemanticAnalyzer extends Ba curr = genGroupByPlan1MR(dest, qb, curr); } } + if (LOG.isDebugEnabled()) { + LOG.debug("RR before GB " + opParseCtx.get(gbySource).getRowResolver() + + " after GB " + opParseCtx.get(curr).getRowResolver()); + } - curr = genPostGroupByBodyPlan(curr, dest, qb, aliasToOpInfo); + curr = genPostGroupByBodyPlan(curr, dest, qb, aliasToOpInfo, gbySource); } } else { curr = genGroupByPlan1ReduceMultiGBY(commonGroupByDestGroup, qb, input, aliasToOpInfo); @@ -8891,7 +8926,7 @@ public class SemanticAnalyzer extends Ba } private Operator genPostGroupByBodyPlan(Operator curr, String dest, QB qb, - Map<String, Operator> aliasToOpInfo) + Map<String, Operator> aliasToOpInfo, Operator gbySource) throws SemanticException { QBParseInfo qbp = qb.getParseInfo(); @@ -8909,7 +8944,7 @@ public class SemanticAnalyzer extends Ba curr = genWindowingPlan(qb.getWindowingSpec(dest), curr); } - curr = genSelectPlan(dest, qb, curr); + curr = genSelectPlan(dest, qb, curr, gbySource); Integer limit = qbp.getDestLimit(dest); // Expressions are not supported currently without a alias. @@ -9866,7 +9901,7 @@ public class SemanticAnalyzer extends Ba // Get the UDTF Path QB blankQb = new QB(null, null, false); Operator udtfPath = genSelectPlan((ASTNode) lateralViewTree - .getChild(0), blankQb, lvForward, + .getChild(0), blankQb, lvForward, null, lateralViewTree.getType() == HiveParser.TOK_LATERAL_VIEW_OUTER); // add udtf aliases to QB for (String udtfAlias : blankQb.getAliases()) { @@ -12415,7 +12450,9 @@ public class SemanticAnalyzer extends Ba private boolean isAcidTable(Table tab) { if (tab == null) return false; if (!SessionState.get().getTxnMgr().supportsAcid()) return false; - return tab.getProperty(ACID_TABLE_PROPERTY) != null; + String tableIsTransactional = + tab.getProperty(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL); + return tableIsTransactional != null && tableIsTransactional.equalsIgnoreCase("true"); } private boolean isAcidOutputFormat(Class<? extends HiveOutputFormat> of) { @@ -14203,8 +14240,8 @@ public class SemanticAnalyzer extends Ba if (expr.getType() == HiveParser.TOK_ALLCOLREF) { pos = genColListRegex(".*", expr.getChildCount() == 0 ? null : getUnescapedName((ASTNode) expr.getChild(0)) - .toLowerCase(), expr, col_list, excludedColumns, inputRR, pos, out_rwsch, - tabAliasesForAllProjs, true); + .toLowerCase(), expr, col_list, excludedColumns, inputRR, null, pos, + out_rwsch, tabAliasesForAllProjs, true); selectStar = true; } else if (expr.getType() == HiveParser.TOK_TABLE_OR_COL && !hasAsClause && !inputRR.getIsExprResolver() @@ -14213,7 +14250,8 @@ public class SemanticAnalyzer extends Ba // This can only happen without AS clause // We don't allow this for ExprResolver - the Group By case pos = genColListRegex(unescapeIdentifier(expr.getChild(0).getText()), null, expr, - col_list, excludedColumns, inputRR, pos, out_rwsch, tabAliasesForAllProjs, true); + col_list, excludedColumns, inputRR, null, pos, out_rwsch, tabAliasesForAllProjs, + true); } else if (expr.getType() == HiveParser.DOT && expr.getChild(0).getType() == HiveParser.TOK_TABLE_OR_COL && inputRR.hasTableAlias(unescapeIdentifier(expr.getChild(0).getChild(0).getText() @@ -14224,7 +14262,8 @@ public class SemanticAnalyzer extends Ba // We don't allow this for ExprResolver - the Group By case pos = genColListRegex(unescapeIdentifier(expr.getChild(1).getText()), unescapeIdentifier(expr.getChild(0).getChild(0).getText().toLowerCase()), expr, - col_list, excludedColumns, inputRR, pos, out_rwsch, tabAliasesForAllProjs, true); + col_list, excludedColumns, inputRR, null, pos, out_rwsch, tabAliasesForAllProjs, + true); } else if (expr.toStringTree().contains("TOK_FUNCTIONDI") && !(srcRel instanceof HiveAggregateRel)) { // Likely a malformed query eg, select hash(distinct c1) from t1; throw new OptiqSemanticException("Distinct without an aggreggation."); Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java Tue Nov 18 00:48:40 2014 @@ -150,6 +150,7 @@ public class TezCompiler extends TaskCom LOG.info("Found cycle in operator plan..."); cycleFree = false; removeEventOperator(component); + break; } } LOG.info("Cycle free: " + cycleFree); @@ -227,7 +228,7 @@ public class TezCompiler extends TaskCom for (Operator<?> child : children) { if (!indexes.containsKey(child)) { connect(child, index, nodes, indexes, lowLinks, components); - lowLinks.put(child, Math.min(lowLinks.get(o), lowLinks.get(child))); + lowLinks.put(o, Math.min(lowLinks.get(o), lowLinks.get(child))); } else if (nodes.contains(child)) { lowLinks.put(o, Math.min(lowLinks.get(o), indexes.get(child))); } Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java Tue Nov 18 00:48:40 2014 @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.parse; import java.sql.Date; +import java.sql.Timestamp; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -73,6 +74,8 @@ import org.apache.hadoop.hive.serde2.typ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; +import com.google.common.collect.Lists; + /** * The Factory for creating typecheck processors. The typecheck processors are @@ -170,7 +173,8 @@ public class TypeCheckProcFactory { tf.getStrExprProcessor()); opRules.put(new RuleRegExp("R4", HiveParser.KW_TRUE + "%|" + HiveParser.KW_FALSE + "%"), tf.getBoolExprProcessor()); - opRules.put(new RuleRegExp("R5", HiveParser.TOK_DATELITERAL + "%"), tf.getDateExprProcessor()); + opRules.put(new RuleRegExp("R5", HiveParser.TOK_DATELITERAL + "%|" + + HiveParser.TOK_TIMESTAMPLITERAL + "%"), tf.getDateTimeExprProcessor()); opRules.put(new RuleRegExp("R6", HiveParser.TOK_TABLE_OR_COL + "%"), tf.getColumnExprProcessor()); opRules.put(new RuleRegExp("R7", HiveParser.TOK_SUBQUERY_OP + "%"), @@ -182,9 +186,8 @@ public class TypeCheckProcFactory { opRules, tcCtx); GraphWalker ogw = new DefaultGraphWalker(disp); - // Create a list of topop nodes - ArrayList<Node> topNodes = new ArrayList<Node>(); - topNodes.add(expr); + // Create a list of top nodes + ArrayList<Node> topNodes = Lists.<Node>newArrayList(expr); HashMap<Node, Object> nodeOutputs = new LinkedHashMap<Node, Object>(); ogw.startWalking(topNodes, nodeOutputs); @@ -420,7 +423,7 @@ public class TypeCheckProcFactory { /** * Processor for date constants. */ - public static class DateExprProcessor implements NodeProcessor { + public static class DateTimeExprProcessor implements NodeProcessor { @Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, @@ -437,14 +440,24 @@ public class TypeCheckProcFactory { } ASTNode expr = (ASTNode) nd; + String timeString = BaseSemanticAnalyzer.stripQuotes(expr.getText()); // Get the string value and convert to a Date value. try { - String dateString = BaseSemanticAnalyzer.stripQuotes(expr.getText()); - Date date = Date.valueOf(dateString); - return new ExprNodeConstantDesc(TypeInfoFactory.dateTypeInfo, date); - } catch (IllegalArgumentException err) { - throw new SemanticException("Unable to convert date literal string to date value.", err); + // todo replace below with joda-time, which supports timezone + if (expr.getType() == HiveParser.TOK_DATELITERAL) { + PrimitiveTypeInfo typeInfo = TypeInfoFactory.dateTypeInfo; + return new ExprNodeConstantDesc(typeInfo, + Date.valueOf(timeString)); + } + if (expr.getType() == HiveParser.TOK_TIMESTAMPLITERAL) { + return new ExprNodeConstantDesc(TypeInfoFactory.timestampTypeInfo, + Timestamp.valueOf(timeString)); + } + throw new IllegalArgumentException("Invalid time literal type " + expr.getType()); + } catch (Exception err) { + throw new SemanticException( + "Unable to convert time literal '" + timeString + "' to time value.", err); } } } @@ -454,8 +467,8 @@ public class TypeCheckProcFactory { * * @return DateExprProcessor. */ - public DateExprProcessor getDateExprProcessor() { - return new DateExprProcessor(); + public DateTimeExprProcessor getDateTimeExprProcessor() { + return new DateTimeExprProcessor(); } /** Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactory.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactory.java?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactory.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactory.java Tue Nov 18 00:48:40 2014 @@ -21,6 +21,8 @@ import java.io.Serializable; import java.util.HashSet; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate; +import org.apache.hadoop.hive.common.classification.InterfaceStability.Evolving; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; @@ -32,6 +34,8 @@ import org.apache.hadoop.hive.ql.parse.S * tasks. Every method in this class may return null, indicating no task * needs to be executed or can throw a SemanticException. */ +@LimitedPrivate(value = { "Apache Hive, Apache Sentry (incubating)" }) +@Evolving public interface HiveAuthorizationTaskFactory { public Task<? extends Serializable> createCreateRoleTask(ASTNode node, HashSet<ReadEntity> inputs, HashSet<WriteEntity> outputs) throws SemanticException; Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactoryImpl.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactoryImpl.java?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactoryImpl.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactoryImpl.java Tue Nov 18 00:48:40 2014 @@ -238,7 +238,7 @@ public class HiveAuthorizationTaskFactor return subject; } - private PrivilegeObjectDesc parsePrivObject(ASTNode ast) throws SemanticException { + protected PrivilegeObjectDesc parsePrivObject(ASTNode ast) throws SemanticException { PrivilegeObjectDesc subject = new PrivilegeObjectDesc(); ASTNode child = (ASTNode) ast.getChild(0); ASTNode gchild = (ASTNode)child.getChild(0); @@ -246,6 +246,8 @@ public class HiveAuthorizationTaskFactor subject.setTable(true); String[] qualified = BaseSemanticAnalyzer.getQualifiedTableName(gchild); subject.setObject(BaseSemanticAnalyzer.getDotName(qualified)); + } else if (child.getType() == HiveParser.TOK_URI_TYPE || child.getType() == HiveParser.TOK_SERVER_TYPE) { + throw new SemanticException("Hive authorization does not support the URI or SERVER objects"); } else { subject.setTable(false); subject.setObject(BaseSemanticAnalyzer.unescapeIdentifier(gchild.getText())); Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java Tue Nov 18 00:48:40 2014 @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.hive.ql.plan; import java.util.ArrayList; Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java Tue Nov 18 00:48:40 2014 @@ -134,6 +134,7 @@ public class JDBCStatsAggregator impleme } }; + fileID = JDBCStatsUtils.truncateRowId(fileID); String keyPrefix = Utilities.escapeSqlLike(fileID) + "%"; for (int failures = 0;; failures++) { try { @@ -147,7 +148,7 @@ public class JDBCStatsAggregator impleme if (result.next()) { retval = result.getLong(1); } else { - LOG.warn("Warning. Nothing published. Nothing aggregated."); + LOG.warn("Nothing published. Nothing aggregated."); return null; } return Long.toString(retval); @@ -217,6 +218,7 @@ public class JDBCStatsAggregator impleme }; try { + rowID = JDBCStatsUtils.truncateRowId(rowID); String keyPrefix = Utilities.escapeSqlLike(rowID) + "%"; PreparedStatement delStmt = Utilities.prepareWithRetry(conn, Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java Tue Nov 18 00:48:40 2014 @@ -139,7 +139,11 @@ public class JDBCStatsPublisher implemen + " stats: " + JDBCStatsUtils.getSupportedStatistics()); return false; } - LOG.info("Stats publishing for key " + fileID); + String rowId = JDBCStatsUtils.truncateRowId(fileID); + if (LOG.isInfoEnabled()) { + String truncateSuffix = (rowId != fileID) ? " (from " + fileID + ")" : ""; // object equality + LOG.info("Stats publishing for key " + rowId + truncateSuffix); + } Utilities.SQLCommand<Void> execUpdate = new Utilities.SQLCommand<Void>() { @Override @@ -153,7 +157,7 @@ public class JDBCStatsPublisher implemen for (int failures = 0;; failures++) { try { - insStmt.setString(1, fileID); + insStmt.setString(1, rowId); for (int i = 0; i < JDBCStatsUtils.getSupportedStatistics().size(); i++) { insStmt.setString(i + 2, stats.get(supportedStatistics.get(i))); } @@ -172,10 +176,10 @@ public class JDBCStatsPublisher implemen for (i = 0; i < JDBCStatsUtils.getSupportedStatistics().size(); i++) { updStmt.setString(i + 1, stats.get(supportedStatistics.get(i))); } - updStmt.setString(supportedStatistics.size() + 1, fileID); + updStmt.setString(supportedStatistics.size() + 1, rowId); updStmt.setString(supportedStatistics.size() + 2, stats.get(JDBCStatsUtils.getBasicStat())); - updStmt.setString(supportedStatistics.size() + 3, fileID); + updStmt.setString(supportedStatistics.size() + 3, rowId); Utilities.executeWithRetry(execUpdate, updStmt, waitWindow, maxRetries); return true; } catch (SQLRecoverableException ue) { Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsSetupConstants.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsSetupConstants.java?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsSetupConstants.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsSetupConstants.java Tue Nov 18 00:48:40 2014 @@ -34,4 +34,7 @@ public final class JDBCStatsSetupConstan public static final String PART_STAT_RAW_DATA_SIZE_COLUMN_NAME = "RAW_DATA_SIZE"; + // 255 is an old value that we will keep for now; it can be increased to 4000; limits are + // MySQL - 65535, SQL Server - 8000, Oracle - 4000, Derby - 32762, Postgres - large. + public static final int ID_COLUMN_VARCHAR_SIZE = 255; } Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsUtils.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsUtils.java?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsUtils.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsUtils.java Tue Nov 18 00:48:40 2014 @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.hadoop.util.hash.MurmurHash; public class JDBCStatsUtils { @@ -124,9 +125,10 @@ public class JDBCStatsUtils { * Prepares CREATE TABLE query */ public static String getCreate(String comment) { - String create = "CREATE TABLE /* " + comment + " */ " + JDBCStatsUtils.getStatTableName() + - " (" + getTimestampColumnName() + " TIMESTAMP DEFAULT CURRENT_TIMESTAMP, " + - JDBCStatsUtils.getIdColumnName() + " VARCHAR(255) PRIMARY KEY "; + String create = "CREATE TABLE /* " + comment + " */ " + JDBCStatsUtils.getStatTableName() + + " (" + getTimestampColumnName() + " TIMESTAMP DEFAULT CURRENT_TIMESTAMP, " + + JDBCStatsUtils.getIdColumnName() + " VARCHAR(" + + JDBCStatsSetupConstants.ID_COLUMN_VARCHAR_SIZE + ") PRIMARY KEY "; for (int i = 0; i < supportedStats.size(); i++) { create += ", " + getStatColumnName(supportedStats.get(i)) + " BIGINT "; } @@ -191,4 +193,13 @@ public class JDBCStatsUtils { return delete; } + /** + * Make sure the row ID fits into the row ID column in the table. + * @param rowId Row ID. + * @return Resulting row ID truncated to correct length, if necessary. + */ + public static String truncateRowId(String rowId) { + return (rowId.length() <= JDBCStatsSetupConstants.ID_COLUMN_VARCHAR_SIZE) + ? rowId : Integer.toHexString(MurmurHash.getInstance().hash(rowId.getBytes())); + } } Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java Tue Nov 18 00:48:40 2014 @@ -69,7 +69,7 @@ public class Cleaner extends CompactorTh // This is solely for testing. It checks if the test has set the looped value to false, // and if so remembers that and then sets it to true at the end. We have to check here // first to make sure we go through a complete iteration of the loop before resetting it. - boolean setLooped = !looped.boolVal; + boolean setLooped = !looped.get(); // Make sure nothing escapes this run method and kills the metastore at large, // so wrap it in a big catch Throwable statement. try { @@ -137,16 +137,16 @@ public class Cleaner extends CompactorTh // Now, go back to bed until it's time to do this again long elapsedTime = System.currentTimeMillis() - startedAt; - if (elapsedTime >= cleanerCheckInterval || stop.boolVal) continue; + if (elapsedTime >= cleanerCheckInterval || stop.get()) continue; else Thread.sleep(cleanerCheckInterval - elapsedTime); } catch (Throwable t) { LOG.error("Caught an exception in the main loop of compactor cleaner, " + StringUtils.stringifyException(t)); } if (setLooped) { - looped.boolVal = true; + looped.set(true); } - } while (!stop.boolVal); + } while (!stop.get()); } private Set<Long> findRelatedLocks(CompactionInfo ci, ShowLocksResponse locksResponse) { Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java Tue Nov 18 00:48:40 2014 @@ -40,6 +40,7 @@ import java.io.IOException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; /** * Superclass for all threads in the compactor. @@ -52,8 +53,8 @@ abstract class CompactorThread extends T protected CompactionTxnHandler txnHandler; protected RawStore rs; protected int threadId; - protected BooleanPointer stop; - protected BooleanPointer looped; + protected AtomicBoolean stop; + protected AtomicBoolean looped; @Override public void setHiveConf(HiveConf conf) { @@ -67,7 +68,7 @@ abstract class CompactorThread extends T } @Override - public void init(BooleanPointer stop, BooleanPointer looped) throws MetaException { + public void init(AtomicBoolean stop, AtomicBoolean looped) throws MetaException { this.stop = stop; this.looped = looped; setPriority(MIN_PRIORITY); Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java Tue Nov 18 00:48:40 2014 @@ -33,6 +33,7 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.metastore.txn.CompactionInfo; import org.apache.hadoop.hive.metastore.txn.TxnHandler; import org.apache.hadoop.hive.ql.io.AcidUtils; @@ -44,6 +45,7 @@ import java.security.PrivilegedException import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; /** * A class to initiate compactions. This will run in a separate thread. @@ -52,8 +54,6 @@ public class Initiator extends Compactor static final private String CLASS_NAME = Initiator.class.getName(); static final private Log LOG = LogFactory.getLog(CLASS_NAME); - static final private String NO_COMPACTION = "NO_AUTO_COMPACTION"; - private long checkInterval; @Override @@ -85,9 +85,8 @@ public class Initiator extends Compactor try { Table t = resolveTable(ci); // check if no compaction set for this table - if (t.getParameters().get(NO_COMPACTION) != null) { - LOG.info("Table " + tableName(t) + " marked " + NO_COMPACTION + - " so we will not compact it."); + if (noAutoCompactSet(t)) { + LOG.info("Table " + tableName(t) + " marked true so we will not compact it."); continue; } @@ -126,10 +125,10 @@ public class Initiator extends Compactor } long elapsedTime = System.currentTimeMillis() - startedAt; - if (elapsedTime >= checkInterval || stop.boolVal) continue; + if (elapsedTime >= checkInterval || stop.get()) continue; else Thread.sleep(checkInterval - elapsedTime); - } while (!stop.boolVal); + } while (!stop.get()); } catch (Throwable t) { LOG.error("Caught an exception in the main loop of compactor initiator, exiting " + StringUtils.stringifyException(t)); @@ -137,7 +136,7 @@ public class Initiator extends Compactor } @Override - public void init(BooleanPointer stop, BooleanPointer looped) throws MetaException { + public void init(AtomicBoolean stop, AtomicBoolean looped) throws MetaException { super.init(stop, looped); checkInterval = conf.getTimeVar(HiveConf.ConfVars.HIVE_COMPACTOR_CHECK_INTERVAL, TimeUnit.MILLISECONDS) ; @@ -278,4 +277,16 @@ public class Initiator extends Compactor rqst.setRunas(runAs); txnHandler.compact(rqst); } + + // Because TABLE_NO_AUTO_COMPACT was originally assumed to be NO_AUTO_COMPACT and then was moved + // to no_auto_compact, we need to check it in both cases. + private boolean noAutoCompactSet(Table t) { + String noAutoCompact = + t.getParameters().get(hive_metastoreConstants.TABLE_NO_AUTO_COMPACT); + if (noAutoCompact == null) { + noAutoCompact = + t.getParameters().get(hive_metastoreConstants.TABLE_NO_AUTO_COMPACT.toUpperCase()); + } + return noAutoCompact != null && noAutoCompact.equalsIgnoreCase("true"); + } } Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java (original) +++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java Tue Nov 18 00:48:40 2014 @@ -42,6 +42,7 @@ import java.security.PrivilegedException import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; /** * A class to do compactions. This will run in a separate thread. It will spin on the @@ -77,7 +78,7 @@ public class Worker extends CompactorThr do { CompactionInfo ci = txnHandler.findNextToCompact(name); - if (ci == null && !stop.boolVal) { + if (ci == null && !stop.get()) { try { Thread.sleep(SLEEP_TIME); continue; @@ -160,7 +161,7 @@ public class Worker extends CompactorThr ". Marking clean to avoid repeated failures, " + StringUtils.stringifyException(e)); txnHandler.markCleaned(ci); } - } while (!stop.boolVal); + } while (!stop.get()); } catch (Throwable t) { LOG.error("Caught an exception in the main loop of compactor worker " + name + ", exiting " + StringUtils.stringifyException(t)); @@ -168,7 +169,7 @@ public class Worker extends CompactorThr } @Override - public void init(BooleanPointer stop, BooleanPointer looped) throws MetaException { + public void init(AtomicBoolean stop, AtomicBoolean looped) throws MetaException { super.init(stop, looped); StringBuilder name = new StringBuilder(hostname()); Modified: hive/branches/spark/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto (original) +++ hive/branches/spark/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto Tue Nov 18 00:48:40 2014 @@ -191,8 +191,15 @@ message PostScript { optional uint64 footerLength = 1; optional CompressionKind compression = 2; optional uint64 compressionBlockSize = 3; + // the version of the file format + // [0, 11] = Hive 0.11 + // [0, 12] = Hive 0.12 repeated uint32 version = 4 [packed = true]; optional uint64 metadataLength = 5; + // Version of the writer: + // 0 (or missing) = original + // 1 = HIVE-8732 fixed + optional uint32 writerVersion = 6; // Leave this last in the record optional string magic = 8000; } Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java (original) +++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java Tue Nov 18 00:48:40 2014 @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.hive.ql.exec; import org.apache.commons.logging.Log; Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestDecimalUtil.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestDecimalUtil.java?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestDecimalUtil.java (original) +++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestDecimalUtil.java Tue Nov 18 00:48:40 2014 @@ -38,8 +38,9 @@ public class TestDecimalUtil { DecimalUtil.floor(0, d1, dcv); Assert.assertEquals(0, expected1.compareTo(dcv.vector[0].getHiveDecimal())); + // As of HIVE-8745, these decimal values should be trimmed of trailing zeros. HiveDecimal d2 = HiveDecimal.create("23.00000"); - Assert.assertEquals(5, d2.scale()); + Assert.assertEquals(0, d2.scale()); HiveDecimal expected2 = HiveDecimal.create("23"); DecimalUtil.floor(0, d2, dcv); Assert.assertEquals(0, expected2.compareTo(dcv.vector[0].getHiveDecimal())); @@ -50,19 +51,19 @@ public class TestDecimalUtil { Assert.assertEquals(0, expected3.compareTo(dcv.vector[0].getHiveDecimal())); HiveDecimal d4 = HiveDecimal.create("-17.00000"); - Assert.assertEquals(5, d4.scale()); + Assert.assertEquals(0, d4.scale()); HiveDecimal expected4 = HiveDecimal.create("-17"); DecimalUtil.floor(0, d4, dcv); Assert.assertEquals(0, expected4.compareTo(dcv.vector[0].getHiveDecimal())); HiveDecimal d5 = HiveDecimal.create("-0.30000"); - Assert.assertEquals(5, d5.scale()); + Assert.assertEquals(1, d5.scale()); HiveDecimal expected5 = HiveDecimal.create("-1"); DecimalUtil.floor(0, d5, dcv); Assert.assertEquals(0, expected5.compareTo(dcv.vector[0].getHiveDecimal())); HiveDecimal d6 = HiveDecimal.create("0.30000"); - Assert.assertEquals(5, d6.scale()); + Assert.assertEquals(1, d6.scale()); HiveDecimal expected6 = HiveDecimal.create("0"); DecimalUtil.floor(0, d6, dcv); Assert.assertEquals(0, expected6.compareTo(dcv.vector[0].getHiveDecimal())); @@ -76,8 +77,9 @@ public class TestDecimalUtil { DecimalUtil.ceiling(0, d1, dcv); Assert.assertEquals(0, expected1.compareTo(dcv.vector[0].getHiveDecimal())); + // As of HIVE-8745, these decimal values should be trimmed of trailing zeros. HiveDecimal d2 = HiveDecimal.create("23.00000"); - Assert.assertEquals(5, d2.scale()); + Assert.assertEquals(0, d2.scale()); HiveDecimal expected2 = HiveDecimal.create("23"); DecimalUtil.ceiling(0, d2, dcv); Assert.assertEquals(0, expected2.compareTo(dcv.vector[0].getHiveDecimal())); @@ -88,19 +90,19 @@ public class TestDecimalUtil { Assert.assertEquals(0, expected3.compareTo(dcv.vector[0].getHiveDecimal())); HiveDecimal d4 = HiveDecimal.create("-17.00000"); - Assert.assertEquals(5, d4.scale()); + Assert.assertEquals(0, d4.scale()); HiveDecimal expected4 = HiveDecimal.create("-17"); DecimalUtil.ceiling(0, d4, dcv); Assert.assertEquals(0, expected4.compareTo(dcv.vector[0].getHiveDecimal())); HiveDecimal d5 = HiveDecimal.create("-0.30000"); - Assert.assertEquals(5, d5.scale()); + Assert.assertEquals(1, d5.scale()); HiveDecimal expected5 = HiveDecimal.create("0"); DecimalUtil.ceiling(0, d5, dcv); Assert.assertEquals(0, expected5.compareTo(dcv.vector[0].getHiveDecimal())); HiveDecimal d6 = HiveDecimal.create("0.30000"); - Assert.assertEquals(5, d6.scale()); + Assert.assertEquals(1, d6.scale()); HiveDecimal expected6 = HiveDecimal.create("1"); DecimalUtil.ceiling(0, d6, dcv); Assert.assertEquals(0, expected6.compareTo(dcv.vector[0].getHiveDecimal())); @@ -127,8 +129,9 @@ public class TestDecimalUtil { DecimalUtil.round(0, d1, dcv); Assert.assertEquals(0, expected1.compareTo(dcv.vector[0].getHiveDecimal())); + // As of HIVE-8745, these decimal values should be trimmed of trailing zeros. HiveDecimal d2 = HiveDecimal.create("23.00000"); - Assert.assertEquals(5, d2.scale()); + Assert.assertEquals(0, d2.scale()); HiveDecimal expected2 = HiveDecimal.create("23"); DecimalUtil.round(0, d2, dcv); Assert.assertEquals(0, expected2.compareTo(dcv.vector[0].getHiveDecimal())); @@ -139,7 +142,7 @@ public class TestDecimalUtil { Assert.assertEquals(0, expected3.compareTo(dcv.vector[0].getHiveDecimal())); HiveDecimal d4 = HiveDecimal.create("-17.00000"); - Assert.assertEquals(5, d4.scale()); + Assert.assertEquals(0, d4.scale()); HiveDecimal expected4 = HiveDecimal.create("-17"); DecimalUtil.round(0, d4, dcv); Assert.assertEquals(0, expected4.compareTo(dcv.vector[0].getHiveDecimal())); @@ -163,8 +166,9 @@ public class TestDecimalUtil { DecimalUtil.round(0, d1, dcv); Assert.assertEquals(0, expected1.compareTo(dcv.vector[0].getHiveDecimal())); + // As of HIVE-8745, these decimal values should be trimmed of trailing zeros. HiveDecimal d2 = HiveDecimal.create("23.56700"); - Assert.assertEquals(5, d2.scale()); + Assert.assertEquals(3, d2.scale()); HiveDecimal expected2 = HiveDecimal.create("23.567"); DecimalUtil.round(0, d2, dcv); Assert.assertEquals(0, expected2.compareTo(dcv.vector[0].getHiveDecimal())); @@ -175,7 +179,7 @@ public class TestDecimalUtil { Assert.assertEquals(0, expected3.compareTo(dcv.vector[0].getHiveDecimal())); HiveDecimal d4 = HiveDecimal.create("-17.23400"); - Assert.assertEquals(5, d4.scale()); + Assert.assertEquals(3, d4.scale()); HiveDecimal expected4 = HiveDecimal.create("-17.234"); DecimalUtil.round(0, d4, dcv); Assert.assertEquals(0, expected4.compareTo(dcv.vector[0].getHiveDecimal())); @@ -204,8 +208,9 @@ public class TestDecimalUtil { DecimalUtil.negate(0, d2, dcv); Assert.assertEquals(0, expected2.compareTo(dcv.vector[0].getHiveDecimal())); + // As of HIVE-8745, these decimal values should be trimmed of trailing zeros. HiveDecimal d3 = HiveDecimal.create("0.00000"); - Assert.assertEquals(5, d3.scale()); + Assert.assertEquals(0, d3.scale()); HiveDecimal expected3 = HiveDecimal.create("0"); DecimalUtil.negate(0, d3, dcv); Assert.assertEquals(0, expected3.compareTo(dcv.vector[0].getHiveDecimal())); @@ -223,7 +228,7 @@ public class TestDecimalUtil { Assert.assertEquals(-1, lcv.vector[0]); HiveDecimal d3 = HiveDecimal.create("0.00000"); - Assert.assertEquals(5, d3.scale()); + Assert.assertEquals(0, d3.scale()); d3.setScale(5); DecimalUtil.sign(0, d3, lcv); Assert.assertEquals(0, lcv.vector[0]); Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java (original) +++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java Tue Nov 18 00:48:40 2014 @@ -323,18 +323,19 @@ public class TestVectorTypeCasts { expr.evaluate(b); BytesColumnVector r = (BytesColumnVector) b.cols[1]; - byte[] v = toBytes("1.10"); + // As of HIVE-8745, these decimal values should be trimmed of trailing zeros. + byte[] v = toBytes("1.1"); assertTrue(((Integer) v.length).toString() + " " + r.length[0], v.length == r.length[0]); Assert.assertEquals(0, StringExpr.compare(v, 0, v.length, r.vector[0], r.start[0], r.length[0])); - v = toBytes("-2.20"); + v = toBytes("-2.2"); Assert.assertEquals(0, StringExpr.compare(v, 0, v.length, r.vector[1], r.start[1], r.length[1])); - v = toBytes("9999999999999999.00"); + v = toBytes("9999999999999999"); Assert.assertEquals(0, StringExpr.compare(v, 0, v.length, r.vector[2], r.start[2], r.length[2])); Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java (original) +++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java Tue Nov 18 00:48:40 2014 @@ -21,27 +21,6 @@ import static org.junit.Assert.assertArr import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.sql.Date; -import java.sql.Timestamp; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.TimeZone; -import java.util.TreeSet; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FSDataInputStream; @@ -66,9 +45,9 @@ import org.apache.hadoop.hive.ql.io.Comb import org.apache.hadoop.hive.ql.io.HiveInputFormat; import org.apache.hadoop.hive.ql.io.HiveOutputFormat; import org.apache.hadoop.hive.ql.io.InputFormatChecker; -import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory; import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; +import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory; import org.apache.hadoop.hive.ql.plan.MapWork; import org.apache.hadoop.hive.ql.plan.PartitionDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; @@ -104,6 +83,27 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestName; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.sql.Date; +import java.sql.Timestamp; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.TimeZone; +import java.util.TreeSet; + public class TestInputOutputFormat { Path workDir = new Path(System.getProperty("test.tmp.dir","target/tmp")); @@ -1032,6 +1032,24 @@ public class TestInputOutputFormat { reader.close(); } + static class SimpleRow implements Writable { + Text z; + + public SimpleRow(Text t) { + this.z = t; + } + + @Override + public void write(DataOutput dataOutput) throws IOException { + throw new UnsupportedOperationException("unsupported"); + } + + @Override + public void readFields(DataInput dataInput) throws IOException { + throw new UnsupportedOperationException("unsupported"); + } + } + static class NestedRow implements Writable { int z; MyRow r; @@ -1620,14 +1638,14 @@ public class TestInputOutputFormat { assertEquals("mock:/combinationAcid/p=0/base_0000010/bucket_00000", split.getPath().toString()); assertEquals(0, split.getStart()); - assertEquals(580, split.getLength()); + assertEquals(582, split.getLength()); split = (HiveInputFormat.HiveInputSplit) splits[1]; assertEquals("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat", split.inputFormatClassName()); assertEquals("mock:/combinationAcid/p=0/base_0000010/bucket_00001", split.getPath().toString()); assertEquals(0, split.getStart()); - assertEquals(601, split.getLength()); + assertEquals(603, split.getLength()); CombineHiveInputFormat.CombineHiveInputSplit combineSplit = (CombineHiveInputFormat.CombineHiveInputSplit) splits[2]; assertEquals(BUCKETS, combineSplit.getNumPaths()); @@ -1635,7 +1653,7 @@ public class TestInputOutputFormat { assertEquals("mock:/combinationAcid/p=1/00000" + bucket + "_0", combineSplit.getPath(bucket).toString()); assertEquals(0, combineSplit.getOffset(bucket)); - assertEquals(225, combineSplit.getLength(bucket)); + assertEquals(227, combineSplit.getLength(bucket)); } String[] hosts = combineSplit.getLocations(); assertEquals(2, hosts.length); @@ -1685,4 +1703,89 @@ public class TestInputOutputFormat { assertEquals("cost", leaves.get(0).getColumnName()); assertEquals(PredicateLeaf.Operator.IS_NULL, leaves.get(0).getOperator()); } + + @Test + @SuppressWarnings("unchecked,deprecation") + public void testSplitElimination() throws Exception { + Properties properties = new Properties(); + StructObjectInspector inspector; + synchronized (TestOrcFile.class) { + inspector = (StructObjectInspector) + ObjectInspectorFactory.getReflectionObjectInspector(NestedRow.class, + ObjectInspectorFactory.ObjectInspectorOptions.JAVA); + } + SerDe serde = new OrcSerde(); + OutputFormat<?, ?> outFormat = new OrcOutputFormat(); + conf.setInt("mapred.max.split.size", 50); + RecordWriter writer = + outFormat.getRecordWriter(fs, conf, testFilePath.toString(), + Reporter.NULL); + writer.write(NullWritable.get(), + serde.serialize(new NestedRow(1,2,3), inspector)); + writer.write(NullWritable.get(), + serde.serialize(new NestedRow(4,5,6), inspector)); + writer.write(NullWritable.get(), + serde.serialize(new NestedRow(7,8,9), inspector)); + writer.close(Reporter.NULL); + serde = new OrcSerde(); + SearchArgument sarg = + SearchArgumentFactory.newBuilder() + .startAnd() + .lessThan("z", new Integer(0)) + .end() + .build(); + conf.set("sarg.pushdown", sarg.toKryo()); + conf.set("hive.io.file.readcolumn.names", "z,r"); + properties.setProperty("columns", "z,r"); + properties.setProperty("columns.types", "int:struct<x:int,y:int>"); + SerDeUtils.initializeSerDe(serde, conf, properties, null); + inspector = (StructObjectInspector) serde.getObjectInspector(); + InputFormat<?,?> in = new OrcInputFormat(); + FileInputFormat.setInputPaths(conf, testFilePath.toString()); + InputSplit[] splits = in.getSplits(conf, 1); + assertEquals(0, splits.length); + } + + @Test + @SuppressWarnings("unchecked,deprecation") + public void testSplitEliminationNullStats() throws Exception { + Properties properties = new Properties(); + StructObjectInspector inspector; + synchronized (TestOrcFile.class) { + inspector = (StructObjectInspector) + ObjectInspectorFactory.getReflectionObjectInspector(SimpleRow.class, + ObjectInspectorFactory.ObjectInspectorOptions.JAVA); + } + SerDe serde = new OrcSerde(); + OutputFormat<?, ?> outFormat = new OrcOutputFormat(); + conf.setInt("mapred.max.split.size", 50); + RecordWriter writer = + outFormat.getRecordWriter(fs, conf, testFilePath.toString(), + Reporter.NULL); + writer.write(NullWritable.get(), + serde.serialize(new SimpleRow(null), inspector)); + writer.write(NullWritable.get(), + serde.serialize(new SimpleRow(null), inspector)); + writer.write(NullWritable.get(), + serde.serialize(new SimpleRow(null), inspector)); + writer.close(Reporter.NULL); + serde = new OrcSerde(); + SearchArgument sarg = + SearchArgumentFactory.newBuilder() + .startAnd() + .lessThan("z", new String("foo")) + .end() + .build(); + conf.set("sarg.pushdown", sarg.toKryo()); + conf.set("hive.io.file.readcolumn.names", "z"); + properties.setProperty("columns", "z"); + properties.setProperty("columns.types", "string"); + SerDeUtils.initializeSerDe(serde, conf, properties, null); + inspector = (StructObjectInspector) serde.getObjectInspector(); + InputFormat<?,?> in = new OrcInputFormat(); + FileInputFormat.setInputPaths(conf, testFilePath.toString()); + InputSplit[] splits = in.getSplits(conf, 1); + assertEquals(0, splits.length); + } + } Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/TestParquetByteInspector.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/TestParquetByteInspector.java?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/TestParquetByteInspector.java (original) +++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/TestParquetByteInspector.java Tue Nov 18 00:48:40 2014 @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.hive.ql.io.parquet.serde.primitive; import static org.junit.Assert.assertEquals; Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/TestParquetShortInspector.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/TestParquetShortInspector.java?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/TestParquetShortInspector.java (original) +++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/TestParquetShortInspector.java Tue Nov 18 00:48:40 2014 @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.hive.ql.io.parquet.serde.primitive; import static org.junit.Assert.assertEquals; Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java (original) +++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java Tue Nov 18 00:48:40 2014 @@ -2830,7 +2830,7 @@ public class TestSearchArgumentImpl { .build(); assertEquals("leaf-0 = (LESS_THAN x 1970-01-11)\n" + "leaf-1 = (LESS_THAN_EQUALS y hi)\n" + - "leaf-2 = (EQUALS z 1.0)\n" + + "leaf-2 = (EQUALS z 1)\n" + "expr = (and leaf-0 leaf-1 leaf-2)", sarg.toString()); sarg = SearchArgumentFactory.newBuilder() Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java (original) +++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java Tue Nov 18 00:48:40 2014 @@ -32,6 +32,7 @@ import org.apache.hadoop.fs.FSDataInputS import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.QueryPlan; import org.apache.hadoop.hive.ql.exec.ExplainTask; @@ -267,7 +268,7 @@ public class TestUpdateDeleteSemanticAna // I have to create the tables here (rather than in setup()) because I need the Hive // connection, which is conviently created by the semantic analyzer. Map<String, String> params = new HashMap<String, String>(1); - params.put(SemanticAnalyzer.ACID_TABLE_PROPERTY, "true"); + params.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "true"); db.createTable("T", Arrays.asList("a", "b"), null, OrcInputFormat.class, OrcOutputFormat.class, 2, Arrays.asList("a"), params); db.createTable("U", Arrays.asList("a", "b"), Arrays.asList("ds"), OrcInputFormat.class, Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestHiveAuthorizationTaskFactory.java URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestHiveAuthorizationTaskFactory.java?rev=1640263&r1=1640262&r2=1640263&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestHiveAuthorizationTaskFactory.java (original) +++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestHiveAuthorizationTaskFactory.java Tue Nov 18 00:48:40 2014 @@ -22,18 +22,23 @@ import java.util.HashMap; import junit.framework.Assert; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.DDLSemanticAnalyzer; +import org.apache.hadoop.hive.ql.parse.HiveParser; import org.apache.hadoop.hive.ql.parse.ParseDriver; +import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.DDLWork; import org.apache.hadoop.hive.ql.plan.GrantDesc; import org.apache.hadoop.hive.ql.plan.GrantRevokeRoleDDL; import org.apache.hadoop.hive.ql.plan.PrincipalDesc; import org.apache.hadoop.hive.ql.plan.PrivilegeDesc; +import org.apache.hadoop.hive.ql.plan.PrivilegeObjectDesc; import org.apache.hadoop.hive.ql.plan.RevokeDesc; import org.apache.hadoop.hive.ql.plan.RoleDDLDesc; import org.apache.hadoop.hive.ql.plan.RoleDDLDesc.RoleOperation; @@ -47,6 +52,33 @@ import org.mockito.Mockito; public class TestHiveAuthorizationTaskFactory { + public static class DummyHiveAuthorizationTaskFactoryImpl extends HiveAuthorizationTaskFactoryImpl { + + static String uriPath = ""; + static String serverName = ""; + + public DummyHiveAuthorizationTaskFactoryImpl(HiveConf conf, Hive db) { + super(conf, db); + } + + @Override + protected PrivilegeObjectDesc parsePrivObject(ASTNode ast) throws SemanticException { + ASTNode child = (ASTNode) ast.getChild(0); + ASTNode gchild = (ASTNode)child.getChild(0); + if (child.getType() == HiveParser.TOK_URI_TYPE) { + uriPath = gchild.getText().replaceAll("'", "").replaceAll("\"", ""); + } else if (child.getType() == HiveParser.TOK_SERVER_TYPE) { + serverName = gchild.getText(); + } + return super.parsePrivObject(ast); + } + + public static void reset() { + uriPath = ""; + serverName = ""; + } + } + private static final String SELECT = "SELECT"; private static final String DB = "default"; private static final String TABLE = "table1"; @@ -67,6 +99,8 @@ public class TestHiveAuthorizationTaskFa @Before public void setup() throws Exception { conf = new HiveConf(); + conf.setVar(ConfVars.HIVE_AUTHORIZATION_TASK_FACTORY, + TestHiveAuthorizationTaskFactory.DummyHiveAuthorizationTaskFactoryImpl.class.getName()); db = Mockito.mock(Hive.class); table = new Table(DB, TABLE); partition = new Partition(table); @@ -81,6 +115,7 @@ public class TestHiveAuthorizationTaskFa HadoopDefaultAuthenticator auth = new HadoopDefaultAuthenticator(); auth.setConf(conf); currentUser = auth.getUserName(); + DummyHiveAuthorizationTaskFactoryImpl.reset(); } /** @@ -414,6 +449,34 @@ public class TestHiveAuthorizationTaskFa Assert.assertTrue("Expected table", grantDesc.getHiveObj().getTable()); } + /** + * GRANT ALL ON URI + */ + @Test + public void testGrantUri() throws Exception { + String uriPath = "/tmp"; + try { + analyze("GRANT ALL ON URI '" + uriPath + "' TO USER user2"); + Assert.fail("Grant on URI should fail"); + } catch (SemanticException e) { + Assert.assertEquals(uriPath, DummyHiveAuthorizationTaskFactoryImpl.uriPath); + } + } + + /** + * GRANT ALL ON SERVER + */ + @Test + public void testGrantServer() throws Exception { + String serverName = "foo"; + try { + analyze("GRANT ALL ON SERVER " + serverName + " TO USER user2"); + Assert.fail("Grant on Server should fail"); + } catch (SemanticException e) { + Assert.assertEquals(serverName, DummyHiveAuthorizationTaskFactoryImpl.serverName); + } + } + private DDLWork analyze(String command) throws Exception { return AuthorizationTestUtil.analyze(command, conf, db); }
