Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileMergeMapper.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileMergeMapper.java?rev=1614793&r1=1614792&r2=1614793&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileMergeMapper.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileMergeMapper.java Wed Jul 30 23:50:11 2014 @@ -22,89 +22,25 @@ import java.io.IOException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.io.RCFile; import org.apache.hadoop.hive.ql.io.RCFileOutputFormat; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx; -import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; +import org.apache.hadoop.hive.ql.io.merge.MergeMapper; import org.apache.hadoop.hive.shims.CombineHiveKey; -import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.compress.CompressionCodec; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.MapReduceBase; import org.apache.hadoop.mapred.Mapper; import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.Reporter; @SuppressWarnings("deprecation") -public class RCFileMergeMapper extends MapReduceBase implements +public class RCFileMergeMapper extends MergeMapper implements Mapper<Object, RCFileValueBufferWrapper, Object, Object> { - private JobConf jc; - Class<? extends Writable> outputClass; RCFile.Writer outWriter; - Path finalPath; - FileSystem fs; - - boolean exception = false; - boolean autoDelete = false; - Path outPath; - CompressionCodec codec = null; int columnNumber = 0; - - boolean hasDynamicPartitions = false; - boolean isListBucketingDML = false; - boolean isListBucketingAlterTableConcatenate = false; - int listBucketingDepth; // used as depth for dir-calculation and if it is list bucketing case. - boolean tmpPathFixedConcatenate = false; - boolean tmpPathFixed = false; - Path tmpPath; - Path taskTmpPath; - Path dpPath; - public final static Log LOG = LogFactory.getLog("RCFileMergeMapper"); - public RCFileMergeMapper() { - } - - @Override - public void configure(JobConf job) { - jc = job; - hasDynamicPartitions = HiveConf.getBoolVar(job, - HiveConf.ConfVars.HIVEMERGECURRENTJOBHASDYNAMICPARTITIONS); - isListBucketingAlterTableConcatenate = HiveConf.getBoolVar(job, - HiveConf.ConfVars.HIVEMERGECURRENTJOBCONCATENATELISTBUCKETING); - listBucketingDepth = HiveConf.getIntVar(job, - HiveConf.ConfVars.HIVEMERGECURRENTJOBCONCATENATELISTBUCKETINGDEPTH); - - Path specPath = RCFileBlockMergeOutputFormat.getMergeOutputPath(job); - Path tmpPath = Utilities.toTempPath(specPath); - Path taskTmpPath = Utilities.toTaskTempPath(specPath); - updatePaths(tmpPath, taskTmpPath); - try { - fs = specPath.getFileSystem(job); - autoDelete = fs.deleteOnExit(outPath); - } catch (IOException e) { - this.exception = true; - throw new RuntimeException(e); - } - } - - private void updatePaths(Path tmpPath, Path taskTmpPath) { - String taskId = Utilities.getTaskId(jc); - this.tmpPath = tmpPath; - this.taskTmpPath = taskTmpPath; - finalPath = new Path(tmpPath, taskId); - outPath = new Path(taskTmpPath, Utilities.toTempPath(taskId)); - } - @Override public void map(Object k, RCFileValueBufferWrapper value, OutputCollector<Object, Object> output, Reporter reporter) @@ -118,35 +54,7 @@ public class RCFileMergeMapper extends M key = (RCFileKeyBufferWrapper) k; } - /** - * 1. boolean isListBucketingAlterTableConcatenate will be true only if it is alter table ... - * concatenate on stored-as-dir so it will handle list bucketing alter table merge in the if - * cause with the help of fixTmpPathConcatenate - * 2. If it is DML, isListBucketingAlterTableConcatenate will be false so that it will be - * handled by else cause. In this else cause, we have another if check. - * 2.1 the if check will make sure DP or LB, we will fix path with the help of fixTmpPath(..). - * Since both has sub-directories. it includes SP + LB. - * 2.2 only SP without LB, we dont fix path. - */ - // Fix temp path for alter table ... concatenate - if (isListBucketingAlterTableConcatenate) { - if (this.tmpPathFixedConcatenate) { - checkPartitionsMatch(key.inputPath.getParent()); - } else { - fixTmpPathConcatenate(key.inputPath.getParent()); - tmpPathFixedConcatenate = true; - } - } else { - if (hasDynamicPartitions || (listBucketingDepth > 0)) { - if (tmpPathFixed) { - checkPartitionsMatch(key.inputPath.getParent()); - } else { - // We haven't fixed the TMP path for this mapper yet - fixTmpPath(key.inputPath.getParent()); - tmpPathFixed = true; - } - } - } + fixTmpPathAlterTable(key.inputPath.getParent()); if (outWriter == null) { codec = key.codec; @@ -172,106 +80,6 @@ public class RCFileMergeMapper extends M } } - /** - * Validates that each input path belongs to the same partition - * since each mapper merges the input to a single output directory - * - * @param inputPath - * @throws HiveException - */ - private void checkPartitionsMatch(Path inputPath) throws HiveException { - if (!dpPath.equals(inputPath)) { - // Temp partition input path does not match exist temp path - String msg = "Multiple partitions for one block merge mapper: " + - dpPath + " NOT EQUAL TO " + inputPath; - LOG.error(msg); - throw new HiveException(msg); - } - } - - /** - * Fixes tmpPath to point to the correct partition. - * Before this is called, tmpPath will default to the root tmp table dir - * fixTmpPath(..) works for DP + LB + multiple skewed values + merge. reason: - * 1. fixTmpPath(..) compares inputPath and tmpDepth, find out path difference and put it into - * newPath. Then add newpath to existing this.tmpPath and this.taskTmpPath. - * 2. The path difference between inputPath and tmpDepth can be DP or DP+LB. It will automatically - * handle it. - * 3. For example, - * if inputpath is <prefix>/-ext-10002/hr=a1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/ - * HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME - * tmppath is <prefix>/_tmp.-ext-10000 - * newpath will be hr=a1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME - * Then, this.tmpPath and this.taskTmpPath will be update correctly. - * We have list_bucket_dml_6.q cover this case: DP + LP + multiple skewed values + merge. - * @param inputPath - * @throws HiveException - * @throws IOException - */ - private void fixTmpPath(Path inputPath) - throws HiveException, IOException { - dpPath = inputPath; - Path newPath = new Path("."); - int inputDepth = inputPath.depth(); - int tmpDepth = tmpPath.depth(); - - // Build the path from bottom up - while (inputPath != null && inputPath.depth() > tmpDepth) { - newPath = new Path(inputPath.getName(), newPath); - inputDepth--; - inputPath = inputPath.getParent(); - } - - Path newTmpPath = new Path(tmpPath, newPath); - Path newTaskTmpPath = new Path(taskTmpPath, newPath); - if (!fs.exists(newTmpPath)) { - fs.mkdirs(newTmpPath); - } - updatePaths(newTmpPath, newTaskTmpPath); - } - - /** - * Fixes tmpPath to point to the correct list bucketing sub-directories. - * Before this is called, tmpPath will default to the root tmp table dir - * Reason to add a new method instead of changing fixTmpPath() - * Reason 1: logic has slightly difference - * fixTmpPath(..) needs 2 variables in order to decide path delta which is in variable newPath. - * 1. inputPath.depth() - * 2. tmpPath.depth() - * fixTmpPathConcatenate needs 2 variables too but one of them is different from fixTmpPath(..) - * 1. inputPath.depth() - * 2. listBucketingDepth - * Reason 2: less risks - * The existing logic is a little not trivial around map() and fixTmpPath(). In order to ensure - * minimum impact on existing flow, we try to avoid change on existing code/flow but add new code - * for new feature. - * - * @param inputPath - * @throws HiveException - * @throws IOException - */ - private void fixTmpPathConcatenate(Path inputPath) - throws HiveException, IOException { - dpPath = inputPath; - Path newPath = new Path("."); - - int depth = listBucketingDepth; - // Build the path from bottom up. pick up list bucketing subdirectories - while ((inputPath != null) && (depth > 0)) { - newPath = new Path(inputPath.getName(), newPath); - inputPath = inputPath.getParent(); - depth--; - } - - Path newTmpPath = new Path(tmpPath, newPath); - Path newTaskTmpPath = new Path(taskTmpPath, newPath); - if (!fs.exists(newTmpPath)) { - fs.mkdirs(newTmpPath); - } - updatePaths(newTmpPath, newTaskTmpPath); - } - - @Override public void close() throws IOException { // close writer @@ -282,42 +90,7 @@ public class RCFileMergeMapper extends M outWriter.close(); outWriter = null; - if (!exception) { - FileStatus fss = fs.getFileStatus(outPath); - LOG.info("renamed path " + outPath + " to " + finalPath - + " . File size is " + fss.getLen()); - if (!fs.rename(outPath, finalPath)) { - throw new IOException("Unable to rename output to " + finalPath); - } - } else { - if (!autoDelete) { - fs.delete(outPath, true); - } - } - } - - public static String BACKUP_PREFIX = "_backup."; - - public static Path backupOutputPath(FileSystem fs, Path outpath, JobConf job) - throws IOException, HiveException { - if (fs.exists(outpath)) { - Path backupPath = new Path(outpath.getParent(), BACKUP_PREFIX - + outpath.getName()); - Utilities.rename(fs, outpath, backupPath); - return backupPath; - } else { - return null; - } - } - - public static void jobClose(Path outputPath, boolean success, JobConf job, - LogHelper console, DynamicPartitionCtx dynPartCtx, Reporter reporter - ) throws HiveException, IOException { - FileSystem fs = outputPath.getFileSystem(job); - Path backupPath = backupOutputPath(fs, outputPath, job); - Utilities.mvFileToFinalPath(outputPath, job, success, LOG, dynPartCtx, null, - reporter); - fs.delete(backupPath, true); + super.close(); } }
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java?rev=1614793&r1=1614792&r2=1614793&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java Wed Jul 30 23:50:11 2014 @@ -98,7 +98,8 @@ public class GenMRFileSink1 implements N if (chDir) { // Merge the files in the destination table/partitions by creating Map-only merge job - // If underlying data is RCFile a RCFileBlockMerge task would be created. + // If underlying data is RCFile or OrcFile, RCFileBlockMerge task or + // OrcFileStripeMerge task would be created. LOG.info("using CombineHiveInputformat for the merge job"); GenMapRedUtils.createMRWorkForMergingFiles(fsOp, finalName, ctx.getDependencyTaskForMultiInsert(), ctx.getMvTask(), Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java?rev=1614793&r1=1614792&r2=1614793&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java Wed Jul 30 23:50:11 2014 @@ -31,15 +31,16 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Properties; import java.util.Set; -import java.lang.StringBuffer; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.ql.ErrorMsg; +import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.ql.Context; +import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.ConditionalTask; import org.apache.hadoop.hive.ql.exec.DemuxOperator; @@ -64,11 +65,10 @@ import org.apache.hadoop.hive.ql.exec.mr import org.apache.hadoop.hive.ql.exec.mr.MapRedTask; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.io.RCFileInputFormat; -import org.apache.hadoop.hive.ql.io.rcfile.merge.MergeWork; +import org.apache.hadoop.hive.ql.io.merge.MergeWork; +import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMRUnionCtx; import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx; import org.apache.hadoop.hive.ql.optimizer.listbucketingpruner.ListBucketingPruner; @@ -1245,23 +1245,33 @@ public final class GenMapRedUtils { MapWork cplan; Serializable work; - if (conf.getBoolVar(ConfVars.HIVEMERGERCFILEBLOCKLEVEL) && - fsInputDesc.getTableInfo().getInputFileFormatClass().equals(RCFileInputFormat.class)) { + if ((conf.getBoolVar(ConfVars.HIVEMERGERCFILEBLOCKLEVEL) && + fsInputDesc.getTableInfo().getInputFileFormatClass().equals(RCFileInputFormat.class)) || + (conf.getBoolVar(ConfVars.HIVEMERGEORCFILESTRIPELEVEL) && + fsInputDesc.getTableInfo().getInputFileFormatClass().equals(OrcInputFormat.class))) { // Check if InputFormatClass is valid - String inputFormatClass = conf.getVar(ConfVars.HIVEMERGEINPUTFORMATBLOCKLEVEL); + final String inputFormatClass; + if (fsInputDesc.getTableInfo().getInputFileFormatClass().equals(RCFileInputFormat.class)) { + inputFormatClass = conf.getVar(ConfVars.HIVEMERGEINPUTFORMATBLOCKLEVEL); + } else { + inputFormatClass = conf.getVar(ConfVars.HIVEMERGEINPUTFORMATSTRIPELEVEL); + } try { Class c = Class.forName(inputFormatClass); - LOG.info("RCFile format- Using block level merge"); - cplan = GenMapRedUtils.createRCFileMergeTask(fsInputDesc, finalName, + if(fsInputDesc.getTableInfo().getInputFileFormatClass().equals(OrcInputFormat.class)) { + LOG.info("OrcFile format - Using stripe level merge"); + } else { + LOG.info("RCFile format- Using block level merge"); + } + cplan = GenMapRedUtils.createMergeTask(fsInputDesc, finalName, dpCtx != null && dpCtx.getNumDPCols() > 0); work = cplan; } catch (ClassNotFoundException e) { String msg = "Illegal input format class: " + inputFormatClass; throw new SemanticException(msg); } - } else { cplan = createMRWorkForMergingFiles(conf, tsMerge, fsInputDesc); if (conf.getVar(ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) { @@ -1474,20 +1484,22 @@ public final class GenMapRedUtils { } /** - * Create a block level merge task for RCFiles. + * Create a block level merge task for RCFiles or stripe level merge task for + * ORCFiles * * @param fsInputDesc * @param finalName - * @return MergeWork if table is stored as RCFile, + * @return MergeWork if table is stored as RCFile or ORCFile, * null otherwise */ - public static MapWork createRCFileMergeTask(FileSinkDesc fsInputDesc, + public static MapWork createMergeTask(FileSinkDesc fsInputDesc, Path finalName, boolean hasDynamicPartitions) throws SemanticException { Path inputDir = fsInputDesc.getFinalDirName(); TableDesc tblDesc = fsInputDesc.getTableInfo(); - if (tblDesc.getInputFileFormatClass().equals(RCFileInputFormat.class)) { + if (tblDesc.getInputFileFormatClass().equals(RCFileInputFormat.class) || + tblDesc.getInputFileFormatClass().equals(OrcInputFormat.class)) { ArrayList<Path> inputDirs = new ArrayList<Path>(1); ArrayList<String> inputDirstr = new ArrayList<String>(1); if (!hasDynamicPartitions @@ -1497,7 +1509,8 @@ public final class GenMapRedUtils { } MergeWork work = new MergeWork(inputDirs, finalName, - hasDynamicPartitions, fsInputDesc.getDynPartCtx()); + hasDynamicPartitions, fsInputDesc.getDynPartCtx(), + tblDesc.getInputFileFormatClass()); LinkedHashMap<String, ArrayList<String>> pathToAliases = new LinkedHashMap<String, ArrayList<String>>(); pathToAliases.put(inputDir.toString(), (ArrayList<String>) inputDirstr.clone()); @@ -1515,7 +1528,8 @@ public final class GenMapRedUtils { return work; } - throw new SemanticException("createRCFileMergeTask called on non-RCFile table"); + throw new SemanticException("createMergeTask called on a table with file" + + " format other than RCFile or ORCFile"); } /** Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTablePartMergeFilesDesc.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTablePartMergeFilesDesc.java?rev=1614793&r1=1614792&r2=1614793&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTablePartMergeFilesDesc.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTablePartMergeFilesDesc.java Wed Jul 30 23:50:11 2014 @@ -25,6 +25,7 @@ import java.util.List; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.ListBucketingCtx; +import org.apache.hadoop.mapred.InputFormat; @Explain(displayName = "Alter Table Partition Merge Files") public class AlterTablePartMergeFilesDesc { @@ -35,6 +36,7 @@ public class AlterTablePartMergeFilesDes private List<Path> inputDir = new ArrayList<Path>(); private Path outputDir = null; + private Class<? extends InputFormat> inputFormatClass; public AlterTablePartMergeFilesDesc(String tableName, HashMap<String, String> partSpec) { @@ -90,4 +92,12 @@ public class AlterTablePartMergeFilesDes this.lbCtx = lbCtx; } + public Class<? extends InputFormat> getInputFormatClass() { + return inputFormatClass; + } + + public void setInputFormatClass(Class<? extends InputFormat> inputFormatClass) { + this.inputFormatClass = inputFormatClass; + } + } Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=1614793&r1=1614792&r2=1614793&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Wed Jul 30 23:50:11 2014 @@ -69,6 +69,7 @@ import org.apache.hadoop.hive.ql.index.H import org.apache.hadoop.hive.ql.index.HiveIndexHandler; import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.ql.io.RCFileInputFormat; +import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager; import org.apache.hadoop.hive.ql.lockmgr.LockException; @@ -1520,11 +1521,13 @@ public class DDLSemanticAnalyzer extends tblObj.getSkewedColValueLocationMaps(), tblObj.isStoredAsSubDirectories(), conf); } - // throw a HiveException for non-rcfile. - if (!inputFormatClass.equals(RCFileInputFormat.class)) { + // throw a HiveException for other than rcfile and orcfile. + if (!((inputFormatClass.equals(RCFileInputFormat.class) || + (inputFormatClass.equals(OrcInputFormat.class))))) { throw new SemanticException( - "Only RCFileFormat is supportted right now."); + "Only RCFile and ORCFile Formats are supportted right now."); } + mergeDesc.setInputFormatClass(inputFormatClass); // throw a HiveException if the table/partition is bucketized if (bucketCols != null && bucketCols.size() > 0) { Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java?rev=1614793&r1=1614792&r2=1614793&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java Wed Jul 30 23:50:11 2014 @@ -313,7 +313,8 @@ public class GenTezUtils { if (chDir) { // Merge the files in the destination table/partitions by creating Map-only merge job - // If underlying data is RCFile a RCFileBlockMerge task would be created. + // If underlying data is RCFile or OrcFile, RCFileBlockMerge task or + // OrcFileStripeMerge task would be created. LOG.info("using CombineHiveInputformat for the merge job"); GenMapRedUtils.createMRWorkForMergingFiles(fileSink, finalName, context.dependencyTask, context.moveTask, Added: hive/trunk/ql/src/test/queries/clientnegative/orc_merge1.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/orc_merge1.q?rev=1614793&view=auto ============================================================================== --- hive/trunk/ql/src/test/queries/clientnegative/orc_merge1.q (added) +++ hive/trunk/ql/src/test/queries/clientnegative/orc_merge1.q Wed Jul 30 23:50:11 2014 @@ -0,0 +1,17 @@ +DROP TABLE orcfile_merge; +set hive.exec.dynamic.partition=true; +set hive.exec.dynamic.partition.mode=nonstrict; + +set hive.exec.orc.default.buffer.size=1000; +CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC; + +INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart; + +set hive.exec.orc.default.buffer.size=2000; +INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart; + +-- will fail because of different compression buffer size +ALTER TABLE orcfile_merge PARTITION(ds='2008-04-08',part=11) CONCATENATE; + +DROP TABLE orcfile_merge; Added: hive/trunk/ql/src/test/queries/clientnegative/orc_merge2.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/orc_merge2.q?rev=1614793&view=auto ============================================================================== --- hive/trunk/ql/src/test/queries/clientnegative/orc_merge2.q (added) +++ hive/trunk/ql/src/test/queries/clientnegative/orc_merge2.q Wed Jul 30 23:50:11 2014 @@ -0,0 +1,17 @@ +DROP TABLE orcfile_merge; +set hive.exec.dynamic.partition=true; +set hive.exec.dynamic.partition.mode=nonstrict; + +set hive.exec.orc.default.compress=ZLIB; +CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC; + +INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart; + +set hive.exec.orc.default.compress=NONE; +INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart; + +-- will fail because of different compression codec +ALTER TABLE orcfile_merge PARTITION(ds='2008-04-08',part=11) CONCATENATE; + +DROP TABLE orcfile_merge; Added: hive/trunk/ql/src/test/queries/clientnegative/orc_merge3.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/orc_merge3.q?rev=1614793&view=auto ============================================================================== --- hive/trunk/ql/src/test/queries/clientnegative/orc_merge3.q (added) +++ hive/trunk/ql/src/test/queries/clientnegative/orc_merge3.q Wed Jul 30 23:50:11 2014 @@ -0,0 +1,17 @@ +DROP TABLE orcfile_merge; +set hive.exec.dynamic.partition=true; +set hive.exec.dynamic.partition.mode=nonstrict; + +set hive.exec.orc.default.row.index.stride=1000; +CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC; + +INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart; + +set hive.exec.orc.default.row.index.stride=2000; +INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart; + +-- will fail because of different row index stride +ALTER TABLE orcfile_merge PARTITION(ds='2008-04-08',part=11) CONCATENATE; + +DROP TABLE orcfile_merge; Added: hive/trunk/ql/src/test/queries/clientnegative/orc_merge4.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/orc_merge4.q?rev=1614793&view=auto ============================================================================== --- hive/trunk/ql/src/test/queries/clientnegative/orc_merge4.q (added) +++ hive/trunk/ql/src/test/queries/clientnegative/orc_merge4.q Wed Jul 30 23:50:11 2014 @@ -0,0 +1,17 @@ +DROP TABLE orcfile_merge; +set hive.exec.dynamic.partition=true; +set hive.exec.dynamic.partition.mode=nonstrict; + +CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC; + +INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart; + +ALTER TABLE orcfile_merge ADD COLUMNS (newkey int); + +INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT key,value,key,ds,hr FROM srcpart; + +-- will fail because of different column count +ALTER TABLE orcfile_merge PARTITION(ds='2008-04-08',part=11) CONCATENATE; + +DROP TABLE orcfile_merge; Added: hive/trunk/ql/src/test/queries/clientnegative/orc_merge5.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/orc_merge5.q?rev=1614793&view=auto ============================================================================== --- hive/trunk/ql/src/test/queries/clientnegative/orc_merge5.q (added) +++ hive/trunk/ql/src/test/queries/clientnegative/orc_merge5.q Wed Jul 30 23:50:11 2014 @@ -0,0 +1,17 @@ +DROP TABLE orcfile_merge; +set hive.exec.dynamic.partition=true; +set hive.exec.dynamic.partition.mode=nonstrict; + +set hive.exec.orc.write.format=0.11; +CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC; + +INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart; + +set hive.exec.orc.write.format=0.12; +INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart; + +-- will fail because of different write format +ALTER TABLE orcfile_merge PARTITION(ds='2008-04-08',part=11) CONCATENATE; + +DROP TABLE orcfile_merge; Added: hive/trunk/ql/src/test/queries/clientpositive/alter_merge_2_orc.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/alter_merge_2_orc.q?rev=1614793&view=auto ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/alter_merge_2_orc.q (added) +++ hive/trunk/ql/src/test/queries/clientpositive/alter_merge_2_orc.q Wed Jul 30 23:50:11 2014 @@ -0,0 +1,19 @@ +create table src_orc_merge_test_part(key int, value string) partitioned by (ds string, ts string) stored as orc; + +alter table src_orc_merge_test_part add partition (ds='2012-01-03', ts='2012-01-03+14:46:31'); +desc extended src_orc_merge_test_part partition (ds='2012-01-03', ts='2012-01-03+14:46:31'); + +insert overwrite table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src; +insert into table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src limit 100; +insert into table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src limit 10; + +select count(1) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31'; +select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31'; + +alter table src_orc_merge_test_part partition (ds='2012-01-03', ts='2012-01-03+14:46:31') concatenate; + + +select count(1) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31'; +select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31'; + +drop table src_orc_merge_test_part; Added: hive/trunk/ql/src/test/queries/clientpositive/alter_merge_orc.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/alter_merge_orc.q?rev=1614793&view=auto ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/alter_merge_orc.q (added) +++ hive/trunk/ql/src/test/queries/clientpositive/alter_merge_orc.q Wed Jul 30 23:50:11 2014 @@ -0,0 +1,41 @@ +create table src_orc_merge_test(key int, value string) stored as orc; + +insert overwrite table src_orc_merge_test select * from src; +insert into table src_orc_merge_test select * from src; +insert into table src_orc_merge_test select * from src; + +show table extended like `src_orc_merge_test`; + +select count(1) from src_orc_merge_test; +select sum(hash(key)), sum(hash(value)) from src_orc_merge_test; + +alter table src_orc_merge_test concatenate; + +show table extended like `src_orc_merge_test`; + +select count(1) from src_orc_merge_test; +select sum(hash(key)), sum(hash(value)) from src_orc_merge_test; + + +create table src_orc_merge_test_part(key int, value string) partitioned by (ds string) stored as orc; + +alter table src_orc_merge_test_part add partition (ds='2011'); + +insert overwrite table src_orc_merge_test_part partition (ds='2011') select * from src; +insert into table src_orc_merge_test_part partition (ds='2011') select * from src; +insert into table src_orc_merge_test_part partition (ds='2011') select * from src; + +show table extended like `src_orc_merge_test_part` partition (ds='2011'); + +select count(1) from src_orc_merge_test_part; +select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part; + +alter table src_orc_merge_test_part partition (ds='2011') concatenate; + +show table extended like `src_orc_merge_test_part` partition (ds='2011'); + +select count(1) from src_orc_merge_test_part; +select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part; + +drop table src_orc_merge_test; +drop table src_orc_merge_test_part; Added: hive/trunk/ql/src/test/queries/clientpositive/alter_merge_stats_orc.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/alter_merge_stats_orc.q?rev=1614793&view=auto ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/alter_merge_stats_orc.q (added) +++ hive/trunk/ql/src/test/queries/clientpositive/alter_merge_stats_orc.q Wed Jul 30 23:50:11 2014 @@ -0,0 +1,39 @@ +set hive.exec.submitviachild=true; +set hive.exec.submit.local.task.via.child=true; +create table src_orc_merge_test_stat(key int, value string) stored as orc; + +insert overwrite table src_orc_merge_test_stat select * from src; +insert into table src_orc_merge_test_stat select * from src; +insert into table src_orc_merge_test_stat select * from src; + +show table extended like `src_orc_merge_test_stat`; +desc extended src_orc_merge_test_stat; + +analyze table src_orc_merge_test_stat compute statistics noscan; +desc formatted src_orc_merge_test_stat; + +alter table src_orc_merge_test_stat concatenate; +analyze table src_orc_merge_test_stat compute statistics noscan; +desc formatted src_orc_merge_test_stat; + + +create table src_orc_merge_test_part_stat(key int, value string) partitioned by (ds string) stored as orc; + +alter table src_orc_merge_test_part_stat add partition (ds='2011'); + +insert overwrite table src_orc_merge_test_part_stat partition (ds='2011') select * from src; +insert into table src_orc_merge_test_part_stat partition (ds='2011') select * from src; +insert into table src_orc_merge_test_part_stat partition (ds='2011') select * from src; + +show table extended like `src_orc_merge_test_part_stat` partition (ds='2011'); +desc formatted src_orc_merge_test_part_stat partition (ds='2011'); + +analyze table src_orc_merge_test_part_stat partition(ds='2011') compute statistics noscan; +desc formatted src_orc_merge_test_part_stat partition (ds='2011'); + +alter table src_orc_merge_test_part_stat partition (ds='2011') concatenate; +analyze table src_orc_merge_test_part_stat partition(ds='2011') compute statistics noscan; +desc formatted src_orc_merge_test_part_stat partition (ds='2011'); + +drop table src_orc_merge_test_stat; +drop table src_orc_merge_test_part_stat; Added: hive/trunk/ql/src/test/queries/clientpositive/orc_merge1.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/orc_merge1.q?rev=1614793&view=auto ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/orc_merge1.q (added) +++ hive/trunk/ql/src/test/queries/clientpositive/orc_merge1.q Wed Jul 30 23:50:11 2014 @@ -0,0 +1,51 @@ +set hive.merge.orcfile.stripe.level=false; +set hive.exec.dynamic.partition=true; +set hive.exec.dynamic.partition.mode=nonstrict; + +DROP TABLE orcfile_merge1; +DROP TABLE orcfile_merge1b; + +CREATE TABLE orcfile_merge1 (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC; +CREATE TABLE orcfile_merge1b (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC; + +-- Use non stipe-level merge +EXPLAIN + INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) + SELECT key, value, PMOD(HASH(key), 100) as part + FROM src; + +INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) + SELECT key, value, PMOD(HASH(key), 100) as part + FROM src; + +DESC FORMATTED orcfile_merge1 partition (ds='1', part='50'); + +set hive.merge.orcfile.stripe.level=true; +EXPLAIN + INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part) + SELECT key, value, PMOD(HASH(key), 100) as part + FROM src; + +INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part) + SELECT key, value, PMOD(HASH(key), 100) as part + FROM src; + +DESC FORMATTED orcfile_merge1 partition (ds='1', part='50'); + +-- Verify +SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) + FROM orcfile_merge1 WHERE ds='1' +) t; + +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; + +SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) + FROM orcfile_merge1b WHERE ds='1' +) t; + +DROP TABLE orcfile_merge1; +DROP TABLE orcfile_merge1b; Added: hive/trunk/ql/src/test/queries/clientpositive/orc_merge2.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/orc_merge2.q?rev=1614793&view=auto ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/orc_merge2.q (added) +++ hive/trunk/ql/src/test/queries/clientpositive/orc_merge2.q Wed Jul 30 23:50:11 2014 @@ -0,0 +1,37 @@ +set hive.merge.orcfile.stripe.level=true; +set hive.exec.dynamic.partition=true; +set hive.exec.dynamic.partition.mode=nonstrict; + +DROP TABLE orcfile_merge2a; + +CREATE TABLE orcfile_merge2a (key INT, value STRING) + PARTITIONED BY (one string, two string, three string) + STORED AS ORC; + +EXPLAIN INSERT OVERWRITE TABLE orcfile_merge2a PARTITION (one='1', two, three) + SELECT key, value, PMOD(HASH(key), 10) as two, + PMOD(HASH(value), 10) as three + FROM src; + +INSERT OVERWRITE TABLE orcfile_merge2a PARTITION (one='1', two, three) + SELECT key, value, PMOD(HASH(key), 10) as two, + PMOD(HASH(value), 10) as three + FROM src; + +DESC FORMATTED orcfile_merge2a partition (one='1', two='0', three='2'); + +SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) + FROM orcfile_merge2a +) t; + +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; + +SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value, '1', PMOD(HASH(key), 10), + PMOD(HASH(value), 10)) USING 'tr \t _' AS (c) + FROM src +) t; + +DROP TABLE orcfile_merge2a; + Added: hive/trunk/ql/src/test/queries/clientpositive/orc_merge3.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/orc_merge3.q?rev=1614793&view=auto ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/orc_merge3.q (added) +++ hive/trunk/ql/src/test/queries/clientpositive/orc_merge3.q Wed Jul 30 23:50:11 2014 @@ -0,0 +1,37 @@ +set hive.merge.orcfile.stripe.level=true; + +DROP TABLE orcfile_merge3a; +DROP TABLE orcfile_merge3b; + +CREATE TABLE orcfile_merge3a (key int, value string) + PARTITIONED BY (ds string) STORED AS TEXTFILE; +CREATE TABLE orcfile_merge3b (key int, value string) STORED AS ORC; + +INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='1') + SELECT * FROM src; +INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='2') + SELECT * FROM src; + +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +EXPLAIN INSERT OVERWRITE TABLE orcfile_merge3b + SELECT key, value FROM orcfile_merge3a; + +INSERT OVERWRITE TABLE orcfile_merge3b + SELECT key, value FROM orcfile_merge3a; + +DESC FORMATTED orcfile_merge3b; + +SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) + FROM orcfile_merge3a +) t; + +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; + +SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) + FROM orcfile_merge3b +) t; + +DROP TABLE orcfile_merge3a; +DROP TABLE orcfile_merge3b; Added: hive/trunk/ql/src/test/queries/clientpositive/orc_merge4.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/orc_merge4.q?rev=1614793&view=auto ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/orc_merge4.q (added) +++ hive/trunk/ql/src/test/queries/clientpositive/orc_merge4.q Wed Jul 30 23:50:11 2014 @@ -0,0 +1,47 @@ +set hive.merge.orcfile.stripe.level=true; + +DROP TABLE orcfile_merge3a; +DROP TABLE orcfile_merge3b; + +CREATE TABLE orcfile_merge3a (key int, value string) + PARTITIONED BY (ds string) STORED AS ORC; +CREATE TABLE orcfile_merge3b (key int, value string) STORED AS TEXTFILE; + +set hive.merge.mapfiles=false; +INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='1') + SELECT * FROM src; + +DESC FORMATTED orcfile_merge3a PARTITION (ds='1'); + +set hive.merge.mapfiles=true; +INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='1') + SELECT * FROM src; + +INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='2') + SELECT * FROM src; + +DESC FORMATTED orcfile_merge3a PARTITION (ds='1'); +DESC FORMATTED orcfile_merge3a PARTITION (ds='2'); + +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +EXPLAIN INSERT OVERWRITE TABLE orcfile_merge3b + SELECT key, value FROM orcfile_merge3a; +INSERT OVERWRITE TABLE orcfile_merge3b + SELECT key, value FROM orcfile_merge3a; + +SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) + FROM orcfile_merge3a +) t; + +set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat; +set mapred.max.split.size=100; +set mapref.min.split.size=1; + +SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) + FROM orcfile_merge3b +) t; + +DROP TABLE orcfile_merge3a; +DROP TABLE orcfile_merge3b; Added: hive/trunk/ql/src/test/results/clientnegative/orc_merge1.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/orc_merge1.q.out?rev=1614793&view=auto ============================================================================== --- hive/trunk/ql/src/test/results/clientnegative/orc_merge1.q.out (added) +++ hive/trunk/ql/src/test/results/clientnegative/orc_merge1.q.out Wed Jul 30 23:50:11 2014 @@ -0,0 +1,74 @@ +PREHOOK: query: DROP TABLE orcfile_merge +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orcfile_merge +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orcfile_merge +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@orcfile_merge +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=12 +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@orcfile_merge +POSTHOOK: query: INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=12 +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- will fail because of different compression buffer size +ALTER TABLE orcfile_merge PARTITION(ds='2008-04-08',part=11) CONCATENATE +PREHOOK: type: ALTER_PARTITION_MERGE +PREHOOK: Input: default@orcfile_merge +PREHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +#### A masked pattern was here #### +FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.DDLTask Added: hive/trunk/ql/src/test/results/clientnegative/orc_merge2.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/orc_merge2.q.out?rev=1614793&view=auto ============================================================================== --- hive/trunk/ql/src/test/results/clientnegative/orc_merge2.q.out (added) +++ hive/trunk/ql/src/test/results/clientnegative/orc_merge2.q.out Wed Jul 30 23:50:11 2014 @@ -0,0 +1,74 @@ +PREHOOK: query: DROP TABLE orcfile_merge +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orcfile_merge +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orcfile_merge +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@orcfile_merge +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=12 +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@orcfile_merge +POSTHOOK: query: INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=12 +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- will fail because of different compression codec +ALTER TABLE orcfile_merge PARTITION(ds='2008-04-08',part=11) CONCATENATE +PREHOOK: type: ALTER_PARTITION_MERGE +PREHOOK: Input: default@orcfile_merge +PREHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +#### A masked pattern was here #### +FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.DDLTask Added: hive/trunk/ql/src/test/results/clientnegative/orc_merge3.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/orc_merge3.q.out?rev=1614793&view=auto ============================================================================== --- hive/trunk/ql/src/test/results/clientnegative/orc_merge3.q.out (added) +++ hive/trunk/ql/src/test/results/clientnegative/orc_merge3.q.out Wed Jul 30 23:50:11 2014 @@ -0,0 +1,74 @@ +PREHOOK: query: DROP TABLE orcfile_merge +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orcfile_merge +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orcfile_merge +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@orcfile_merge +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=12 +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@orcfile_merge +POSTHOOK: query: INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=12 +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- will fail because of different row index stride +ALTER TABLE orcfile_merge PARTITION(ds='2008-04-08',part=11) CONCATENATE +PREHOOK: type: ALTER_PARTITION_MERGE +PREHOOK: Input: default@orcfile_merge +PREHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +#### A masked pattern was here #### +FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.DDLTask Added: hive/trunk/ql/src/test/results/clientnegative/orc_merge4.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/orc_merge4.q.out?rev=1614793&view=auto ============================================================================== --- hive/trunk/ql/src/test/results/clientnegative/orc_merge4.q.out (added) +++ hive/trunk/ql/src/test/results/clientnegative/orc_merge4.q.out Wed Jul 30 23:50:11 2014 @@ -0,0 +1,86 @@ +PREHOOK: query: DROP TABLE orcfile_merge +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orcfile_merge +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orcfile_merge +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@orcfile_merge +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=12 +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE orcfile_merge ADD COLUMNS (newkey int) +PREHOOK: type: ALTERTABLE_ADDCOLS +PREHOOK: Input: default@orcfile_merge +PREHOOK: Output: default@orcfile_merge +POSTHOOK: query: ALTER TABLE orcfile_merge ADD COLUMNS (newkey int) +POSTHOOK: type: ALTERTABLE_ADDCOLS +POSTHOOK: Input: default@orcfile_merge +POSTHOOK: Output: default@orcfile_merge +PREHOOK: query: INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT key,value,key,ds,hr FROM srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@orcfile_merge +POSTHOOK: query: INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT key,value,key,ds,hr FROM srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=12 +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).newkey EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).newkey EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).newkey EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).newkey EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- will fail because of different column count +ALTER TABLE orcfile_merge PARTITION(ds='2008-04-08',part=11) CONCATENATE +PREHOOK: type: ALTER_PARTITION_MERGE +PREHOOK: Input: default@orcfile_merge +PREHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +#### A masked pattern was here #### +FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.DDLTask Added: hive/trunk/ql/src/test/results/clientnegative/orc_merge5.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/orc_merge5.q.out?rev=1614793&view=auto ============================================================================== --- hive/trunk/ql/src/test/results/clientnegative/orc_merge5.q.out (added) +++ hive/trunk/ql/src/test/results/clientnegative/orc_merge5.q.out Wed Jul 30 23:50:11 2014 @@ -0,0 +1,74 @@ +PREHOOK: query: DROP TABLE orcfile_merge +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orcfile_merge +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orcfile_merge +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@orcfile_merge +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=12 +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@orcfile_merge +POSTHOOK: query: INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=12 +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- will fail because of different write format +ALTER TABLE orcfile_merge PARTITION(ds='2008-04-08',part=11) CONCATENATE +PREHOOK: type: ALTER_PARTITION_MERGE +PREHOOK: Input: default@orcfile_merge +PREHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +#### A masked pattern was here #### +FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.DDLTask Added: hive/trunk/ql/src/test/results/clientpositive/alter_merge_2_orc.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/alter_merge_2_orc.q.out?rev=1614793&view=auto ============================================================================== --- hive/trunk/ql/src/test/results/clientpositive/alter_merge_2_orc.q.out (added) +++ hive/trunk/ql/src/test/results/clientpositive/alter_merge_2_orc.q.out Wed Jul 30 23:50:11 2014 @@ -0,0 +1,122 @@ +PREHOOK: query: create table src_orc_merge_test_part(key int, value string) partitioned by (ds string, ts string) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: create table src_orc_merge_test_part(key int, value string) partitioned by (ds string, ts string) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_orc_merge_test_part +PREHOOK: query: alter table src_orc_merge_test_part add partition (ds='2012-01-03', ts='2012-01-03+14:46:31') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@src_orc_merge_test_part +POSTHOOK: query: alter table src_orc_merge_test_part add partition (ds='2012-01-03', ts='2012-01-03+14:46:31') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@src_orc_merge_test_part +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +PREHOOK: query: desc extended src_orc_merge_test_part partition (ds='2012-01-03', ts='2012-01-03+14:46:31') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: query: desc extended src_orc_merge_test_part partition (ds='2012-01-03', ts='2012-01-03+14:46:31') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_orc_merge_test_part +key int +value string +ds string +ts string + +# Partition Information +# col_name data_type comment + +ds string +ts string + +#### A masked pattern was here #### +PREHOOK: query: insert overwrite table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +POSTHOOK: query: insert overwrite table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2012-01-03,ts=2012-01-03+14:46:31).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2012-01-03,ts=2012-01-03+14:46:31).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +POSTHOOK: query: insert into table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2012-01-03,ts=2012-01-03+14:46:31).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2012-01-03,ts=2012-01-03+14:46:31).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +POSTHOOK: query: insert into table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2012-01-03,ts=2012-01-03+14:46:31).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2012-01-03,ts=2012-01-03+14:46:31).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select count(1) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Input: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +#### A masked pattern was here #### +POSTHOOK: query: select count(1) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Input: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +#### A masked pattern was here #### +610 +PREHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Input: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Input: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +#### A masked pattern was here #### +159653 47778930280 +PREHOOK: query: alter table src_orc_merge_test_part partition (ds='2012-01-03', ts='2012-01-03+14:46:31') concatenate +PREHOOK: type: ALTER_PARTITION_MERGE +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +POSTHOOK: query: alter table src_orc_merge_test_part partition (ds='2012-01-03', ts='2012-01-03+14:46:31') concatenate +POSTHOOK: type: ALTER_PARTITION_MERGE +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +PREHOOK: query: select count(1) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Input: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +#### A masked pattern was here #### +POSTHOOK: query: select count(1) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Input: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +#### A masked pattern was here #### +610 +PREHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Input: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Input: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +#### A masked pattern was here #### +159653 47778930280 +PREHOOK: query: drop table src_orc_merge_test_part +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Output: default@src_orc_merge_test_part +POSTHOOK: query: drop table src_orc_merge_test_part +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Output: default@src_orc_merge_test_part
