HIVE-11860: StatsNoJobTask fails to collect fast stats when table have subdirectories (Prasanth Jayachandran reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f08a0330 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f08a0330 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f08a0330 Branch: refs/heads/beeline-cli Commit: f08a0330bc98fbf80b878fd931f74df5e724a5cf Parents: 3672a27 Author: Prasanth Jayachandran <[email protected]> Authored: Fri Sep 18 17:13:13 2015 -0500 Committer: Prasanth Jayachandran <[email protected]> Committed: Fri Sep 18 17:13:13 2015 -0500 ---------------------------------------------------------------------- .../test/resources/testconfiguration.properties | 1 + .../hadoop/hive/ql/exec/StatsNoJobTask.java | 25 +- .../queries/clientpositive/union_fast_stats.q | 68 +++ .../clientpositive/tez/union_fast_stats.q.out | 526 +++++++++++++++++++ .../clientpositive/union_fast_stats.q.out | 526 +++++++++++++++++++ 5 files changed, 1128 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f08a0330/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 0d3e1cc..b47d1b5 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -177,6 +177,7 @@ minitez.query.files.shared=alter_merge_2_orc.q,\ union9.q,\ unionDistinct_1.q,\ unionDistinct_2.q,\ + union_fast_stats.q,\ update_after_multiple_inserts.q,\ update_all_non_partitioned.q,\ update_all_partitioned.q,\ http://git-wip-us.apache.org/repos/asf/hive/blob/f08a0330/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java index 4ecb20f..0d99cbc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java @@ -32,7 +32,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.hive.common.HiveStatsUtils; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; @@ -46,7 +46,6 @@ import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec; import org.apache.hadoop.hive.ql.plan.StatsNoJobWork; import org.apache.hadoop.hive.ql.plan.api.StageType; -import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.mapred.FileSplit; import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.InputSplit; @@ -144,9 +143,8 @@ public class StatsNoJobTask extends Task<StatsNoJobWork> implements Serializable long fileSize = 0; long numFiles = 0; FileSystem fs = dir.getFileSystem(conf); - List<FileStatus> fileList = - ShimLoader.getHadoopShims().listLocatedStatus(fs, dir, - hiddenFileFilter); + FileStatus[] fileList = HiveStatsUtils.getFileStatusRecurse(dir, -1, fs); + boolean statsAvailable = false; for(FileStatus file: fileList) { if (!file.isDir()) { @@ -155,7 +153,6 @@ public class StatsNoJobTask extends Task<StatsNoJobWork> implements Serializable InputSplit dummySplit = new FileSplit(file.getPath(), 0, 0, new String[] { partn.getLocation() }); org.apache.hadoop.mapred.RecordReader<?, ?> recordReader = - (org.apache.hadoop.mapred.RecordReader<?, ?>) inputFormat.getRecordReader(dummySplit, jc, Reporter.NULL); StatsProvidingRecordReader statsRR; if (recordReader instanceof StatsProvidingRecordReader) { @@ -242,9 +239,8 @@ public class StatsNoJobTask extends Task<StatsNoJobWork> implements Serializable long fileSize = 0; long numFiles = 0; FileSystem fs = dir.getFileSystem(conf); - List<FileStatus> fileList = - ShimLoader.getHadoopShims().listLocatedStatus(fs, dir, - hiddenFileFilter); + FileStatus[] fileList = HiveStatsUtils.getFileStatusRecurse(dir, -1, fs); + boolean statsAvailable = false; for(FileStatus file: fileList) { if (!file.isDir()) { @@ -252,8 +248,8 @@ public class StatsNoJobTask extends Task<StatsNoJobWork> implements Serializable table.getInputFormatClass(), jc); InputSplit dummySplit = new FileSplit(file.getPath(), 0, 0, new String[] { table .getDataLocation().toString() }); - org.apache.hadoop.mapred.RecordReader<?, ?> recordReader = (org.apache.hadoop.mapred.RecordReader<?, ?>) inputFormat - .getRecordReader(dummySplit, jc, Reporter.NULL); + org.apache.hadoop.mapred.RecordReader<?, ?> recordReader = + inputFormat.getRecordReader(dummySplit, jc, Reporter.NULL); StatsProvidingRecordReader statsRR; if (recordReader instanceof StatsProvidingRecordReader) { statsRR = (StatsProvidingRecordReader) recordReader; @@ -354,13 +350,6 @@ public class StatsNoJobTask extends Task<StatsNoJobWork> implements Serializable } } - private static final PathFilter hiddenFileFilter = new PathFilter() { - public boolean accept(Path p) { - String name = p.getName(); - return !name.startsWith("_") && !name.startsWith("."); - } - }; - private String toString(Map<String, String> parameters) { StringBuilder builder = new StringBuilder(); for (String statType : StatsSetupConst.supportedStats) { http://git-wip-us.apache.org/repos/asf/hive/blob/f08a0330/ql/src/test/queries/clientpositive/union_fast_stats.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/union_fast_stats.q b/ql/src/test/queries/clientpositive/union_fast_stats.q new file mode 100644 index 0000000..616af43 --- /dev/null +++ b/ql/src/test/queries/clientpositive/union_fast_stats.q @@ -0,0 +1,68 @@ +set hive.stats.dbclass=fs; +set hive.merge.mapfiles=false; +set hive.merge.mapredfiles=false; +set hive.merge.tezfiles=false; + +drop table small_alltypesorc1a; +drop table small_alltypesorc2a; +drop table small_alltypesorc3a; +drop table small_alltypesorc4a; +drop table small_alltypesorc_a; + +create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5; +create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5; +create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5; +create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5; + +create table small_alltypesorc_a stored as orc as select * from +(select * from (select * from small_alltypesorc1a) sq1 + union all + select * from (select * from small_alltypesorc2a) sq2 + union all + select * from (select * from small_alltypesorc3a) sq3 + union all + select * from (select * from small_alltypesorc4a) sq4) q; + +desc formatted small_alltypesorc_a; + +ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS; + +desc formatted small_alltypesorc_a; + +insert into table small_alltypesorc_a select * from small_alltypesorc1a; + +desc formatted small_alltypesorc_a; + +set hive.merge.mapfiles=true; +set hive.merge.mapredfiles=true; +set hive.merge.tezfiles=true; + +drop table small_alltypesorc1a; +drop table small_alltypesorc2a; +drop table small_alltypesorc3a; +drop table small_alltypesorc4a; +drop table small_alltypesorc_a; + +create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5; +create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5; +create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5; +create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5; + +create table small_alltypesorc_a stored as orc as select * from +(select * from (select * from small_alltypesorc1a) sq1 + union all + select * from (select * from small_alltypesorc2a) sq2 + union all + select * from (select * from small_alltypesorc3a) sq3 + union all + select * from (select * from small_alltypesorc4a) sq4) q; + +desc formatted small_alltypesorc_a; + +ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS; + +desc formatted small_alltypesorc_a; + +insert into table small_alltypesorc_a select * from small_alltypesorc1a; + +desc formatted small_alltypesorc_a; http://git-wip-us.apache.org/repos/asf/hive/blob/f08a0330/ql/src/test/results/clientpositive/tez/union_fast_stats.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/tez/union_fast_stats.q.out b/ql/src/test/results/clientpositive/tez/union_fast_stats.q.out new file mode 100644 index 0000000..d0d82a0 --- /dev/null +++ b/ql/src/test/results/clientpositive/tez/union_fast_stats.q.out @@ -0,0 +1,526 @@ +PREHOOK: query: drop table small_alltypesorc1a +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table small_alltypesorc1a +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table small_alltypesorc2a +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table small_alltypesorc2a +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table small_alltypesorc3a +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table small_alltypesorc3a +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table small_alltypesorc4a +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table small_alltypesorc4a +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table small_alltypesorc_a +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table small_alltypesorc_a +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: database:default +PREHOOK: Output: default@small_alltypesorc1a +POSTHOOK: query: create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@small_alltypesorc1a +PREHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: database:default +PREHOOK: Output: default@small_alltypesorc2a +POSTHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@small_alltypesorc2a +PREHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: database:default +PREHOOK: Output: default@small_alltypesorc3a +POSTHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@small_alltypesorc3a +PREHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: database:default +PREHOOK: Output: default@small_alltypesorc4a +POSTHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@small_alltypesorc4a +PREHOOK: query: create table small_alltypesorc_a stored as orc as select * from +(select * from (select * from small_alltypesorc1a) sq1 + union all + select * from (select * from small_alltypesorc2a) sq2 + union all + select * from (select * from small_alltypesorc3a) sq3 + union all + select * from (select * from small_alltypesorc4a) sq4) q +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@small_alltypesorc1a +PREHOOK: Input: default@small_alltypesorc2a +PREHOOK: Input: default@small_alltypesorc3a +PREHOOK: Input: default@small_alltypesorc4a +PREHOOK: Output: database:default +PREHOOK: Output: default@small_alltypesorc_a +POSTHOOK: query: create table small_alltypesorc_a stored as orc as select * from +(select * from (select * from small_alltypesorc1a) sq1 + union all + select * from (select * from small_alltypesorc2a) sq2 + union all + select * from (select * from small_alltypesorc3a) sq3 + union all + select * from (select * from small_alltypesorc4a) sq4) q +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@small_alltypesorc1a +POSTHOOK: Input: default@small_alltypesorc2a +POSTHOOK: Input: default@small_alltypesorc3a +POSTHOOK: Input: default@small_alltypesorc4a +POSTHOOK: Output: database:default +POSTHOOK: Output: default@small_alltypesorc_a +PREHOOK: query: desc formatted small_alltypesorc_a +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@small_alltypesorc_a +POSTHOOK: query: desc formatted small_alltypesorc_a +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@small_alltypesorc_a +# col_name data_type comment + +ctinyint tinyint +csmallint smallint +cint int +cbigint bigint +cfloat float +cdouble double +cstring1 string +cstring2 string +ctimestamp1 timestamp +ctimestamp2 timestamp +cboolean1 boolean +cboolean2 boolean + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 4 + numRows 0 + rawDataSize 0 + totalSize 3915 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS +PREHOOK: type: QUERY +PREHOOK: Input: default@small_alltypesorc_a +PREHOOK: Output: default@small_alltypesorc_a +POSTHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@small_alltypesorc_a +POSTHOOK: Output: default@small_alltypesorc_a +PREHOOK: query: desc formatted small_alltypesorc_a +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@small_alltypesorc_a +POSTHOOK: query: desc formatted small_alltypesorc_a +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@small_alltypesorc_a +# col_name data_type comment + +ctinyint tinyint +csmallint smallint +cint int +cbigint bigint +cfloat float +cdouble double +cstring1 string +cstring2 string +ctimestamp1 timestamp +ctimestamp2 timestamp +cboolean1 boolean +cboolean2 boolean + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 4 + numRows 15 + rawDataSize 3483 + totalSize 3915 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: insert into table small_alltypesorc_a select * from small_alltypesorc1a +PREHOOK: type: QUERY +PREHOOK: Input: default@small_alltypesorc1a +PREHOOK: Output: default@small_alltypesorc_a +POSTHOOK: query: insert into table small_alltypesorc_a select * from small_alltypesorc1a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@small_alltypesorc1a +POSTHOOK: Output: default@small_alltypesorc_a +POSTHOOK: Lineage: small_alltypesorc_a.cbigint SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cboolean1 SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cboolean1, type:boolean, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cboolean2 SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cboolean2, type:boolean, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cdouble SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cfloat SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cint SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.csmallint SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:csmallint, type:smallint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cstring1 SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cstring2 SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cstring2, type:string, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.ctimestamp1 SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.ctimestamp2 SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.ctinyint SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +PREHOOK: query: desc formatted small_alltypesorc_a +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@small_alltypesorc_a +POSTHOOK: query: desc formatted small_alltypesorc_a +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@small_alltypesorc_a +# col_name data_type comment + +ctinyint tinyint +csmallint smallint +cint int +cbigint bigint +cfloat float +cdouble double +cstring1 string +cstring2 string +ctimestamp1 timestamp +ctimestamp2 timestamp +cboolean1 boolean +cboolean2 boolean + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 5 + numRows 20 + rawDataSize 4552 + totalSize 5225 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table small_alltypesorc1a +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@small_alltypesorc1a +PREHOOK: Output: default@small_alltypesorc1a +POSTHOOK: query: drop table small_alltypesorc1a +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@small_alltypesorc1a +POSTHOOK: Output: default@small_alltypesorc1a +PREHOOK: query: drop table small_alltypesorc2a +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@small_alltypesorc2a +PREHOOK: Output: default@small_alltypesorc2a +POSTHOOK: query: drop table small_alltypesorc2a +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@small_alltypesorc2a +POSTHOOK: Output: default@small_alltypesorc2a +PREHOOK: query: drop table small_alltypesorc3a +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@small_alltypesorc3a +PREHOOK: Output: default@small_alltypesorc3a +POSTHOOK: query: drop table small_alltypesorc3a +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@small_alltypesorc3a +POSTHOOK: Output: default@small_alltypesorc3a +PREHOOK: query: drop table small_alltypesorc4a +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@small_alltypesorc4a +PREHOOK: Output: default@small_alltypesorc4a +POSTHOOK: query: drop table small_alltypesorc4a +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@small_alltypesorc4a +POSTHOOK: Output: default@small_alltypesorc4a +PREHOOK: query: drop table small_alltypesorc_a +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@small_alltypesorc_a +PREHOOK: Output: default@small_alltypesorc_a +POSTHOOK: query: drop table small_alltypesorc_a +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@small_alltypesorc_a +POSTHOOK: Output: default@small_alltypesorc_a +PREHOOK: query: create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: database:default +PREHOOK: Output: default@small_alltypesorc1a +POSTHOOK: query: create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@small_alltypesorc1a +PREHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: database:default +PREHOOK: Output: default@small_alltypesorc2a +POSTHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@small_alltypesorc2a +PREHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: database:default +PREHOOK: Output: default@small_alltypesorc3a +POSTHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@small_alltypesorc3a +PREHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: database:default +PREHOOK: Output: default@small_alltypesorc4a +POSTHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@small_alltypesorc4a +PREHOOK: query: create table small_alltypesorc_a stored as orc as select * from +(select * from (select * from small_alltypesorc1a) sq1 + union all + select * from (select * from small_alltypesorc2a) sq2 + union all + select * from (select * from small_alltypesorc3a) sq3 + union all + select * from (select * from small_alltypesorc4a) sq4) q +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@small_alltypesorc1a +PREHOOK: Input: default@small_alltypesorc2a +PREHOOK: Input: default@small_alltypesorc3a +PREHOOK: Input: default@small_alltypesorc4a +PREHOOK: Output: database:default +PREHOOK: Output: default@small_alltypesorc_a +POSTHOOK: query: create table small_alltypesorc_a stored as orc as select * from +(select * from (select * from small_alltypesorc1a) sq1 + union all + select * from (select * from small_alltypesorc2a) sq2 + union all + select * from (select * from small_alltypesorc3a) sq3 + union all + select * from (select * from small_alltypesorc4a) sq4) q +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@small_alltypesorc1a +POSTHOOK: Input: default@small_alltypesorc2a +POSTHOOK: Input: default@small_alltypesorc3a +POSTHOOK: Input: default@small_alltypesorc4a +POSTHOOK: Output: database:default +POSTHOOK: Output: default@small_alltypesorc_a +PREHOOK: query: desc formatted small_alltypesorc_a +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@small_alltypesorc_a +POSTHOOK: query: desc formatted small_alltypesorc_a +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@small_alltypesorc_a +# col_name data_type comment + +ctinyint tinyint +csmallint smallint +cint int +cbigint bigint +cfloat float +cdouble double +cstring1 string +cstring2 string +ctimestamp1 timestamp +ctimestamp2 timestamp +cboolean1 boolean +cboolean2 boolean + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 5 + rawDataSize 1069 + totalSize 3177 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS +PREHOOK: type: QUERY +PREHOOK: Input: default@small_alltypesorc_a +PREHOOK: Output: default@small_alltypesorc_a +POSTHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@small_alltypesorc_a +POSTHOOK: Output: default@small_alltypesorc_a +PREHOOK: query: desc formatted small_alltypesorc_a +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@small_alltypesorc_a +POSTHOOK: query: desc formatted small_alltypesorc_a +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@small_alltypesorc_a +# col_name data_type comment + +ctinyint tinyint +csmallint smallint +cint int +cbigint bigint +cfloat float +cdouble double +cstring1 string +cstring2 string +ctimestamp1 timestamp +ctimestamp2 timestamp +cboolean1 boolean +cboolean2 boolean + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 15 + rawDataSize 3320 + totalSize 3177 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: insert into table small_alltypesorc_a select * from small_alltypesorc1a +PREHOOK: type: QUERY +PREHOOK: Input: default@small_alltypesorc1a +PREHOOK: Output: default@small_alltypesorc_a +POSTHOOK: query: insert into table small_alltypesorc_a select * from small_alltypesorc1a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@small_alltypesorc1a +POSTHOOK: Output: default@small_alltypesorc_a +POSTHOOK: Lineage: small_alltypesorc_a.cbigint SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cboolean1 SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cboolean1, type:boolean, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cboolean2 SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cboolean2, type:boolean, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cdouble SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cfloat SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cint SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.csmallint SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:csmallint, type:smallint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cstring1 SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cstring2 SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cstring2, type:string, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.ctimestamp1 SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.ctimestamp2 SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.ctinyint SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +PREHOOK: query: desc formatted small_alltypesorc_a +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@small_alltypesorc_a +POSTHOOK: query: desc formatted small_alltypesorc_a +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@small_alltypesorc_a +# col_name data_type comment + +ctinyint tinyint +csmallint smallint +cint int +cbigint bigint +cfloat float +cdouble double +cstring1 string +cstring2 string +ctimestamp1 timestamp +ctimestamp2 timestamp +cboolean1 boolean +cboolean2 boolean + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 2 + numRows 20 + rawDataSize 4389 + totalSize 4487 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 http://git-wip-us.apache.org/repos/asf/hive/blob/f08a0330/ql/src/test/results/clientpositive/union_fast_stats.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/union_fast_stats.q.out b/ql/src/test/results/clientpositive/union_fast_stats.q.out new file mode 100644 index 0000000..71a0486 --- /dev/null +++ b/ql/src/test/results/clientpositive/union_fast_stats.q.out @@ -0,0 +1,526 @@ +PREHOOK: query: drop table small_alltypesorc1a +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table small_alltypesorc1a +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table small_alltypesorc2a +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table small_alltypesorc2a +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table small_alltypesorc3a +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table small_alltypesorc3a +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table small_alltypesorc4a +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table small_alltypesorc4a +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table small_alltypesorc_a +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table small_alltypesorc_a +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: database:default +PREHOOK: Output: default@small_alltypesorc1a +POSTHOOK: query: create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@small_alltypesorc1a +PREHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: database:default +PREHOOK: Output: default@small_alltypesorc2a +POSTHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@small_alltypesorc2a +PREHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: database:default +PREHOOK: Output: default@small_alltypesorc3a +POSTHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@small_alltypesorc3a +PREHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: database:default +PREHOOK: Output: default@small_alltypesorc4a +POSTHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@small_alltypesorc4a +PREHOOK: query: create table small_alltypesorc_a stored as orc as select * from +(select * from (select * from small_alltypesorc1a) sq1 + union all + select * from (select * from small_alltypesorc2a) sq2 + union all + select * from (select * from small_alltypesorc3a) sq3 + union all + select * from (select * from small_alltypesorc4a) sq4) q +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@small_alltypesorc1a +PREHOOK: Input: default@small_alltypesorc2a +PREHOOK: Input: default@small_alltypesorc3a +PREHOOK: Input: default@small_alltypesorc4a +PREHOOK: Output: database:default +PREHOOK: Output: default@small_alltypesorc_a +POSTHOOK: query: create table small_alltypesorc_a stored as orc as select * from +(select * from (select * from small_alltypesorc1a) sq1 + union all + select * from (select * from small_alltypesorc2a) sq2 + union all + select * from (select * from small_alltypesorc3a) sq3 + union all + select * from (select * from small_alltypesorc4a) sq4) q +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@small_alltypesorc1a +POSTHOOK: Input: default@small_alltypesorc2a +POSTHOOK: Input: default@small_alltypesorc3a +POSTHOOK: Input: default@small_alltypesorc4a +POSTHOOK: Output: database:default +POSTHOOK: Output: default@small_alltypesorc_a +PREHOOK: query: desc formatted small_alltypesorc_a +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@small_alltypesorc_a +POSTHOOK: query: desc formatted small_alltypesorc_a +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@small_alltypesorc_a +# col_name data_type comment + +ctinyint tinyint +csmallint smallint +cint int +cbigint bigint +cfloat float +cdouble double +cstring1 string +cstring2 string +ctimestamp1 timestamp +ctimestamp2 timestamp +cboolean1 boolean +cboolean2 boolean + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 4 + numRows 15 + rawDataSize 3483 + totalSize 3915 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS +PREHOOK: type: QUERY +PREHOOK: Input: default@small_alltypesorc_a +PREHOOK: Output: default@small_alltypesorc_a +POSTHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@small_alltypesorc_a +POSTHOOK: Output: default@small_alltypesorc_a +PREHOOK: query: desc formatted small_alltypesorc_a +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@small_alltypesorc_a +POSTHOOK: query: desc formatted small_alltypesorc_a +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@small_alltypesorc_a +# col_name data_type comment + +ctinyint tinyint +csmallint smallint +cint int +cbigint bigint +cfloat float +cdouble double +cstring1 string +cstring2 string +ctimestamp1 timestamp +ctimestamp2 timestamp +cboolean1 boolean +cboolean2 boolean + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 4 + numRows 15 + rawDataSize 3483 + totalSize 3915 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: insert into table small_alltypesorc_a select * from small_alltypesorc1a +PREHOOK: type: QUERY +PREHOOK: Input: default@small_alltypesorc1a +PREHOOK: Output: default@small_alltypesorc_a +POSTHOOK: query: insert into table small_alltypesorc_a select * from small_alltypesorc1a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@small_alltypesorc1a +POSTHOOK: Output: default@small_alltypesorc_a +POSTHOOK: Lineage: small_alltypesorc_a.cbigint SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cboolean1 SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cboolean1, type:boolean, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cboolean2 SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cboolean2, type:boolean, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cdouble SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cfloat SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cint SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.csmallint SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:csmallint, type:smallint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cstring1 SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cstring2 SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cstring2, type:string, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.ctimestamp1 SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.ctimestamp2 SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.ctinyint SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +PREHOOK: query: desc formatted small_alltypesorc_a +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@small_alltypesorc_a +POSTHOOK: query: desc formatted small_alltypesorc_a +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@small_alltypesorc_a +# col_name data_type comment + +ctinyint tinyint +csmallint smallint +cint int +cbigint bigint +cfloat float +cdouble double +cstring1 string +cstring2 string +ctimestamp1 timestamp +ctimestamp2 timestamp +cboolean1 boolean +cboolean2 boolean + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 5 + numRows 20 + rawDataSize 4552 + totalSize 5225 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table small_alltypesorc1a +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@small_alltypesorc1a +PREHOOK: Output: default@small_alltypesorc1a +POSTHOOK: query: drop table small_alltypesorc1a +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@small_alltypesorc1a +POSTHOOK: Output: default@small_alltypesorc1a +PREHOOK: query: drop table small_alltypesorc2a +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@small_alltypesorc2a +PREHOOK: Output: default@small_alltypesorc2a +POSTHOOK: query: drop table small_alltypesorc2a +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@small_alltypesorc2a +POSTHOOK: Output: default@small_alltypesorc2a +PREHOOK: query: drop table small_alltypesorc3a +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@small_alltypesorc3a +PREHOOK: Output: default@small_alltypesorc3a +POSTHOOK: query: drop table small_alltypesorc3a +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@small_alltypesorc3a +POSTHOOK: Output: default@small_alltypesorc3a +PREHOOK: query: drop table small_alltypesorc4a +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@small_alltypesorc4a +PREHOOK: Output: default@small_alltypesorc4a +POSTHOOK: query: drop table small_alltypesorc4a +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@small_alltypesorc4a +POSTHOOK: Output: default@small_alltypesorc4a +PREHOOK: query: drop table small_alltypesorc_a +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@small_alltypesorc_a +PREHOOK: Output: default@small_alltypesorc_a +POSTHOOK: query: drop table small_alltypesorc_a +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@small_alltypesorc_a +POSTHOOK: Output: default@small_alltypesorc_a +PREHOOK: query: create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: database:default +PREHOOK: Output: default@small_alltypesorc1a +POSTHOOK: query: create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@small_alltypesorc1a +PREHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: database:default +PREHOOK: Output: default@small_alltypesorc2a +POSTHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@small_alltypesorc2a +PREHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: database:default +PREHOOK: Output: default@small_alltypesorc3a +POSTHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@small_alltypesorc3a +PREHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: database:default +PREHOOK: Output: default@small_alltypesorc4a +POSTHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@small_alltypesorc4a +PREHOOK: query: create table small_alltypesorc_a stored as orc as select * from +(select * from (select * from small_alltypesorc1a) sq1 + union all + select * from (select * from small_alltypesorc2a) sq2 + union all + select * from (select * from small_alltypesorc3a) sq3 + union all + select * from (select * from small_alltypesorc4a) sq4) q +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@small_alltypesorc1a +PREHOOK: Input: default@small_alltypesorc2a +PREHOOK: Input: default@small_alltypesorc3a +PREHOOK: Input: default@small_alltypesorc4a +PREHOOK: Output: database:default +PREHOOK: Output: default@small_alltypesorc_a +POSTHOOK: query: create table small_alltypesorc_a stored as orc as select * from +(select * from (select * from small_alltypesorc1a) sq1 + union all + select * from (select * from small_alltypesorc2a) sq2 + union all + select * from (select * from small_alltypesorc3a) sq3 + union all + select * from (select * from small_alltypesorc4a) sq4) q +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@small_alltypesorc1a +POSTHOOK: Input: default@small_alltypesorc2a +POSTHOOK: Input: default@small_alltypesorc3a +POSTHOOK: Input: default@small_alltypesorc4a +POSTHOOK: Output: database:default +POSTHOOK: Output: default@small_alltypesorc_a +PREHOOK: query: desc formatted small_alltypesorc_a +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@small_alltypesorc_a +POSTHOOK: query: desc formatted small_alltypesorc_a +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@small_alltypesorc_a +# col_name data_type comment + +ctinyint tinyint +csmallint smallint +cint int +cbigint bigint +cfloat float +cdouble double +cstring1 string +cstring2 string +ctimestamp1 timestamp +ctimestamp2 timestamp +cboolean1 boolean +cboolean2 boolean + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 15 + rawDataSize 3483 + totalSize 3176 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS +PREHOOK: type: QUERY +PREHOOK: Input: default@small_alltypesorc_a +PREHOOK: Output: default@small_alltypesorc_a +POSTHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@small_alltypesorc_a +POSTHOOK: Output: default@small_alltypesorc_a +PREHOOK: query: desc formatted small_alltypesorc_a +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@small_alltypesorc_a +POSTHOOK: query: desc formatted small_alltypesorc_a +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@small_alltypesorc_a +# col_name data_type comment + +ctinyint tinyint +csmallint smallint +cint int +cbigint bigint +cfloat float +cdouble double +cstring1 string +cstring2 string +ctimestamp1 timestamp +ctimestamp2 timestamp +cboolean1 boolean +cboolean2 boolean + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 15 + rawDataSize 3320 + totalSize 3176 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: insert into table small_alltypesorc_a select * from small_alltypesorc1a +PREHOOK: type: QUERY +PREHOOK: Input: default@small_alltypesorc1a +PREHOOK: Output: default@small_alltypesorc_a +POSTHOOK: query: insert into table small_alltypesorc_a select * from small_alltypesorc1a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@small_alltypesorc1a +POSTHOOK: Output: default@small_alltypesorc_a +POSTHOOK: Lineage: small_alltypesorc_a.cbigint SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cboolean1 SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cboolean1, type:boolean, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cboolean2 SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cboolean2, type:boolean, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cdouble SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cfloat SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cint SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.csmallint SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:csmallint, type:smallint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cstring1 SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.cstring2 SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:cstring2, type:string, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.ctimestamp1 SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.ctimestamp2 SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc_a.ctinyint SIMPLE [(small_alltypesorc1a)small_alltypesorc1a.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +PREHOOK: query: desc formatted small_alltypesorc_a +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@small_alltypesorc_a +POSTHOOK: query: desc formatted small_alltypesorc_a +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@small_alltypesorc_a +# col_name data_type comment + +ctinyint tinyint +csmallint smallint +cint int +cbigint bigint +cfloat float +cdouble double +cstring1 string +cstring2 string +ctimestamp1 timestamp +ctimestamp2 timestamp +cboolean1 boolean +cboolean2 boolean + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 2 + numRows 20 + rawDataSize 4389 + totalSize 4486 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1
