http://git-wip-us.apache.org/repos/asf/hive/blob/18fb1b3f/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 2795e3d..9e012ce 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -194,6 +194,7 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\ orc_ppd_schema_evol_1b.q,\ orc_ppd_schema_evol_2a.q,\ orc_ppd_schema_evol_2b.q,\ + orc_schema_evol_quoted.q,\ parallel.q,\ partition_pruning.q,\ ptf.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/18fb1b3f/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java index 3c11847..f34f393 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java @@ -325,7 +325,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>, */ TypeDescription schema = getDesiredRowTypeDescr(conf, false, Integer.MAX_VALUE); - Reader.Options options = new Reader.Options().range(offset, length); + Reader.Options options = new Reader.Options(conf).range(offset, length); options.schema(schema); boolean isOriginal = isOriginal(file); if (schema == null) { @@ -2148,7 +2148,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>, */ TypeDescription schema = OrcInputFormat.getDesiredRowTypeDescr(conf, true, Integer.MAX_VALUE); - Reader.Options readerOptions = new Reader.Options().schema(schema); + Reader.Options readerOptions = new Reader.Options(conf).schema(schema); // TODO: Convert genIncludedColumns and setSearchArgument to use TypeDescription. final List<OrcProto.Type> schemaTypes = OrcUtils.getOrcTypes(schema); readerOptions.include(OrcInputFormat.genIncludedColumns(schema, conf)); @@ -2397,7 +2397,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>, mergerOptions.rootPath(deltaDirectory[0].getParent()); } return new OrcRawRecordMerger(conf, collapseEvents, null, isOriginal, - bucket, validWriteIdList, new Reader.Options(), deltaDirectory, mergerOptions); + bucket, validWriteIdList, new Reader.Options(conf), deltaDirectory, mergerOptions); } /** http://git-wip-us.apache.org/repos/asf/hive/blob/18fb1b3f/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java index 892fcc0..8cb9af7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java @@ -92,7 +92,7 @@ public class VectorizedOrcInputFormat extends FileInputFormat<NullWritable, Vect } } List<OrcProto.Type> types = OrcUtils.getOrcTypes(schema); - Reader.Options options = new Reader.Options().schema(schema); + Reader.Options options = new Reader.Options(conf).schema(schema); this.offset = fileSplit.getStart(); this.length = fileSplit.getLength(); http://git-wip-us.apache.org/repos/asf/hive/blob/18fb1b3f/ql/src/test/queries/clientpositive/orc_schema_evol_quoted.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/orc_schema_evol_quoted.q b/ql/src/test/queries/clientpositive/orc_schema_evol_quoted.q new file mode 100644 index 0000000..25461d2 --- /dev/null +++ b/ql/src/test/queries/clientpositive/orc_schema_evol_quoted.q @@ -0,0 +1,42 @@ +set hive.support.quoted.identifiers=column; +set orc.force.positional.evolution=true; +set hive.vectorized.execution.enabled=false; + +CREATE TABLE studenttab10k( + name string, + age int, + gpa double +) +ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' +STORED AS TEXTFILE; + +LOAD DATA LOCAL INPATH '../../data/files/studenttab10k' OVERWRITE INTO TABLE studenttab10k; + +create table temptable1( + `!@#$%^&*()_name` string, + age int, + gpa double +) stored as orc; + +insert overwrite table temptable1 select * from studenttab10k; + +alter table temptable1 change age `!@#$%^&*()_age` int; +alter table temptable1 change gpa `!@#$%^&*()_gpa` double; + +select `!@#$%^&*()_age`, count(*) from temptable1 group by `!@#$%^&*()_age` order by `!@#$%^&*()_age`; + +drop table temptable1; +set hive.vectorized.execution.enabled=true; + +create table temptable1( + `!@#$%^&*()_name` string, + age int, + gpa double +) stored as orc; + +insert overwrite table temptable1 select * from studenttab10k; + +alter table temptable1 change age `!@#$%^&*()_age` int; +alter table temptable1 change gpa `!@#$%^&*()_gpa` double; + +select `!@#$%^&*()_age`, count(*) from temptable1 group by `!@#$%^&*()_age` order by `!@#$%^&*()_age`; http://git-wip-us.apache.org/repos/asf/hive/blob/18fb1b3f/ql/src/test/results/clientpositive/llap/orc_schema_evol_quoted.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/orc_schema_evol_quoted.q.out b/ql/src/test/results/clientpositive/llap/orc_schema_evol_quoted.q.out new file mode 100644 index 0000000..7e65fd8 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/orc_schema_evol_quoted.q.out @@ -0,0 +1,258 @@ +PREHOOK: query: CREATE TABLE studenttab10k( + name string, + age int, + gpa double +) +ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' +STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@studenttab10k +POSTHOOK: query: CREATE TABLE studenttab10k( + name string, + age int, + gpa double +) +ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' +STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@studenttab10k +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/studenttab10k' OVERWRITE INTO TABLE studenttab10k +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@studenttab10k +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/studenttab10k' OVERWRITE INTO TABLE studenttab10k +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@studenttab10k +PREHOOK: query: create table temptable1( + `!@#$%^&*()_name` string, + age int, + gpa double +) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@temptable1 +POSTHOOK: query: create table temptable1( + `!@#$%^&*()_name` string, + age int, + gpa double +) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@temptable1 +PREHOOK: query: insert overwrite table temptable1 select * from studenttab10k +PREHOOK: type: QUERY +PREHOOK: Input: default@studenttab10k +PREHOOK: Output: default@temptable1 +POSTHOOK: query: insert overwrite table temptable1 select * from studenttab10k +POSTHOOK: type: QUERY +POSTHOOK: Input: default@studenttab10k +POSTHOOK: Output: default@temptable1 +POSTHOOK: Lineage: temptable1.!@#$%^&*()_name SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:name, type:string, comment:null), ] +POSTHOOK: Lineage: temptable1.age SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:age, type:int, comment:null), ] +POSTHOOK: Lineage: temptable1.gpa SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:gpa, type:double, comment:null), ] +PREHOOK: query: alter table temptable1 change age `!@#$%^&*()_age` int +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@temptable1 +PREHOOK: Output: default@temptable1 +POSTHOOK: query: alter table temptable1 change age `!@#$%^&*()_age` int +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@temptable1 +POSTHOOK: Output: default@temptable1 +PREHOOK: query: alter table temptable1 change gpa `!@#$%^&*()_gpa` double +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@temptable1 +PREHOOK: Output: default@temptable1 +POSTHOOK: query: alter table temptable1 change gpa `!@#$%^&*()_gpa` double +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@temptable1 +POSTHOOK: Output: default@temptable1 +PREHOOK: query: select `!@#$%^&*()_age`, count(*) from temptable1 group by `!@#$%^&*()_age` order by `!@#$%^&*()_age` +PREHOOK: type: QUERY +PREHOOK: Input: default@temptable1 +#### A masked pattern was here #### +POSTHOOK: query: select `!@#$%^&*()_age`, count(*) from temptable1 group by `!@#$%^&*()_age` order by `!@#$%^&*()_age` +POSTHOOK: type: QUERY +POSTHOOK: Input: default@temptable1 +#### A masked pattern was here #### +18 160 +19 164 +20 166 +21 165 +22 184 +23 148 +24 166 +25 170 +26 174 +27 173 +28 181 +29 173 +30 159 +31 147 +32 176 +33 160 +34 165 +35 163 +36 150 +37 177 +38 163 +39 162 +40 164 +41 156 +42 141 +43 156 +44 144 +45 142 +46 174 +47 178 +48 169 +49 158 +50 162 +51 174 +52 175 +53 171 +54 179 +55 166 +56 175 +57 184 +58 176 +59 162 +60 168 +61 175 +62 185 +63 175 +64 181 +65 166 +66 151 +67 165 +68 191 +69 177 +70 142 +71 175 +72 182 +73 168 +74 181 +75 149 +76 162 +77 155 +PREHOOK: query: drop table temptable1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@temptable1 +PREHOOK: Output: default@temptable1 +POSTHOOK: query: drop table temptable1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@temptable1 +POSTHOOK: Output: default@temptable1 +PREHOOK: query: create table temptable1( + `!@#$%^&*()_name` string, + age int, + gpa double +) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@temptable1 +POSTHOOK: query: create table temptable1( + `!@#$%^&*()_name` string, + age int, + gpa double +) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@temptable1 +PREHOOK: query: insert overwrite table temptable1 select * from studenttab10k +PREHOOK: type: QUERY +PREHOOK: Input: default@studenttab10k +PREHOOK: Output: default@temptable1 +POSTHOOK: query: insert overwrite table temptable1 select * from studenttab10k +POSTHOOK: type: QUERY +POSTHOOK: Input: default@studenttab10k +POSTHOOK: Output: default@temptable1 +POSTHOOK: Lineage: temptable1.!@#$%^&*()_name SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:name, type:string, comment:null), ] +POSTHOOK: Lineage: temptable1.age SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:age, type:int, comment:null), ] +POSTHOOK: Lineage: temptable1.gpa SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:gpa, type:double, comment:null), ] +PREHOOK: query: alter table temptable1 change age `!@#$%^&*()_age` int +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@temptable1 +PREHOOK: Output: default@temptable1 +POSTHOOK: query: alter table temptable1 change age `!@#$%^&*()_age` int +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@temptable1 +POSTHOOK: Output: default@temptable1 +PREHOOK: query: alter table temptable1 change gpa `!@#$%^&*()_gpa` double +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@temptable1 +PREHOOK: Output: default@temptable1 +POSTHOOK: query: alter table temptable1 change gpa `!@#$%^&*()_gpa` double +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@temptable1 +POSTHOOK: Output: default@temptable1 +PREHOOK: query: select `!@#$%^&*()_age`, count(*) from temptable1 group by `!@#$%^&*()_age` order by `!@#$%^&*()_age` +PREHOOK: type: QUERY +PREHOOK: Input: default@temptable1 +#### A masked pattern was here #### +POSTHOOK: query: select `!@#$%^&*()_age`, count(*) from temptable1 group by `!@#$%^&*()_age` order by `!@#$%^&*()_age` +POSTHOOK: type: QUERY +POSTHOOK: Input: default@temptable1 +#### A masked pattern was here #### +18 160 +19 164 +20 166 +21 165 +22 184 +23 148 +24 166 +25 170 +26 174 +27 173 +28 181 +29 173 +30 159 +31 147 +32 176 +33 160 +34 165 +35 163 +36 150 +37 177 +38 163 +39 162 +40 164 +41 156 +42 141 +43 156 +44 144 +45 142 +46 174 +47 178 +48 169 +49 158 +50 162 +51 174 +52 175 +53 171 +54 179 +55 166 +56 175 +57 184 +58 176 +59 162 +60 168 +61 175 +62 185 +63 175 +64 181 +65 166 +66 151 +67 165 +68 191 +69 177 +70 142 +71 175 +72 182 +73 168 +74 181 +75 149 +76 162 +77 155 http://git-wip-us.apache.org/repos/asf/hive/blob/18fb1b3f/ql/src/test/results/clientpositive/orc_schema_evol_quoted.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/orc_schema_evol_quoted.q.out b/ql/src/test/results/clientpositive/orc_schema_evol_quoted.q.out new file mode 100644 index 0000000..7e65fd8 --- /dev/null +++ b/ql/src/test/results/clientpositive/orc_schema_evol_quoted.q.out @@ -0,0 +1,258 @@ +PREHOOK: query: CREATE TABLE studenttab10k( + name string, + age int, + gpa double +) +ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' +STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@studenttab10k +POSTHOOK: query: CREATE TABLE studenttab10k( + name string, + age int, + gpa double +) +ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' +STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@studenttab10k +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/studenttab10k' OVERWRITE INTO TABLE studenttab10k +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@studenttab10k +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/studenttab10k' OVERWRITE INTO TABLE studenttab10k +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@studenttab10k +PREHOOK: query: create table temptable1( + `!@#$%^&*()_name` string, + age int, + gpa double +) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@temptable1 +POSTHOOK: query: create table temptable1( + `!@#$%^&*()_name` string, + age int, + gpa double +) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@temptable1 +PREHOOK: query: insert overwrite table temptable1 select * from studenttab10k +PREHOOK: type: QUERY +PREHOOK: Input: default@studenttab10k +PREHOOK: Output: default@temptable1 +POSTHOOK: query: insert overwrite table temptable1 select * from studenttab10k +POSTHOOK: type: QUERY +POSTHOOK: Input: default@studenttab10k +POSTHOOK: Output: default@temptable1 +POSTHOOK: Lineage: temptable1.!@#$%^&*()_name SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:name, type:string, comment:null), ] +POSTHOOK: Lineage: temptable1.age SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:age, type:int, comment:null), ] +POSTHOOK: Lineage: temptable1.gpa SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:gpa, type:double, comment:null), ] +PREHOOK: query: alter table temptable1 change age `!@#$%^&*()_age` int +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@temptable1 +PREHOOK: Output: default@temptable1 +POSTHOOK: query: alter table temptable1 change age `!@#$%^&*()_age` int +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@temptable1 +POSTHOOK: Output: default@temptable1 +PREHOOK: query: alter table temptable1 change gpa `!@#$%^&*()_gpa` double +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@temptable1 +PREHOOK: Output: default@temptable1 +POSTHOOK: query: alter table temptable1 change gpa `!@#$%^&*()_gpa` double +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@temptable1 +POSTHOOK: Output: default@temptable1 +PREHOOK: query: select `!@#$%^&*()_age`, count(*) from temptable1 group by `!@#$%^&*()_age` order by `!@#$%^&*()_age` +PREHOOK: type: QUERY +PREHOOK: Input: default@temptable1 +#### A masked pattern was here #### +POSTHOOK: query: select `!@#$%^&*()_age`, count(*) from temptable1 group by `!@#$%^&*()_age` order by `!@#$%^&*()_age` +POSTHOOK: type: QUERY +POSTHOOK: Input: default@temptable1 +#### A masked pattern was here #### +18 160 +19 164 +20 166 +21 165 +22 184 +23 148 +24 166 +25 170 +26 174 +27 173 +28 181 +29 173 +30 159 +31 147 +32 176 +33 160 +34 165 +35 163 +36 150 +37 177 +38 163 +39 162 +40 164 +41 156 +42 141 +43 156 +44 144 +45 142 +46 174 +47 178 +48 169 +49 158 +50 162 +51 174 +52 175 +53 171 +54 179 +55 166 +56 175 +57 184 +58 176 +59 162 +60 168 +61 175 +62 185 +63 175 +64 181 +65 166 +66 151 +67 165 +68 191 +69 177 +70 142 +71 175 +72 182 +73 168 +74 181 +75 149 +76 162 +77 155 +PREHOOK: query: drop table temptable1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@temptable1 +PREHOOK: Output: default@temptable1 +POSTHOOK: query: drop table temptable1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@temptable1 +POSTHOOK: Output: default@temptable1 +PREHOOK: query: create table temptable1( + `!@#$%^&*()_name` string, + age int, + gpa double +) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@temptable1 +POSTHOOK: query: create table temptable1( + `!@#$%^&*()_name` string, + age int, + gpa double +) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@temptable1 +PREHOOK: query: insert overwrite table temptable1 select * from studenttab10k +PREHOOK: type: QUERY +PREHOOK: Input: default@studenttab10k +PREHOOK: Output: default@temptable1 +POSTHOOK: query: insert overwrite table temptable1 select * from studenttab10k +POSTHOOK: type: QUERY +POSTHOOK: Input: default@studenttab10k +POSTHOOK: Output: default@temptable1 +POSTHOOK: Lineage: temptable1.!@#$%^&*()_name SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:name, type:string, comment:null), ] +POSTHOOK: Lineage: temptable1.age SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:age, type:int, comment:null), ] +POSTHOOK: Lineage: temptable1.gpa SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:gpa, type:double, comment:null), ] +PREHOOK: query: alter table temptable1 change age `!@#$%^&*()_age` int +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@temptable1 +PREHOOK: Output: default@temptable1 +POSTHOOK: query: alter table temptable1 change age `!@#$%^&*()_age` int +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@temptable1 +POSTHOOK: Output: default@temptable1 +PREHOOK: query: alter table temptable1 change gpa `!@#$%^&*()_gpa` double +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@temptable1 +PREHOOK: Output: default@temptable1 +POSTHOOK: query: alter table temptable1 change gpa `!@#$%^&*()_gpa` double +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@temptable1 +POSTHOOK: Output: default@temptable1 +PREHOOK: query: select `!@#$%^&*()_age`, count(*) from temptable1 group by `!@#$%^&*()_age` order by `!@#$%^&*()_age` +PREHOOK: type: QUERY +PREHOOK: Input: default@temptable1 +#### A masked pattern was here #### +POSTHOOK: query: select `!@#$%^&*()_age`, count(*) from temptable1 group by `!@#$%^&*()_age` order by `!@#$%^&*()_age` +POSTHOOK: type: QUERY +POSTHOOK: Input: default@temptable1 +#### A masked pattern was here #### +18 160 +19 164 +20 166 +21 165 +22 184 +23 148 +24 166 +25 170 +26 174 +27 173 +28 181 +29 173 +30 159 +31 147 +32 176 +33 160 +34 165 +35 163 +36 150 +37 177 +38 163 +39 162 +40 164 +41 156 +42 141 +43 156 +44 144 +45 142 +46 174 +47 178 +48 169 +49 158 +50 162 +51 174 +52 175 +53 171 +54 179 +55 166 +56 175 +57 184 +58 176 +59 162 +60 168 +61 175 +62 185 +63 175 +64 181 +65 166 +66 151 +67 165 +68 191 +69 177 +70 142 +71 175 +72 182 +73 168 +74 181 +75 149 +76 162 +77 155
