Modified: hive/branches/spark/ql/src/test/results/clientpositive/vectorization_short_regress.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/vectorization_short_regress.q.out?rev=1626482&r1=1626481&r2=1626482&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/clientpositive/vectorization_short_regress.q.out (original) +++ hive/branches/spark/ql/src/test/results/clientpositive/vectorization_short_regress.q.out Sat Sep 20 17:34:39 2014 @@ -6615,3 +6615,567 @@ POSTHOOK: Input: default@alltypesorc #### A masked pattern was here #### false 11.0 -11.0 -2.389090909090909 -17881597706 -1.7881597716175E10 3.8953387713327066E17 6.0 -0.8249999999999993 -2454.8879999999995 3.8953385925167296E17 -2145884705 1.66288903197104486E18 0.8249999999999993 4.7840233756130287E-17 4.098424268084119E-17 0.8249999999999993 -1051696618 28.692556844886422 2.980633855245E9 -4.032330473245E9 85.79562278396777 4.032330473245E9 -3983699.3106060605 3983699.3106060605 4.1896430920933255E15 true 79.553 -79.553 -0.33034580136836733 -401322621137 -4.01322621147175E11 7.9255373737244976E16 34.727455139160156 -69.3780014038086 4856.6352637899645 7.9254972414623824E16 -2130544867 2.30133924842409523E18 69.3780014038086 3.456813247089758E-17 2.0387240975807185E-18 69.3780014038086 2182477964777 34.654968050508266 2.959326820263E9 2.179518637956737E12 9461.197516216069 -2.179518637956737E12 4.592756659884259E8 -4.592756659884259E8 1.002359020778021E21 +PREHOOK: query: -- These tests verify COUNT on empty or null colulmns work correctly. +create table test_count(i int) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_count +POSTHOOK: query: -- These tests verify COUNT on empty or null colulmns work correctly. +create table test_count(i int) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_count +PREHOOK: query: explain +select count(*) from test_count +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(*) from test_count +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: test_count + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from test_count +PREHOOK: type: QUERY +PREHOOK: Input: default@test_count +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from test_count +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_count +#### A masked pattern was here #### +0 +PREHOOK: query: explain +select count(i) from test_count +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(i) from test_count +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: test_count + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: i (type: int) + outputColumnNames: i + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Group By Operator + aggregations: count(i) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(i) from test_count +PREHOOK: type: QUERY +PREHOOK: Input: default@test_count +#### A masked pattern was here #### +POSTHOOK: query: select count(i) from test_count +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_count +#### A masked pattern was here #### +0 +PREHOOK: query: create table alltypesnull like alltypesorc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@alltypesnull +POSTHOOK: query: create table alltypesnull like alltypesorc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@alltypesnull +PREHOOK: query: alter table alltypesnull set fileformat textfile +PREHOOK: type: ALTERTABLE_FILEFORMAT +PREHOOK: Input: default@alltypesnull +PREHOOK: Output: default@alltypesnull +POSTHOOK: query: alter table alltypesnull set fileformat textfile +POSTHOOK: type: ALTERTABLE_FILEFORMAT +POSTHOOK: Input: default@alltypesnull +POSTHOOK: Output: default@alltypesnull +PREHOOK: query: insert into table alltypesnull select null, null, null, null, null, null, null, null, null, null, null, null from alltypesorc +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@alltypesnull +POSTHOOK: query: insert into table alltypesnull select null, null, null, null, null, null, null, null, null, null, null, null from alltypesorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@alltypesnull +POSTHOOK: Lineage: alltypesnull.cbigint EXPRESSION [] +POSTHOOK: Lineage: alltypesnull.cboolean1 EXPRESSION [] +POSTHOOK: Lineage: alltypesnull.cboolean2 EXPRESSION [] +POSTHOOK: Lineage: alltypesnull.cdouble EXPRESSION [] +POSTHOOK: Lineage: alltypesnull.cfloat EXPRESSION [] +POSTHOOK: Lineage: alltypesnull.cint EXPRESSION [] +POSTHOOK: Lineage: alltypesnull.csmallint EXPRESSION [] +POSTHOOK: Lineage: alltypesnull.cstring1 SIMPLE [] +POSTHOOK: Lineage: alltypesnull.cstring2 SIMPLE [] +POSTHOOK: Lineage: alltypesnull.ctimestamp1 EXPRESSION [] +POSTHOOK: Lineage: alltypesnull.ctimestamp2 EXPRESSION [] +POSTHOOK: Lineage: alltypesnull.ctinyint EXPRESSION [] +PREHOOK: query: create table alltypesnullorc stored as orc as select * from alltypesnull +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesnull +PREHOOK: Output: database:default +PREHOOK: Output: default@alltypesnullorc +POSTHOOK: query: create table alltypesnullorc stored as orc as select * from alltypesnull +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesnull +POSTHOOK: Output: database:default +POSTHOOK: Output: default@alltypesnullorc +PREHOOK: query: explain +select count(*) from alltypesnullorc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(*) from alltypesnullorc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: alltypesnullorc + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from alltypesnullorc +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from alltypesnullorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +12288 +PREHOOK: query: explain +select count(ctinyint) from alltypesnullorc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(ctinyint) from alltypesnullorc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: alltypesnullorc + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ctinyint (type: tinyint) + outputColumnNames: ctinyint + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(ctinyint) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(ctinyint) from alltypesnullorc +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +POSTHOOK: query: select count(ctinyint) from alltypesnullorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +0 +PREHOOK: query: explain +select count(cint) from alltypesnullorc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(cint) from alltypesnullorc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: alltypesnullorc + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cint (type: int) + outputColumnNames: cint + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(cint) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(cint) from alltypesnullorc +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +POSTHOOK: query: select count(cint) from alltypesnullorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +0 +PREHOOK: query: explain +select count(cfloat) from alltypesnullorc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(cfloat) from alltypesnullorc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: alltypesnullorc + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cfloat (type: float) + outputColumnNames: cfloat + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(cfloat) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(cfloat) from alltypesnullorc +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +POSTHOOK: query: select count(cfloat) from alltypesnullorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +0 +PREHOOK: query: explain +select count(cstring1) from alltypesnullorc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(cstring1) from alltypesnullorc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: alltypesnullorc + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cstring1 (type: string) + outputColumnNames: cstring1 + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(cstring1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(cstring1) from alltypesnullorc +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +POSTHOOK: query: select count(cstring1) from alltypesnullorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +0 +PREHOOK: query: explain +select count(cboolean1) from alltypesnullorc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(cboolean1) from alltypesnullorc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: alltypesnullorc + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cboolean1 (type: boolean) + outputColumnNames: cboolean1 + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(cboolean1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(cboolean1) from alltypesnullorc +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +POSTHOOK: query: select count(cboolean1) from alltypesnullorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +0
Modified: hive/branches/spark/serde/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/pom.xml?rev=1626482&r1=1626481&r2=1626482&view=diff ============================================================================== --- hive/branches/spark/serde/pom.xml (original) +++ hive/branches/spark/serde/pom.xml Sat Sep 20 17:34:39 2014 @@ -70,7 +70,13 @@ <artifactId>libthrift</artifactId> <version>${libthrift.version}</version> </dependency> - <!-- test inter-project --> + <dependency> + <groupId>net.sf.opencsv</groupId> + <artifactId>opencsv</artifactId> + <version>${opencsv.version}</version> + </dependency> + + <!-- test inter-project --> <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> Modified: hive/branches/spark/serde/src/gen/thrift/gen-cpp/complex_types.cpp URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/gen/thrift/gen-cpp/complex_types.cpp?rev=1626482&r1=1626481&r2=1626482&view=diff ============================================================================== --- hive/branches/spark/serde/src/gen/thrift/gen-cpp/complex_types.cpp (original) +++ hive/branches/spark/serde/src/gen/thrift/gen-cpp/complex_types.cpp Sat Sep 20 17:34:39 2014 @@ -622,4 +622,100 @@ void swap(Complex &a, Complex &b) { swap(a.__isset, b.__isset); } +const char* SetIntString::ascii_fingerprint = "842B41C940D05DFB16183142A90DFC54"; +const uint8_t SetIntString::binary_fingerprint[16] = {0x84,0x2B,0x41,0xC9,0x40,0xD0,0x5D,0xFB,0x16,0x18,0x31,0x42,0xA9,0x0D,0xFC,0x54}; + +uint32_t SetIntString::read(::apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_SET) { + { + this->sIntString.clear(); + uint32_t _size64; + ::apache::thrift::protocol::TType _etype67; + xfer += iprot->readSetBegin(_etype67, _size64); + uint32_t _i68; + for (_i68 = 0; _i68 < _size64; ++_i68) + { + IntString _elem69; + xfer += _elem69.read(iprot); + this->sIntString.insert(_elem69); + } + xfer += iprot->readSetEnd(); + } + this->__isset.sIntString = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(this->aString); + this->__isset.aString = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t SetIntString::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("SetIntString"); + + xfer += oprot->writeFieldBegin("sIntString", ::apache::thrift::protocol::T_SET, 1); + { + xfer += oprot->writeSetBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(this->sIntString.size())); + std::set<IntString> ::const_iterator _iter70; + for (_iter70 = this->sIntString.begin(); _iter70 != this->sIntString.end(); ++_iter70) + { + xfer += (*_iter70).write(oprot); + } + xfer += oprot->writeSetEnd(); + } + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("aString", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeString(this->aString); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(SetIntString &a, SetIntString &b) { + using ::std::swap; + swap(a.sIntString, b.sIntString); + swap(a.aString, b.aString); + swap(a.__isset, b.__isset); +} + Modified: hive/branches/spark/serde/src/gen/thrift/gen-cpp/complex_types.h URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/gen/thrift/gen-cpp/complex_types.h?rev=1626482&r1=1626481&r2=1626482&view=diff ============================================================================== --- hive/branches/spark/serde/src/gen/thrift/gen-cpp/complex_types.h (original) +++ hive/branches/spark/serde/src/gen/thrift/gen-cpp/complex_types.h Sat Sep 20 17:34:39 2014 @@ -296,6 +296,57 @@ class Complex { void swap(Complex &a, Complex &b); +typedef struct _SetIntString__isset { + _SetIntString__isset() : sIntString(false), aString(false) {} + bool sIntString; + bool aString; +} _SetIntString__isset; + +class SetIntString { + public: + + static const char* ascii_fingerprint; // = "842B41C940D05DFB16183142A90DFC54"; + static const uint8_t binary_fingerprint[16]; // = {0x84,0x2B,0x41,0xC9,0x40,0xD0,0x5D,0xFB,0x16,0x18,0x31,0x42,0xA9,0x0D,0xFC,0x54}; + + SetIntString() : aString() { + } + + virtual ~SetIntString() throw() {} + + std::set<IntString> sIntString; + std::string aString; + + _SetIntString__isset __isset; + + void __set_sIntString(const std::set<IntString> & val) { + sIntString = val; + } + + void __set_aString(const std::string& val) { + aString = val; + } + + bool operator == (const SetIntString & rhs) const + { + if (!(sIntString == rhs.sIntString)) + return false; + if (!(aString == rhs.aString)) + return false; + return true; + } + bool operator != (const SetIntString &rhs) const { + return !(*this == rhs); + } + + bool operator < (const SetIntString & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + +void swap(SetIntString &a, SetIntString &b); + #endif Modified: hive/branches/spark/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java?rev=1626482&r1=1626481&r2=1626482&view=diff ============================================================================== --- hive/branches/spark/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java (original) +++ hive/branches/spark/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java Sat Sep 20 17:34:39 2014 @@ -37,8 +37,6 @@ public class serdeConstants { public static final String SERIALIZATION_CLASS = "serialization.class"; - public static final String SERIALIZATION_TYPE = "serialization.type"; - public static final String SERIALIZATION_FORMAT = "serialization.format"; public static final String SERIALIZATION_DDL = "serialization.ddl"; Modified: hive/branches/spark/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/Complex.java URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/Complex.java?rev=1626482&r1=1626481&r2=1626482&view=diff ============================================================================== --- hive/branches/spark/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/Complex.java (original) +++ hive/branches/spark/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/Complex.java Sat Sep 20 17:34:39 2014 @@ -1267,7 +1267,7 @@ public class Complex implements org.apac for (int _i28 = 0; _i28 < _map27.size; ++_i28) { String _key29; // required - String _val30; // optional + String _val30; // required _key29 = iprot.readString(); _val30 = iprot.readString(); struct.mStringString.put(_key29, _val30); @@ -1287,7 +1287,7 @@ public class Complex implements org.apac for (int _i32 = 0; _i32 < _map31.size; ++_i32) { String _key33; // required - Map<String,Map<String,PropValueUnion>> _val34; // optional + Map<String,Map<String,PropValueUnion>> _val34; // required _key33 = iprot.readString(); { org.apache.thrift.protocol.TMap _map35 = iprot.readMapBegin(); @@ -1295,7 +1295,7 @@ public class Complex implements org.apac for (int _i36 = 0; _i36 < _map35.size; ++_i36) { String _key37; // required - Map<String,PropValueUnion> _val38; // optional + Map<String,PropValueUnion> _val38; // required _key37 = iprot.readString(); { org.apache.thrift.protocol.TMap _map39 = iprot.readMapBegin(); @@ -1303,7 +1303,7 @@ public class Complex implements org.apac for (int _i40 = 0; _i40 < _map39.size; ++_i40) { String _key41; // required - PropValueUnion _val42; // optional + PropValueUnion _val42; // required _key41 = iprot.readString(); _val42 = new PropValueUnion(); _val42.read(iprot); @@ -1651,7 +1651,7 @@ public class Complex implements org.apac for (int _i67 = 0; _i67 < _map66.size; ++_i67) { String _key68; // required - String _val69; // optional + String _val69; // required _key68 = iprot.readString(); _val69 = iprot.readString(); struct.mStringString.put(_key68, _val69); @@ -1666,7 +1666,7 @@ public class Complex implements org.apac for (int _i71 = 0; _i71 < _map70.size; ++_i71) { String _key72; // required - Map<String,Map<String,PropValueUnion>> _val73; // optional + Map<String,Map<String,PropValueUnion>> _val73; // required _key72 = iprot.readString(); { org.apache.thrift.protocol.TMap _map74 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.MAP, iprot.readI32()); @@ -1674,7 +1674,7 @@ public class Complex implements org.apac for (int _i75 = 0; _i75 < _map74.size; ++_i75) { String _key76; // required - Map<String,PropValueUnion> _val77; // optional + Map<String,PropValueUnion> _val77; // required _key76 = iprot.readString(); { org.apache.thrift.protocol.TMap _map78 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); @@ -1682,7 +1682,7 @@ public class Complex implements org.apac for (int _i79 = 0; _i79 < _map78.size; ++_i79) { String _key80; // required - PropValueUnion _val81; // optional + PropValueUnion _val81; // required _key80 = iprot.readString(); _val81 = new PropValueUnion(); _val81.read(iprot); Modified: hive/branches/spark/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/PropValueUnion.java URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/PropValueUnion.java?rev=1626482&r1=1626481&r2=1626482&view=diff ============================================================================== --- hive/branches/spark/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/PropValueUnion.java (original) +++ hive/branches/spark/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/PropValueUnion.java Sat Sep 20 17:34:39 2014 @@ -320,7 +320,7 @@ public class PropValueUnion extends org. for (int _i4 = 0; _i4 < _map3.size; ++_i4) { String _key5; // required - String _val6; // optional + String _val6; // required _key5 = iprot.readString(); _val6 = iprot.readString(); unionMStringString.put(_key5, _val6); @@ -438,7 +438,7 @@ public class PropValueUnion extends org. for (int _i13 = 0; _i13 < _map12.size; ++_i13) { String _key14; // required - String _val15; // optional + String _val15; // required _key14 = iprot.readString(); _val15 = iprot.readString(); unionMStringString.put(_key14, _val15); Modified: hive/branches/spark/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/SetIntString.java URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/SetIntString.java?rev=1626482&r1=1626481&r2=1626482&view=diff ============================================================================== --- hive/branches/spark/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/SetIntString.java (original) +++ hive/branches/spark/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/SetIntString.java Sat Sep 20 17:34:39 2014 @@ -6,6 +6,7 @@ */ package org.apache.hadoop.hive.serde2.thrift.test; +import org.apache.commons.lang.builder.HashCodeBuilder; import org.apache.thrift.scheme.IScheme; import org.apache.thrift.scheme.SchemeFactory; import org.apache.thrift.scheme.StandardScheme; @@ -42,8 +43,8 @@ public class SetIntString implements org schemes.put(TupleScheme.class, new SetIntStringTupleSchemeFactory()); } - public Set<IntString> sIntString; // required - public String aString; // required + private Set<IntString> sIntString; // required + private String aString; // required /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { @@ -176,9 +177,8 @@ public class SetIntString implements org return this.sIntString; } - public SetIntString setSIntString(Set<IntString> sIntString) { + public void setSIntString(Set<IntString> sIntString) { this.sIntString = sIntString; - return this; } public void unsetSIntString() { @@ -200,9 +200,8 @@ public class SetIntString implements org return this.aString; } - public SetIntString setAString(String aString) { + public void setAString(String aString) { this.aString = aString; - return this; } public void unsetAString() { @@ -304,7 +303,19 @@ public class SetIntString implements org @Override public int hashCode() { - return 0; + HashCodeBuilder builder = new HashCodeBuilder(); + + boolean present_sIntString = true && (isSetSIntString()); + builder.append(present_sIntString); + if (present_sIntString) + builder.append(sIntString); + + boolean present_aString = true && (isSetAString()); + builder.append(present_aString); + if (present_aString) + builder.append(aString); + + return builder.toHashCode(); } public int compareTo(SetIntString other) { @@ -416,14 +427,14 @@ public class SetIntString implements org case 1: // S_INT_STRING if (schemeField.type == org.apache.thrift.protocol.TType.SET) { { - org.apache.thrift.protocol.TSet _set34 = iprot.readSetBegin(); - struct.sIntString = new HashSet<IntString>(2*_set34.size); - for (int _i35 = 0; _i35 < _set34.size; ++_i35) + org.apache.thrift.protocol.TSet _set82 = iprot.readSetBegin(); + struct.sIntString = new HashSet<IntString>(2*_set82.size); + for (int _i83 = 0; _i83 < _set82.size; ++_i83) { - IntString _elem36; // required - _elem36 = new IntString(); - _elem36.read(iprot); - struct.sIntString.add(_elem36); + IntString _elem84; // required + _elem84 = new IntString(); + _elem84.read(iprot); + struct.sIntString.add(_elem84); } iprot.readSetEnd(); } @@ -446,8 +457,6 @@ public class SetIntString implements org iprot.readFieldEnd(); } iprot.readStructEnd(); - - // check for required fields of primitive type, which can't be checked in the validate method struct.validate(); } @@ -459,9 +468,9 @@ public class SetIntString implements org oprot.writeFieldBegin(S_INT_STRING_FIELD_DESC); { oprot.writeSetBegin(new org.apache.thrift.protocol.TSet(org.apache.thrift.protocol.TType.STRUCT, struct.sIntString.size())); - for (IntString _iter37 : struct.sIntString) + for (IntString _iter85 : struct.sIntString) { - _iter37.write(oprot); + _iter85.write(oprot); } oprot.writeSetEnd(); } @@ -500,9 +509,9 @@ public class SetIntString implements org if (struct.isSetSIntString()) { { oprot.writeI32(struct.sIntString.size()); - for (IntString _iter38 : struct.sIntString) + for (IntString _iter86 : struct.sIntString) { - _iter38.write(oprot); + _iter86.write(oprot); } } } @@ -517,14 +526,14 @@ public class SetIntString implements org BitSet incoming = iprot.readBitSet(2); if (incoming.get(0)) { { - org.apache.thrift.protocol.TSet _set39 = new org.apache.thrift.protocol.TSet(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); - struct.sIntString = new HashSet<IntString>(2*_set39.size); - for (int _i40 = 0; _i40 < _set39.size; ++_i40) + org.apache.thrift.protocol.TSet _set87 = new org.apache.thrift.protocol.TSet(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.sIntString = new HashSet<IntString>(2*_set87.size); + for (int _i88 = 0; _i88 < _set87.size; ++_i88) { - IntString _elem41; // required - _elem41 = new IntString(); - _elem41.read(iprot); - struct.sIntString.add(_elem41); + IntString _elem89; // required + _elem89 = new IntString(); + _elem89.read(iprot); + struct.sIntString.add(_elem89); } } struct.setSIntStringIsSet(true); Modified: hive/branches/spark/serde/src/gen/thrift/gen-py/complex/ttypes.py URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/gen/thrift/gen-py/complex/ttypes.py?rev=1626482&r1=1626481&r2=1626482&view=diff ============================================================================== --- hive/branches/spark/serde/src/gen/thrift/gen-py/complex/ttypes.py (original) +++ hive/branches/spark/serde/src/gen/thrift/gen-py/complex/ttypes.py Sat Sep 20 17:34:39 2014 @@ -487,3 +487,84 @@ class Complex: def __ne__(self, other): return not (self == other) + +class SetIntString: + """ + Attributes: + - sIntString + - aString + """ + + thrift_spec = ( + None, # 0 + (1, TType.SET, 'sIntString', (TType.STRUCT,(IntString, IntString.thrift_spec)), None, ), # 1 + (2, TType.STRING, 'aString', None, None, ), # 2 + ) + + def __init__(self, sIntString=None, aString=None,): + self.sIntString = sIntString + self.aString = aString + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.SET: + self.sIntString = set() + (_etype76, _size73) = iprot.readSetBegin() + for _i77 in xrange(_size73): + _elem78 = IntString() + _elem78.read(iprot) + self.sIntString.add(_elem78) + iprot.readSetEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.aString = iprot.readString(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('SetIntString') + if self.sIntString is not None: + oprot.writeFieldBegin('sIntString', TType.SET, 1) + oprot.writeSetBegin(TType.STRUCT, len(self.sIntString)) + for iter79 in self.sIntString: + iter79.write(oprot) + oprot.writeSetEnd() + oprot.writeFieldEnd() + if self.aString is not None: + oprot.writeFieldBegin('aString', TType.STRING, 2) + oprot.writeString(self.aString) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) Modified: hive/branches/spark/serde/src/gen/thrift/gen-rb/complex_types.rb URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/gen/thrift/gen-rb/complex_types.rb?rev=1626482&r1=1626481&r2=1626482&view=diff ============================================================================== --- hive/branches/spark/serde/src/gen/thrift/gen-rb/complex_types.rb (original) +++ hive/branches/spark/serde/src/gen/thrift/gen-rb/complex_types.rb Sat Sep 20 17:34:39 2014 @@ -119,3 +119,21 @@ class Complex ::Thrift::Struct.generate_accessors self end +class SetIntString + include ::Thrift::Struct, ::Thrift::Struct_Union + SINTSTRING = 1 + ASTRING = 2 + + FIELDS = { + SINTSTRING => {:type => ::Thrift::Types::SET, :name => 'sIntString', :element => {:type => ::Thrift::Types::STRUCT, :class => ::IntString}}, + ASTRING => {:type => ::Thrift::Types::STRING, :name => 'aString'} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self +end + Modified: hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveChar.java URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveChar.java?rev=1626482&r1=1626481&r2=1626482&view=diff ============================================================================== --- hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveChar.java (original) +++ hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveChar.java Sat Sep 20 17:34:39 2014 @@ -55,14 +55,24 @@ public class LazyHiveChar extends @Override public void init(ByteArrayRef bytes, int start, int length) { - String byteData = null; - try { - byteData = Text.decode(bytes.getData(), start, length); - data.set(byteData, maxLength); + if (oi.isEscaped()) { + Text textData = data.getTextValue(); + // This is doing a lot of copying here, this could be improved by enforcing length + // at the same time as escaping rather than as separate steps. + LazyUtils.copyAndEscapeStringDataToText(bytes.getData(), start, length, + oi.getEscapeChar(),textData); + data.set(textData.toString(), maxLength); isNull = false; - } catch (CharacterCodingException e) { - isNull = true; - LOG.debug("Data not in the HiveChar data type range so converted to null.", e); + } else { + String byteData = null; + try { + byteData = Text.decode(bytes.getData(), start, length); + data.set(byteData, maxLength); + isNull = false; + } catch (CharacterCodingException e) { + isNull = true; + LOG.debug("Data not in the HiveChar data type range so converted to null.", e); + } } } Modified: hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveVarchar.java URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveVarchar.java?rev=1626482&r1=1626481&r2=1626482&view=diff ============================================================================== --- hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveVarchar.java (original) +++ hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveVarchar.java Sat Sep 20 17:34:39 2014 @@ -55,14 +55,24 @@ public class LazyHiveVarchar extends @Override public void init(ByteArrayRef bytes, int start, int length) { - String byteData = null; - try { - byteData = Text.decode(bytes.getData(), start, length); - data.set(byteData, maxLength); + if (oi.isEscaped()) { + Text textData = data.getTextValue(); + // This is doing a lot of copying here, this could be improved by enforcing length + // at the same time as escaping rather than as separate steps. + LazyUtils.copyAndEscapeStringDataToText(bytes.getData(), start, length, + oi.getEscapeChar(),textData); + data.set(textData.toString(), maxLength); isNull = false; - } catch (CharacterCodingException e) { - isNull = true; - LOG.debug("Data not in the HiveVarchar data type range so converted to null.", e); + } else { + try { + String byteData = null; + byteData = Text.decode(bytes.getData(), start, length); + data.set(byteData, maxLength); + isNull = false; + } catch (CharacterCodingException e) { + isNull = true; + LOG.debug("Data not in the HiveVarchar data type range so converted to null.", e); + } } } Modified: hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java?rev=1626482&r1=1626481&r2=1626482&view=diff ============================================================================== --- hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java (original) +++ hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java Sat Sep 20 17:34:39 2014 @@ -40,40 +40,7 @@ public class LazyString extends LazyPrim if (oi.isEscaped()) { byte escapeChar = oi.getEscapeChar(); byte[] inputBytes = bytes.getData(); - - // First calculate the length of the output string - int outputLength = 0; - for (int i = 0; i < length; i++) { - if (inputBytes[start + i] != escapeChar) { - outputLength++; - } else { - outputLength++; - i++; - } - } - - // Copy the data over, so that the internal state of Text will be set to - // the required outputLength. - data.set(bytes.getData(), start, outputLength); - - // We need to copy the data byte by byte only in case the - // "outputLength < length" (which means there is at least one escaped - // byte. - if (outputLength < length) { - int k = 0; - byte[] outputBytes = data.getBytes(); - for (int i = 0; i < length; i++) { - byte b = inputBytes[start + i]; - if (b != escapeChar || i == length - 1) { - outputBytes[k++] = b; - } else { - // get the next byte - i++; - outputBytes[k++] = inputBytes[start + i]; - } - } - assert (k == outputLength); - } + LazyUtils.copyAndEscapeStringDataToText(inputBytes, start, length, escapeChar, data); } else { // if the data is not escaped, simply copy the data. data.set(bytes.getData(), start, length); Modified: hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java?rev=1626482&r1=1626481&r2=1626482&view=diff ============================================================================== --- hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java (original) +++ hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java Sat Sep 20 17:34:39 2014 @@ -412,6 +412,44 @@ public final class LazyUtils { } } + public static void copyAndEscapeStringDataToText(byte[] inputBytes, int start, int length, + byte escapeChar, Text data) { + + // First calculate the length of the output string + int outputLength = 0; + for (int i = 0; i < length; i++) { + if (inputBytes[start + i] != escapeChar) { + outputLength++; + } else { + outputLength++; + i++; + } + } + + // Copy the data over, so that the internal state of Text will be set to + // the required outputLength. + data.set(inputBytes, start, outputLength); + + // We need to copy the data byte by byte only in case the + // "outputLength < length" (which means there is at least one escaped + // byte. + if (outputLength < length) { + int k = 0; + byte[] outputBytes = data.getBytes(); + for (int i = 0; i < length; i++) { + byte b = inputBytes[start + i]; + if (b != escapeChar || i == length - 1) { + outputBytes[k++] = b; + } else { + // get the next byte + i++; + outputBytes[k++] = inputBytes[start + i]; + } + } + assert (k == outputLength); + } + } + private LazyUtils() { // prevent instantiation } Modified: hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveCharObjectInspector.java URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveCharObjectInspector.java?rev=1626482&r1=1626481&r2=1626482&view=diff ============================================================================== --- hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveCharObjectInspector.java (original) +++ hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveCharObjectInspector.java Sat Sep 20 17:34:39 2014 @@ -29,12 +29,21 @@ public class LazyHiveCharObjectInspector extends AbstractPrimitiveLazyObjectInspector<HiveCharWritable> implements HiveCharObjectInspector { + private boolean escaped; + private byte escapeChar; + // no-arg ctor required for Kyro public LazyHiveCharObjectInspector() { } public LazyHiveCharObjectInspector(CharTypeInfo typeInfo) { + this(typeInfo, false, (byte)0); + } + + public LazyHiveCharObjectInspector(CharTypeInfo typeInfo, boolean escaped, byte escapeChar) { super(typeInfo); + this.escaped = escaped; + this.escapeChar = escapeChar; } @Override @@ -63,6 +72,14 @@ public class LazyHiveCharObjectInspector return ret; } + public boolean isEscaped() { + return escaped; + } + + public byte getEscapeChar() { + return escapeChar; + } + @Override public String toString() { return getTypeName(); Modified: hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveVarcharObjectInspector.java URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveVarcharObjectInspector.java?rev=1626482&r1=1626481&r2=1626482&view=diff ============================================================================== --- hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveVarcharObjectInspector.java (original) +++ hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveVarcharObjectInspector.java Sat Sep 20 17:34:39 2014 @@ -29,12 +29,21 @@ public class LazyHiveVarcharObjectInspec extends AbstractPrimitiveLazyObjectInspector<HiveVarcharWritable> implements HiveVarcharObjectInspector { + private boolean escaped; + private byte escapeChar; + // no-arg ctor required for Kyro public LazyHiveVarcharObjectInspector() { } public LazyHiveVarcharObjectInspector(VarcharTypeInfo typeInfo) { + this(typeInfo, false, (byte)0); + } + + public LazyHiveVarcharObjectInspector(VarcharTypeInfo typeInfo, boolean escaped, byte escapeChar) { super(typeInfo); + this.escaped = escaped; + this.escapeChar = escapeChar; } @Override @@ -63,6 +72,14 @@ public class LazyHiveVarcharObjectInspec return ret; } + public boolean isEscaped() { + return escaped; + } + + public byte getEscapeChar() { + return escapeChar; + } + @Override public String toString() { return getTypeName(); Modified: hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java?rev=1626482&r1=1626481&r2=1626482&view=diff ============================================================================== --- hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java (original) +++ hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java Sat Sep 20 17:34:39 2014 @@ -78,8 +78,10 @@ public final class LazyPrimitiveObjectIn // prevent instantiation } - private static HashMap<ArrayList<Object>, LazyStringObjectInspector> cachedLazyStringObjectInspector = - new HashMap<ArrayList<Object>, LazyStringObjectInspector>(); + // Lazy object inspectors for string/char/varchar will all be cached in the same map. + // Map key will be list of [typeInfo, isEscaped, escapeChar] + private static HashMap<ArrayList<Object>, AbstractPrimitiveLazyObjectInspector> cachedLazyStringTypeOIs = + new HashMap<ArrayList<Object>, AbstractPrimitiveLazyObjectInspector>(); private static Map<PrimitiveTypeInfo, AbstractPrimitiveLazyObjectInspector<?>> cachedPrimitiveLazyObjectInspectors = @@ -121,6 +123,10 @@ public final class LazyPrimitiveObjectIn switch(primitiveCategory) { case STRING: return getLazyStringObjectInspector(escaped, escapeChar); + case CHAR: + return getLazyHiveCharObjectInspector((CharTypeInfo)typeInfo, escaped, escapeChar); + case VARCHAR: + return getLazyHiveVarcharObjectInspector((VarcharTypeInfo)typeInfo, escaped, escapeChar); case BOOLEAN: return getLazyBooleanObjectInspector(extBoolean); default: @@ -157,13 +163,44 @@ public final class LazyPrimitiveObjectIn public static LazyStringObjectInspector getLazyStringObjectInspector(boolean escaped, byte escapeChar) { ArrayList<Object> signature = new ArrayList<Object>(); + signature.add(TypeInfoFactory.stringTypeInfo); signature.add(Boolean.valueOf(escaped)); signature.add(Byte.valueOf(escapeChar)); - LazyStringObjectInspector result = cachedLazyStringObjectInspector + LazyStringObjectInspector result = (LazyStringObjectInspector) cachedLazyStringTypeOIs .get(signature); if (result == null) { result = new LazyStringObjectInspector(escaped, escapeChar); - cachedLazyStringObjectInspector.put(signature, result); + cachedLazyStringTypeOIs.put(signature, result); + } + return result; + } + + public static LazyHiveCharObjectInspector getLazyHiveCharObjectInspector( + CharTypeInfo typeInfo, boolean escaped, byte escapeChar) { + ArrayList<Object> signature = new ArrayList<Object>(); + signature.add(typeInfo); + signature.add(Boolean.valueOf(escaped)); + signature.add(Byte.valueOf(escapeChar)); + LazyHiveCharObjectInspector result = (LazyHiveCharObjectInspector) cachedLazyStringTypeOIs + .get(signature); + if (result == null) { + result = new LazyHiveCharObjectInspector(typeInfo, escaped, escapeChar); + cachedLazyStringTypeOIs.put(signature, result); + } + return result; + } + + public static LazyHiveVarcharObjectInspector getLazyHiveVarcharObjectInspector( + VarcharTypeInfo typeInfo, boolean escaped, byte escapeChar) { + ArrayList<Object> signature = new ArrayList<Object>(); + signature.add(typeInfo); + signature.add(Boolean.valueOf(escaped)); + signature.add(Byte.valueOf(escapeChar)); + LazyHiveVarcharObjectInspector result = (LazyHiveVarcharObjectInspector) cachedLazyStringTypeOIs + .get(signature); + if (result == null) { + result = new LazyHiveVarcharObjectInspector(typeInfo, escaped, escapeChar); + cachedLazyStringTypeOIs.put(signature, result); } return result; } Modified: hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveCharObjectInspector.java URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveCharObjectInspector.java?rev=1626482&r1=1626481&r2=1626482&view=diff ============================================================================== --- hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveCharObjectInspector.java (original) +++ hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveCharObjectInspector.java Sat Sep 20 17:34:39 2014 @@ -91,6 +91,9 @@ public class WritableHiveCharObjectInspe @Override public Object set(Object o, HiveChar value) { + if (value == null) { + return null; + } HiveCharWritable writable = (HiveCharWritable) o; writable.set(value, getMaxLength()); return o; @@ -98,6 +101,9 @@ public class WritableHiveCharObjectInspe @Override public Object set(Object o, String value) { + if (value == null) { + return null; + } HiveCharWritable writable = (HiveCharWritable) o; writable.set(value, getMaxLength()); return o; Modified: hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java?rev=1626482&r1=1626481&r2=1626482&view=diff ============================================================================== --- hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java (original) +++ hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java Sat Sep 20 17:34:39 2014 @@ -96,6 +96,9 @@ implements SettableHiveVarcharObjectInsp @Override public Object set(Object o, HiveVarchar value) { + if (value == null) { + return null; + } HiveVarcharWritable writable = (HiveVarcharWritable)o; writable.set(value, getMaxLength()); return o; @@ -103,6 +106,9 @@ implements SettableHiveVarcharObjectInsp @Override public Object set(Object o, String value) { + if (value == null) { + return null; + } HiveVarcharWritable writable = (HiveVarcharWritable)o; writable.set(value, getMaxLength()); return o; Modified: hive/branches/spark/service/src/java/org/apache/hive/service/cli/CLIService.java URL: http://svn.apache.org/viewvc/hive/branches/spark/service/src/java/org/apache/hive/service/cli/CLIService.java?rev=1626482&r1=1626481&r2=1626482&view=diff ============================================================================== --- hive/branches/spark/service/src/java/org/apache/hive/service/cli/CLIService.java (original) +++ hive/branches/spark/service/src/java/org/apache/hive/service/cli/CLIService.java Sat Sep 20 17:34:39 2014 @@ -66,7 +66,7 @@ public class CLIService extends Composit private UserGroupInformation httpUGI; public CLIService() { - super("CLIService"); + super(CLIService.class.getSimpleName()); } @Override @@ -201,8 +201,7 @@ public class CLIService extends Composit * @see org.apache.hive.service.cli.ICLIService#closeSession(org.apache.hive.service.cli.SessionHandle) */ @Override - public void closeSession(SessionHandle sessionHandle) - throws HiveSQLException { + public void closeSession(SessionHandle sessionHandle) throws HiveSQLException { sessionManager.closeSession(sessionHandle); LOG.debug(sessionHandle + ": closeSession()"); } @@ -470,4 +469,8 @@ public class CLIService extends Composit sessionManager.getSession(sessionHandle).renewDelegationToken(authFactory, tokenStr); LOG.info(sessionHandle + ": renewDelegationToken()"); } + + public SessionManager getSessionManager() { + return sessionManager; + } } Modified: hive/branches/spark/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java URL: http://svn.apache.org/viewvc/hive/branches/spark/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java?rev=1626482&r1=1626481&r2=1626482&view=diff ============================================================================== --- hive/branches/spark/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java (original) +++ hive/branches/spark/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java Sat Sep 20 17:34:39 2014 @@ -47,7 +47,7 @@ public class OperationManager extends Ab new HashMap<OperationHandle, Operation>(); public OperationManager() { - super("OperationManager"); + super(OperationManager.class.getSimpleName()); } @Override Modified: hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java URL: http://svn.apache.org/viewvc/hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java?rev=1626482&r1=1626481&r2=1626482&view=diff ============================================================================== --- hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java (original) +++ hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java Sat Sep 20 17:34:39 2014 @@ -166,8 +166,8 @@ public class HiveSessionImpl implements IHiveFileProcessor processor = new GlobalHivercFileProcessor(); try { - if (hiveConf.getVar(ConfVars.HIVE_GLOBAL_INIT_FILE_LOCATION) != null) { - String hiverc = hiveConf.getVar(ConfVars.HIVE_GLOBAL_INIT_FILE_LOCATION) + if (hiveConf.getVar(ConfVars.HIVE_SERVER2_GLOBAL_INIT_FILE_LOCATION) != null) { + String hiverc = hiveConf.getVar(ConfVars.HIVE_SERVER2_GLOBAL_INIT_FILE_LOCATION) + File.separator + SessionManager.HIVERCFILE; if (new File(hiverc).exists()) { LOG.info("Running global init file: " + hiverc); Modified: hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/SessionManager.java URL: http://svn.apache.org/viewvc/hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/SessionManager.java?rev=1626482&r1=1626481&r2=1626482&view=diff ============================================================================== --- hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/SessionManager.java (original) +++ hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/SessionManager.java Sat Sep 20 17:34:39 2014 @@ -67,7 +67,7 @@ public class SessionManager extends Comp private volatile boolean shutdown; public SessionManager() { - super("SessionManager"); + super(SessionManager.class.getSimpleName()); } @Override @@ -356,5 +356,9 @@ public class SessionManager extends Comp return backgroundOperationPool.submit(r); } + public int getOpenSessionCount() { + return handleToSession.size(); + } + } Modified: hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java URL: http://svn.apache.org/viewvc/hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java?rev=1626482&r1=1626481&r2=1626482&view=diff ============================================================================== --- hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java (original) +++ hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java Sat Sep 20 17:34:39 2014 @@ -18,7 +18,6 @@ package org.apache.hive.service.cli.thrift; -import java.net.InetSocketAddress; import java.util.concurrent.ExecutorService; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; @@ -40,72 +39,54 @@ import org.apache.thrift.transport.TTran public class ThriftBinaryCLIService extends ThriftCLIService { public ThriftBinaryCLIService(CLIService cliService) { - super(cliService, "ThriftBinaryCLIService"); + super(cliService, ThriftBinaryCLIService.class.getSimpleName()); } @Override public void run() { try { - hiveAuthFactory = new HiveAuthFactory(hiveConf); - TTransportFactory transportFactory = hiveAuthFactory.getAuthTransFactory(); - TProcessorFactory processorFactory = hiveAuthFactory.getAuthProcFactory(this); - - String portString = System.getenv("HIVE_SERVER2_THRIFT_PORT"); - if (portString != null) { - portNum = Integer.valueOf(portString); - } else { - portNum = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT); - } - - String hiveHost = System.getenv("HIVE_SERVER2_THRIFT_BIND_HOST"); - if (hiveHost == null) { - hiveHost = hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST); - } - - if (hiveHost != null && !hiveHost.isEmpty()) { - serverAddress = new InetSocketAddress(hiveHost, portNum); - } else { - serverAddress = new InetSocketAddress(portNum); - } - - minWorkerThreads = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_MIN_WORKER_THREADS); - maxWorkerThreads = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_MAX_WORKER_THREADS); - workerKeepAliveTime = hiveConf.getTimeVar( - ConfVars.HIVE_SERVER2_THRIFT_WORKER_KEEPALIVE_TIME, TimeUnit.SECONDS); + // Server thread pool String threadPoolName = "HiveServer2-Handler-Pool"; ExecutorService executorService = new ThreadPoolExecutor(minWorkerThreads, maxWorkerThreads, workerKeepAliveTime, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(), new ThreadFactoryWithGarbageCleanup(threadPoolName)); + // Thrift configs + hiveAuthFactory = new HiveAuthFactory(hiveConf); + TTransportFactory transportFactory = hiveAuthFactory.getAuthTransFactory(); + TProcessorFactory processorFactory = hiveAuthFactory.getAuthProcFactory(this); TServerSocket serverSocket = null; if (!hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_USE_SSL)) { serverSocket = HiveAuthFactory.getServerSocket(hiveHost, portNum); } else { String keyStorePath = hiveConf.getVar(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH).trim(); if (keyStorePath.isEmpty()) { - throw new IllegalArgumentException(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname + - " Not configured for SSL connection"); + throw new IllegalArgumentException(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname + + " Not configured for SSL connection"); } String keyStorePassword = ShimLoader.getHadoopShims().getPassword(hiveConf, HiveConf.ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname); - serverSocket = HiveAuthFactory.getServerSSLSocket(hiveHost, portNum, - keyStorePath, keyStorePassword); + serverSocket = HiveAuthFactory.getServerSSLSocket(hiveHost, portNum, keyStorePath, + keyStorePassword); } + + // Server args TThreadPoolServer.Args sargs = new TThreadPoolServer.Args(serverSocket) - .processorFactory(processorFactory) - .transportFactory(transportFactory) - .protocolFactory(new TBinaryProtocol.Factory()) - .executorService(executorService); + .processorFactory(processorFactory).transportFactory(transportFactory) + .protocolFactory(new TBinaryProtocol.Factory()).executorService(executorService); + // TCP Server server = new TThreadPoolServer(sargs); - - LOG.info("ThriftBinaryCLIService listening on " + serverAddress); - server.serve(); - + String msg = "Started " + ThriftBinaryCLIService.class.getSimpleName() + " on port " + + portNum + " with " + minWorkerThreads + "..." + maxWorkerThreads + " worker threads"; + LOG.info(msg); } catch (Throwable t) { - LOG.error("Error: ", t); + LOG.fatal( + "Error starting HiveServer2: could not start " + + ThriftBinaryCLIService.class.getSimpleName(), t); + System.exit(-1); } - } + } Modified: hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java URL: http://svn.apache.org/viewvc/hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java?rev=1626482&r1=1626481&r2=1626482&view=diff ============================================================================== --- hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java (original) +++ hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java Sat Sep 20 17:34:39 2014 @@ -22,6 +22,7 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.TimeUnit; import javax.security.auth.login.LoginException; @@ -34,6 +35,7 @@ import org.apache.hive.service.auth.Hive import org.apache.hive.service.auth.TSetIpAddressProcessor; import org.apache.hive.service.cli.*; import org.apache.hive.service.cli.session.SessionManager; +import org.apache.hive.service.server.HiveServer2; import org.apache.thrift.TException; import org.apache.thrift.server.TServer; @@ -48,9 +50,11 @@ public abstract class ThriftCLIService e protected CLIService cliService; private static final TStatus OK_STATUS = new TStatus(TStatusCode.SUCCESS_STATUS); private static final TStatus ERROR_STATUS = new TStatus(TStatusCode.ERROR_STATUS); + protected static HiveAuthFactory hiveAuthFactory; protected int portNum; protected InetSocketAddress serverAddress; + protected String hiveHost; protected TServer server; protected org.eclipse.jetty.server.Server httpServer; @@ -62,8 +66,7 @@ public abstract class ThriftCLIService e protected int minWorkerThreads; protected int maxWorkerThreads; protected long workerKeepAliveTime; - - protected static HiveAuthFactory hiveAuthFactory; + private HiveServer2 hiveServer2; public ThriftCLIService(CLIService cliService, String serviceName) { super(serviceName); @@ -73,6 +76,43 @@ public abstract class ThriftCLIService e @Override public synchronized void init(HiveConf hiveConf) { this.hiveConf = hiveConf; + + // Initialize common server configs needed in both binary & http modes + String portString; + hiveHost = System.getenv("HIVE_SERVER2_THRIFT_BIND_HOST"); + if (hiveHost == null) { + hiveHost = hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST); + } + // HTTP mode + if (HiveServer2.isHTTPTransportMode(hiveConf)) { + workerKeepAliveTime = + hiveConf.getTimeVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_WORKER_KEEPALIVE_TIME, + TimeUnit.SECONDS); + portString = System.getenv("HIVE_SERVER2_THRIFT_HTTP_PORT"); + if (portString != null) { + portNum = Integer.valueOf(portString); + } else { + portNum = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT); + } + } + // Binary mode + else { + workerKeepAliveTime = + hiveConf.getTimeVar(ConfVars.HIVE_SERVER2_THRIFT_WORKER_KEEPALIVE_TIME, TimeUnit.SECONDS); + portString = System.getenv("HIVE_SERVER2_THRIFT_PORT"); + if (portString != null) { + portNum = Integer.valueOf(portString); + } else { + portNum = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT); + } + } + if (hiveHost != null && !hiveHost.isEmpty()) { + serverAddress = new InetSocketAddress(hiveHost, portNum); + } else { + serverAddress = new InetSocketAddress(portNum); + } + minWorkerThreads = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_MIN_WORKER_THREADS); + maxWorkerThreads = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_MAX_WORKER_THREADS); super.init(hiveConf); } @@ -105,6 +145,14 @@ public abstract class ThriftCLIService e super.stop(); } + public int getPortNumber() { + return portNum; + } + + public InetSocketAddress getServerAddress() { + return serverAddress; + } + @Override public TGetDelegationTokenResp GetDelegationToken(TGetDelegationTokenReq req) throws TException { @@ -308,6 +356,24 @@ public abstract class ThriftCLIService e } catch (Exception e) { LOG.warn("Error closing session: ", e); resp.setStatus(HiveSQLException.toTStatus(e)); + } finally { + if (!(isEmbedded) && (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY)) + && (!hiveServer2.isRegisteredWithZooKeeper())) { + // Asynchronously shutdown this instance of HiveServer2, + // if there are no active client sessions + if (cliService.getSessionManager().getOpenSessionCount() == 0) { + LOG.info("This instance of HiveServer2 has been removed from the list of server " + + "instances available for dynamic service discovery. " + + "The last client session has ended - will shutdown now."); + Thread shutdownThread = new Thread() { + @Override + public void run() { + hiveServer2.stop(); + } + }; + shutdownThread.start(); + } + } } return resp; } @@ -591,5 +657,9 @@ public abstract class ThriftCLIService e .equalsIgnoreCase(HiveAuthFactory.AuthTypes.KERBEROS.toString()); } + public void setHiveServer2(HiveServer2 hiveServer2) { + this.hiveServer2 = hiveServer2; + } + } Modified: hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java URL: http://svn.apache.org/viewvc/hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java?rev=1626482&r1=1626481&r2=1626482&view=diff ============================================================================== --- hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java (original) +++ hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java Sat Sep 20 17:34:39 2014 @@ -48,100 +48,94 @@ import org.eclipse.jetty.util.thread.Exe public class ThriftHttpCLIService extends ThriftCLIService { public ThriftHttpCLIService(CLIService cliService) { - super(cliService, "ThriftHttpCLIService"); + super(cliService, ThriftHttpCLIService.class.getSimpleName()); } + /** + * Configure Jetty to serve http requests. Example of a client connection URL: + * http://localhost:10000/servlets/thrifths2/ A gateway may cause actual target URL to differ, + * e.g. http://gateway:port/hive2/servlets/thrifths2/ + */ @Override public void run() { try { - // Configure Jetty to serve http requests - // Example of a client connection URL: http://localhost:10000/servlets/thrifths2/ - // a gateway may cause actual target URL to differ, e.g. http://gateway:port/hive2/servlets/thrifths2/ - + // Verify config validity verifyHttpConfiguration(hiveConf); - String portString = System.getenv("HIVE_SERVER2_THRIFT_HTTP_PORT"); - if (portString != null) { - portNum = Integer.valueOf(portString); - } else { - portNum = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT); - } - - minWorkerThreads = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_MIN_WORKER_THREADS); - maxWorkerThreads = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_MAX_WORKER_THREADS); - workerKeepAliveTime = hiveConf.getTimeVar( - ConfVars.HIVE_SERVER2_THRIFT_HTTP_WORKER_KEEPALIVE_TIME, TimeUnit.SECONDS); - - String httpPath = getHttpPath(hiveConf.getVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH)); - + // HTTP Server httpServer = new org.eclipse.jetty.server.Server(); + + // Server thread pool String threadPoolName = "HiveServer2-HttpHandler-Pool"; ExecutorService executorService = new ThreadPoolExecutor(minWorkerThreads, maxWorkerThreads, workerKeepAliveTime, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), new ThreadFactoryWithGarbageCleanup(threadPoolName)); - ExecutorThreadPool threadPool = new ExecutorThreadPool(executorService); httpServer.setThreadPool(threadPool); - SelectChannelConnector connector = new SelectChannelConnector();; + // Connector configs + SelectChannelConnector connector = new SelectChannelConnector(); boolean useSsl = hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_USE_SSL); String schemeName = useSsl ? "https" : "http"; - String authType = hiveConf.getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION); - // Set during the init phase of HiveServer2 if auth mode is kerberos - // UGI for the hive/_HOST (kerberos) principal - UserGroupInformation serviceUGI = cliService.getServiceUGI(); - // UGI for the http/_HOST (SPNego) principal - UserGroupInformation httpUGI = cliService.getHttpUGI(); - + // Change connector if SSL is used if (useSsl) { String keyStorePath = hiveConf.getVar(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH).trim(); String keyStorePassword = ShimLoader.getHadoopShims().getPassword(hiveConf, HiveConf.ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname); if (keyStorePath.isEmpty()) { - throw new IllegalArgumentException(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname + - " Not configured for SSL connection"); + throw new IllegalArgumentException(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname + + " Not configured for SSL connection"); } SslContextFactory sslContextFactory = new SslContextFactory(); sslContextFactory.setKeyStorePath(keyStorePath); sslContextFactory.setKeyStorePassword(keyStorePassword); connector = new SslSelectChannelConnector(sslContextFactory); } - connector.setPort(portNum); // Linux:yes, Windows:no connector.setReuseAddress(!Shell.WINDOWS); - - int maxIdleTime = (int) hiveConf.getTimeVar( - ConfVars.HIVE_SERVER2_THRIFT_HTTP_MAX_IDLE_TIME, TimeUnit.MILLISECONDS); + int maxIdleTime = (int) hiveConf.getTimeVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_MAX_IDLE_TIME, + TimeUnit.MILLISECONDS); connector.setMaxIdleTime(maxIdleTime); - + httpServer.addConnector(connector); + // Thrift configs hiveAuthFactory = new HiveAuthFactory(hiveConf); TProcessorFactory processorFactory = hiveAuthFactory.getAuthProcFactory(this); TProcessor processor = processorFactory.getProcessor(null); - TProtocolFactory protocolFactory = new TBinaryProtocol.Factory(); + // Set during the init phase of HiveServer2 if auth mode is kerberos + // UGI for the hive/_HOST (kerberos) principal + UserGroupInformation serviceUGI = cliService.getServiceUGI(); + // UGI for the http/_HOST (SPNego) principal + UserGroupInformation httpUGI = cliService.getHttpUGI(); + String authType = hiveConf.getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION); + TServlet thriftHttpServlet = new ThriftHttpServlet(processor, protocolFactory, authType, + serviceUGI, httpUGI); - TServlet thriftHttpServlet = new ThriftHttpServlet(processor, protocolFactory, - authType, serviceUGI, httpUGI); - + // Context handler final ServletContextHandler context = new ServletContextHandler( ServletContextHandler.SESSIONS); context.setContextPath("/"); - + String httpPath = getHttpPath(hiveConf + .getVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH)); httpServer.setHandler(context); context.addServlet(new ServletHolder(thriftHttpServlet), httpPath); // TODO: check defaults: maxTimeout, keepalive, maxBodySize, bodyRecieveDuration, etc. + // Finally, start the server httpServer.start(); - String msg = "Started ThriftHttpCLIService in " + schemeName + " mode on port " + portNum + - " path=" + httpPath + - " with " + minWorkerThreads + ".." + maxWorkerThreads + " worker threads"; + String msg = "Started " + ThriftHttpCLIService.class.getSimpleName() + " in " + schemeName + + " mode on port " + portNum + " path=" + httpPath + " with " + minWorkerThreads + "..." + + maxWorkerThreads + " worker threads"; LOG.info(msg); httpServer.join(); } catch (Throwable t) { - LOG.error("Error: ", t); + LOG.fatal( + "Error starting HiveServer2: could not start " + + ThriftHttpCLIService.class.getSimpleName(), t); + System.exit(-1); } } @@ -191,7 +185,8 @@ public class ThriftHttpCLIService extend // NONE in case of thrift mode uses SASL LOG.warn(ConfVars.HIVE_SERVER2_AUTHENTICATION + " setting to " + authType + ". SASL is not supported with http transport mode," + - " so using equivalent of " + AuthTypes.NOSASL); + " so using equivalent of " + + AuthTypes.NOSASL); } }
