Modified: hive/branches/spark/ql/src/test/results/compiler/plan/cast1.q.xml URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/compiler/plan/cast1.q.xml?rev=1613740&r1=1613739&r2=1613740&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/compiler/plan/cast1.q.xml (original) +++ hive/branches/spark/ql/src/test/results/compiler/plan/cast1.q.xml Sat Jul 26 23:45:46 2014 @@ -378,7 +378,7 @@ <object class="java.util.HashMap"> <void method="put"> <string>_col6</string> - <object id="ExprNodeGenericFuncDesc0" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -410,7 +410,7 @@ </void> <void method="put"> <string>_col5</string> - <object id="ExprNodeGenericFuncDesc1" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -442,7 +442,7 @@ </void> <void method="put"> <string>_col4</string> - <object id="ExprNodeGenericFuncDesc2" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -502,7 +502,7 @@ </void> <void method="put"> <string>_col3</string> - <object id="ExprNodeGenericFuncDesc3" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -541,7 +541,7 @@ </void> <void method="put"> <string>_col2</string> - <object id="ExprNodeGenericFuncDesc4" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -580,7 +580,7 @@ </void> <void method="put"> <string>_col1</string> - <object id="ExprNodeGenericFuncDesc5" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -619,7 +619,7 @@ </void> <void method="put"> <string>_col0</string> - <object id="ExprNodeGenericFuncDesc6" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -663,25 +663,74 @@ <void property="colList"> <object class="java.util.ArrayList"> <void method="add"> - <object idref="ExprNodeGenericFuncDesc6"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo0"/> + </void> + <void property="value"> + <int>5</int> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc5"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo1"/> + </void> + <void property="value"> + <double>5.0</double> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc4"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo1"/> + </void> + <void property="value"> + <double>5.0</double> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc3"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo1"/> + </void> + <void property="value"> + <double>5.0</double> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc2"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo0"/> + </void> + <void property="value"> + <int>5</int> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc1"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo2"/> + </void> + <void property="value"> + <boolean>true</boolean> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc0"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo0"/> + </void> + <void property="value"> + <int>1</int> + </void> + </object> </void> </object> </void> @@ -899,6 +948,9 @@ <void property="signature"> <void method="add"> <object id="ColumnInfo0" class="org.apache.hadoop.hive.ql.exec.ColumnInfo"> + <void property="alias"> + <string>key</string> + </void> <void property="internalName"> <string>key</string> </void>
Modified: hive/branches/spark/ql/src/test/results/compiler/plan/input6.q.xml URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/compiler/plan/input6.q.xml?rev=1613740&r1=1613739&r2=1613740&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/compiler/plan/input6.q.xml (original) +++ hive/branches/spark/ql/src/test/results/compiler/plan/input6.q.xml Sat Jul 26 23:45:46 2014 @@ -797,17 +797,7 @@ </void> <void method="put"> <string>_col0</string> - <object id="ExprNodeColumnDesc1" class="org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc"> - <void property="column"> - <string>key</string> - </void> - <void property="tabAlias"> - <string>src1</string> - </void> - <void property="typeInfo"> - <object idref="PrimitiveTypeInfo0"/> - </void> - </object> + <object id="ExprNodeNullDesc0" class="org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc"/> </void> </object> </void> @@ -816,7 +806,7 @@ <void property="colList"> <object class="java.util.ArrayList"> <void method="add"> - <object idref="ExprNodeColumnDesc1"/> + <object idref="ExprNodeNullDesc0"/> </void> <void method="add"> <object idref="ExprNodeColumnDesc0"/> @@ -848,6 +838,9 @@ <void property="signature"> <void method="add"> <object class="org.apache.hadoop.hive.ql.exec.ColumnInfo"> + <void property="alias"> + <string>key</string> + </void> <void property="internalName"> <string>_col0</string> </void> @@ -932,6 +925,9 @@ <void property="signature"> <void method="add"> <object id="ColumnInfo0" class="org.apache.hadoop.hive.ql.exec.ColumnInfo"> + <void property="alias"> + <string>key</string> + </void> <void property="internalName"> <string>key</string> </void> Modified: hive/branches/spark/ql/src/test/results/compiler/plan/join8.q.xml URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/compiler/plan/join8.q.xml?rev=1613740&r1=1613739&r2=1613740&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/compiler/plan/join8.q.xml (original) +++ hive/branches/spark/ql/src/test/results/compiler/plan/join8.q.xml Sat Jul 26 23:45:46 2014 @@ -1870,9 +1870,15 @@ </void> <void method="add"> <object id="ColumnInfo6" class="org.apache.hadoop.hive.ql.exec.ColumnInfo"> + <void property="alias"> + <string>c3</string> + </void> <void property="internalName"> <string>_col2</string> </void> + <void property="tabAlias"> + <string>b</string> + </void> <void property="type"> <object idref="PrimitiveTypeInfo0"/> </void> Modified: hive/branches/spark/ql/src/test/results/compiler/plan/udf1.q.xml URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/compiler/plan/udf1.q.xml?rev=1613740&r1=1613739&r2=1613740&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/compiler/plan/udf1.q.xml (original) +++ hive/branches/spark/ql/src/test/results/compiler/plan/udf1.q.xml Sat Jul 26 23:45:46 2014 @@ -534,7 +534,7 @@ <object class="java.util.HashMap"> <void method="put"> <string>_col8</string> - <object id="ExprNodeGenericFuncDesc0" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -579,7 +579,7 @@ </void> <void method="put"> <string>_col7</string> - <object id="ExprNodeGenericFuncDesc1" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -624,7 +624,7 @@ </void> <void method="put"> <string>_col6</string> - <object id="ExprNodeGenericFuncDesc2" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -669,7 +669,7 @@ </void> <void method="put"> <string>_col5</string> - <object id="ExprNodeGenericFuncDesc3" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -714,7 +714,7 @@ </void> <void method="put"> <string>_col4</string> - <object id="ExprNodeGenericFuncDesc4" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -759,7 +759,7 @@ </void> <void method="put"> <string>_col3</string> - <object id="ExprNodeGenericFuncDesc5" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -804,7 +804,7 @@ </void> <void method="put"> <string>_col2</string> - <object id="ExprNodeGenericFuncDesc6" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -849,7 +849,7 @@ </void> <void method="put"> <string>_col1</string> - <object id="ExprNodeGenericFuncDesc7" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -894,7 +894,7 @@ </void> <void method="put"> <string>_col9</string> - <object id="ExprNodeGenericFuncDesc8" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -939,7 +939,7 @@ </void> <void method="put"> <string>_col13</string> - <object id="ExprNodeGenericFuncDesc9" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -991,7 +991,7 @@ </void> <void method="put"> <string>_col12</string> - <object id="ExprNodeGenericFuncDesc10" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -1036,7 +1036,7 @@ </void> <void method="put"> <string>_col11</string> - <object id="ExprNodeGenericFuncDesc11" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -1081,7 +1081,7 @@ </void> <void method="put"> <string>_col10</string> - <object id="ExprNodeGenericFuncDesc12" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -1126,7 +1126,7 @@ </void> <void method="put"> <string>_col16</string> - <object id="ExprNodeGenericFuncDesc13" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -1178,7 +1178,7 @@ </void> <void method="put"> <string>_col15</string> - <object id="ExprNodeGenericFuncDesc14" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -1230,7 +1230,7 @@ </void> <void method="put"> <string>_col14</string> - <object id="ExprNodeGenericFuncDesc15" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -1282,7 +1282,7 @@ </void> <void method="put"> <string>_col0</string> - <object id="ExprNodeGenericFuncDesc16" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -1332,55 +1332,174 @@ <void property="colList"> <object class="java.util.ArrayList"> <void method="add"> - <object idref="ExprNodeGenericFuncDesc16"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo0"/> + </void> + <void property="value"> + <boolean>true</boolean> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc7"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo0"/> + </void> + <void property="value"> + <boolean>false</boolean> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc6"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo0"/> + </void> + <void property="value"> + <boolean>true</boolean> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc5"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo0"/> + </void> + <void property="value"> + <boolean>true</boolean> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc4"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo0"/> + </void> + <void property="value"> + <boolean>true</boolean> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc3"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo0"/> + </void> + <void property="value"> + <boolean>false</boolean> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc2"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo0"/> + </void> + <void property="value"> + <boolean>false</boolean> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc1"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo0"/> + </void> + <void property="value"> + <boolean>false</boolean> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc0"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo0"/> + </void> + <void property="value"> + <boolean>true</boolean> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc8"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo0"/> + </void> + <void property="value"> + <boolean>true</boolean> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc12"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo0"/> + </void> + <void property="value"> + <boolean>false</boolean> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc11"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo0"/> + </void> + <void property="value"> + <boolean>true</boolean> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc10"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo0"/> + </void> + <void property="value"> + <boolean>true</boolean> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc9"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo1"/> + </void> + <void property="value"> + <string>acc</string> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc15"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo1"/> + </void> + <void property="value"> + <string>abc</string> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc14"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo1"/> + </void> + <void property="value"> + <string>abb</string> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc13"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo1"/> + </void> + <void property="value"> + <string>hive</string> + </void> + </object> </void> </object> </void> @@ -1788,6 +1907,9 @@ <void property="signature"> <void method="add"> <object id="ColumnInfo0" class="org.apache.hadoop.hive.ql.exec.ColumnInfo"> + <void property="alias"> + <string>key</string> + </void> <void property="internalName"> <string>key</string> </void> Modified: hive/branches/spark/ql/src/test/results/compiler/plan/udf4.q.xml URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/compiler/plan/udf4.q.xml?rev=1613740&r1=1613739&r2=1613740&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/compiler/plan/udf4.q.xml (original) +++ hive/branches/spark/ql/src/test/results/compiler/plan/udf4.q.xml Sat Jul 26 23:45:46 2014 @@ -547,7 +547,7 @@ <object class="java.util.HashMap"> <void method="put"> <string>_col8</string> - <object id="ExprNodeGenericFuncDesc0" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -579,7 +579,7 @@ </void> <void method="put"> <string>_col7</string> - <object id="ExprNodeGenericFuncDesc1" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -625,7 +625,7 @@ </void> <void method="put"> <string>_col6</string> - <object id="ExprNodeGenericFuncDesc2" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -657,7 +657,7 @@ </void> <void method="put"> <string>_col5</string> - <object id="ExprNodeGenericFuncDesc3" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -696,7 +696,7 @@ </void> <void method="put"> <string>_col4</string> - <object id="ExprNodeGenericFuncDesc4" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -721,7 +721,7 @@ </void> <void method="put"> <string>_col3</string> - <object id="ExprNodeGenericFuncDesc5" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -746,7 +746,7 @@ </void> <void method="put"> <string>_col2</string> - <object id="ExprNodeGenericFuncDesc6" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -785,7 +785,7 @@ </void> <void method="put"> <string>_col1</string> - <object id="ExprNodeGenericFuncDesc7" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -810,7 +810,7 @@ </void> <void method="put"> <string>_col9</string> - <object id="ExprNodeGenericFuncDesc8" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -835,7 +835,7 @@ </void> <void method="put"> <string>_col13</string> - <object id="ExprNodeGenericFuncDesc9" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object id="ExprNodeGenericFuncDesc0" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -867,7 +867,7 @@ </void> <void method="put"> <string>_col12</string> - <object id="ExprNodeGenericFuncDesc10" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -892,7 +892,7 @@ </void> <void method="put"> <string>_col11</string> - <object id="ExprNodeGenericFuncDesc11" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -931,7 +931,7 @@ </void> <void method="put"> <string>_col10</string> - <object id="ExprNodeGenericFuncDesc12" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -956,7 +956,7 @@ </void> <void method="put"> <string>_col17</string> - <object id="ExprNodeGenericFuncDesc13" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -1009,7 +1009,7 @@ </void> <void method="put"> <string>_col16</string> - <object id="ExprNodeGenericFuncDesc14" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -1048,7 +1048,7 @@ </void> <void method="put"> <string>_col15</string> - <object id="ExprNodeGenericFuncDesc15" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -1084,7 +1084,7 @@ </void> <void method="put"> <string>_col0</string> - <object id="ExprNodeGenericFuncDesc16" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -1109,7 +1109,7 @@ </void> <void method="put"> <string>_col18</string> - <object id="ExprNodeGenericFuncDesc17" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -1149,61 +1149,173 @@ <void property="colList"> <object class="java.util.ArrayList"> <void method="add"> - <object idref="ExprNodeGenericFuncDesc16"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo0"/> + </void> + <void property="value"> + <double>1.0</double> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc7"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo0"/> + </void> + <void property="value"> + <double>2.0</double> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc6"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo0"/> + </void> + <void property="value"> + <double>-2.0</double> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc5"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo1"/> + </void> + <void property="value"> + <long>1</long> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc4"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo1"/> + </void> + <void property="value"> + <long>1</long> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc3"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo1"/> + </void> + <void property="value"> + <long>-2</long> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc2"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo0"/> + </void> + <void property="value"> + <double>1.0</double> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc1"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc"/> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc0"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo0"/> + </void> + <void property="value"> + <double>0.0</double> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc8"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo1"/> + </void> + <void property="value"> + <long>1</long> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc12"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo1"/> + </void> + <void property="value"> + <long>2</long> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc11"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo1"/> + </void> + <void property="value"> + <long>-1</long> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc10"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo1"/> + </void> + <void property="value"> + <long>1</long> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc9"/> + <object idref="ExprNodeGenericFuncDesc0"/> </void> <void method="add"> <object idref="ExprNodeConstantDesc0"/> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc15"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo2"/> + </void> + <void property="value"> + <int>-3</int> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc14"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo2"/> + </void> + <void property="value"> + <int>3</int> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc13"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo2"/> + </void> + <void property="value"> + <int>-1</int> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc17"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo2"/> + </void> + <void property="value"> + <int>-2</int> + </void> + </object> </void> </object> </void> Modified: hive/branches/spark/ql/src/test/results/compiler/plan/udf6.q.xml URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/compiler/plan/udf6.q.xml?rev=1613740&r1=1613739&r2=1613740&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/compiler/plan/udf6.q.xml (original) +++ hive/branches/spark/ql/src/test/results/compiler/plan/udf6.q.xml Sat Jul 26 23:45:46 2014 @@ -291,7 +291,7 @@ <object class="java.util.HashMap"> <void method="put"> <string>_col1</string> - <object id="ExprNodeGenericFuncDesc0" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -340,7 +340,7 @@ </void> <void method="put"> <string>_col0</string> - <object id="ExprNodeGenericFuncDesc1" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -380,10 +380,24 @@ <void property="colList"> <object class="java.util.ArrayList"> <void method="add"> - <object idref="ExprNodeGenericFuncDesc1"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo0"/> + </void> + <void property="value"> + <string>ab</string> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc0"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo1"/> + </void> + <void property="value"> + <int>1</int> + </void> + </object> </void> </object> </void> Modified: hive/branches/spark/ql/src/test/results/compiler/plan/udf_case.q.xml URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/compiler/plan/udf_case.q.xml?rev=1613740&r1=1613739&r2=1613740&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/compiler/plan/udf_case.q.xml (original) +++ hive/branches/spark/ql/src/test/results/compiler/plan/udf_case.q.xml Sat Jul 26 23:45:46 2014 @@ -346,7 +346,7 @@ <object class="java.util.HashMap"> <void method="put"> <string>_col1</string> - <object id="ExprNodeGenericFuncDesc0" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -411,7 +411,7 @@ </void> <void method="put"> <string>_col0</string> - <object id="ExprNodeGenericFuncDesc1" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -491,10 +491,17 @@ <void property="colList"> <object class="java.util.ArrayList"> <void method="add"> - <object idref="ExprNodeGenericFuncDesc1"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo0"/> + </void> + <void property="value"> + <int>2</int> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc0"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc"/> </void> </object> </void> Modified: hive/branches/spark/ql/src/test/results/compiler/plan/udf_when.q.xml URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/compiler/plan/udf_when.q.xml?rev=1613740&r1=1613739&r2=1613740&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/compiler/plan/udf_when.q.xml (original) +++ hive/branches/spark/ql/src/test/results/compiler/plan/udf_when.q.xml Sat Jul 26 23:45:46 2014 @@ -346,7 +346,7 @@ <object class="java.util.HashMap"> <void method="put"> <string>_col1</string> - <object id="ExprNodeGenericFuncDesc0" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -453,7 +453,7 @@ </void> <void method="put"> <string>_col0</string> - <object id="ExprNodeGenericFuncDesc1" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc"> <void property="children"> <object class="java.util.ArrayList"> <void method="add"> @@ -571,10 +571,17 @@ <void property="colList"> <object class="java.util.ArrayList"> <void method="add"> - <object idref="ExprNodeGenericFuncDesc1"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc"> + <void property="typeInfo"> + <object idref="PrimitiveTypeInfo0"/> + </void> + <void property="value"> + <int>2</int> + </void> + </object> </void> <void method="add"> - <object idref="ExprNodeGenericFuncDesc0"/> + <object class="org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc"/> </void> </object> </void> Modified: hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java?rev=1613740&r1=1613739&r2=1613740&view=diff ============================================================================== --- hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java (original) +++ hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java Sat Jul 26 23:45:46 2014 @@ -262,12 +262,16 @@ class AvroDeserializer { TypeInfo columnType) throws AvroSerdeException { int tag = GenericData.get().resolveUnion(recordSchema, datum); // Determine index of value Schema schema = recordSchema.getTypes().get(tag); - if(schema.getType().equals(Schema.Type.NULL)) { + if (schema.getType().equals(Schema.Type.NULL)) { return null; } - return worker(datum, fileSchema == null ? null : fileSchema.getTypes().get(tag), schema, - SchemaToTypeInfo.generateTypeInfo(schema)); + Schema currentFileSchema = null; + if (fileSchema != null) { + currentFileSchema = + fileSchema.getType() == Type.UNION ? fileSchema.getTypes().get(tag) : fileSchema; + } + return worker(datum, currentFileSchema, schema, SchemaToTypeInfo.generateTypeInfo(schema)); } Modified: hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java?rev=1613740&r1=1613739&r2=1613740&view=diff ============================================================================== --- hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java (original) +++ hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java Sat Jul 26 23:45:46 2014 @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hive.serde2.avro; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Properties; @@ -29,6 +31,7 @@ import org.apache.hadoop.hive.serde2.Ser import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.io.Writable; /** @@ -50,6 +53,8 @@ public class AvroSerDe extends AbstractS private AvroSerializer avroSerializer = null; private boolean badSchema = false; + private static String TABLE_NAME = "name"; + private static String TABLE_COMMENT = "comment"; @Override public void initialize(Configuration configuration, Properties tableProperties, @@ -61,17 +66,56 @@ public class AvroSerDe extends AbstractS @Override public void initialize(Configuration configuration, Properties properties) throws SerDeException { // Reset member variables so we don't get in a half-constructed state - if(schema != null) { + if (schema != null) { LOG.info("Resetting already initialized AvroSerDe"); } schema = null; oi = null; - columnNames = null; + columnNames = null; columnTypes = null; - schema = AvroSerdeUtils.determineSchemaOrReturnErrorSchema(properties); - if(configuration == null) { + final String columnNameProperty = properties.getProperty("columns"); + final String columnTypeProperty = properties.getProperty("columns.types"); + final String columnCommentProperty = properties.getProperty("columns.comments"); + + if (properties.getProperty(AvroSerdeUtils.SCHEMA_LITERAL) != null + || properties.getProperty(AvroSerdeUtils.SCHEMA_URL) != null + || columnNameProperty == null || columnNameProperty.isEmpty() + || columnTypeProperty == null || columnTypeProperty.isEmpty()) { + schema = AvroSerdeUtils.determineSchemaOrReturnErrorSchema(properties); + } else { + // Get column names and sort order + columnNames = Arrays.asList(columnNameProperty.split(",")); + columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty); + + List<String> columnComments; + if (columnCommentProperty.isEmpty()) { + columnComments = new ArrayList<String>(); + } else { + columnComments = Arrays.asList(columnCommentProperty.split(",")); + LOG.info("columnComments is " + columnCommentProperty); + } + if (columnNames.size() != columnTypes.size()) { + throw new IllegalArgumentException("AvroSerde initialization failed. Number of column " + + "name and column type differs. columnNames = " + columnNames + ", columnTypes = " + + columnTypes); + } + + final String tableName = properties.getProperty(TABLE_NAME); + final String tableComment = properties.getProperty(TABLE_COMMENT); + TypeInfoToSchema typeInfoToSchema = new TypeInfoToSchema(); + schema = typeInfoToSchema.convert(columnNames, columnTypes, columnComments, + properties.getProperty(AvroSerdeUtils.SCHEMA_NAMESPACE), + properties.getProperty(AvroSerdeUtils.SCHEMA_NAME, tableName), + properties.getProperty(AvroSerdeUtils.SCHEMA_DOC, tableComment)); + + properties.setProperty(AvroSerdeUtils.SCHEMA_LITERAL, schema.toString()); + } + + LOG.info("Avro schema is " + schema); + + if (configuration == null) { LOG.info("Configuration null, not inserting schema"); } else { configuration.set(AvroSerdeUtils.AVRO_SERDE_SCHEMA, schema.toString(false)); Modified: hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java?rev=1613740&r1=1613739&r2=1613740&view=diff ============================================================================== --- hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java (original) +++ hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java Sat Jul 26 23:45:46 2014 @@ -49,6 +49,9 @@ public class AvroSerdeUtils { public static final String SCHEMA_LITERAL = "avro.schema.literal"; public static final String SCHEMA_URL = "avro.schema.url"; public static final String SCHEMA_NONE = "none"; + public static final String SCHEMA_NAMESPACE = "avro.schema.namespace"; + public static final String SCHEMA_NAME = "avro.schema.name"; + public static final String SCHEMA_DOC = "avro.schema.doc"; public static final String EXCEPTION_MESSAGE = "Neither " + SCHEMA_LITERAL + " nor " + SCHEMA_URL + " specified, can't determine table schema"; public static final String AVRO_SERDE_SCHEMA = "avro.serde.schema"; Modified: hive/branches/spark/service/src/java/org/apache/hive/service/cli/CLIService.java URL: http://svn.apache.org/viewvc/hive/branches/spark/service/src/java/org/apache/hive/service/cli/CLIService.java?rev=1613740&r1=1613739&r2=1613740&view=diff ============================================================================== --- hive/branches/spark/service/src/java/org/apache/hive/service/cli/CLIService.java (original) +++ hive/branches/spark/service/src/java/org/apache/hive/service/cli/CLIService.java Sat Jul 26 23:45:46 2014 @@ -35,6 +35,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.SystemVariables; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.ql.metadata.Hive; @@ -443,7 +444,10 @@ public class CLIService extends Composit // create the give Path if doesn't exists and make it writable private void setupStagingDir(String dirPath, boolean isLocal) throws IOException { - Path scratchDir = new Path(dirPath); + Path scratchDir = getStaticPath(new Path(dirPath)); + if (scratchDir == null) { + return; + } FileSystem fs; if (isLocal) { fs = FileSystem.getLocal(hiveConf); @@ -480,4 +484,16 @@ public class CLIService extends Composit sessionManager.getSession(sessionHandle).renewDelegationToken(authFactory, tokenStr); LOG.info(sessionHandle + ": renewDelegationToken()"); } + + // DOWNLOADED_RESOURCES_DIR for example, which is by default ${system:java.io.tmpdir}/${hive.session.id}_resources, + // {system:java.io.tmpdir} would be already evaluated but ${hive.session.id} would be not in here. + // for that case, this returns evaluatd parts only, in this case, "/tmp" + // what for ${hive.session.id}_resources/${system:java.io.tmpdir}? just don't do that. + private Path getStaticPath(Path path) { + Path current = path; + for (; current != null && SystemVariables.containsVar(current.getName()); + current = current.getParent()) { + } + return current; + } } Modified: hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java URL: http://svn.apache.org/viewvc/hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java?rev=1613740&r1=1613739&r2=1613740&view=diff ============================================================================== --- hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java (original) +++ hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java Sat Jul 26 23:45:46 2014 @@ -24,6 +24,8 @@ import org.apache.hive.service.cli.Sessi import org.apache.hive.service.cli.operation.OperationManager; import org.apache.hive.service.cli.thrift.TProtocolVersion; +import java.util.Map; + /** * Methods that don't need to be executed under a doAs * context are here. Rest of them in HiveSession interface @@ -49,6 +51,12 @@ public interface HiveSessionBase { */ public void setOperationManager(OperationManager operationManager); + /** + * Initialize the session + * @param sessionConfMap + */ + public void initialize(Map<String, String> sessionConfMap); + public SessionHandle getSessionHandle(); public String getUsername(); Modified: hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java URL: http://svn.apache.org/viewvc/hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java?rev=1613740&r1=1613739&r2=1613740&view=diff ============================================================================== --- hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java (original) +++ hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java Sat Jul 26 23:45:46 2014 @@ -18,7 +18,11 @@ package org.apache.hive.service.cli.session; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStreamReader; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -26,6 +30,8 @@ import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.common.cli.HiveFileProcessor; +import org.apache.hadoop.hive.common.cli.IHiveFileProcessor; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; @@ -82,7 +88,7 @@ public class HiveSessionImpl implements private final Set<OperationHandle> opHandleSet = new HashSet<OperationHandle>(); public HiveSessionImpl(TProtocolVersion protocol, String username, String password, - HiveConf serverhiveConf, Map<String, String> sessionConfMap, String ipAddress) { + HiveConf serverhiveConf, String ipAddress) { this.username = username; this.password = password; this.sessionHandle = new SessionHandle(protocol); @@ -98,8 +104,16 @@ public class HiveSessionImpl implements hiveConf.setInt(ListSinkOperator.OUTPUT_PROTOCOL, protocol.getValue()); sessionState = new SessionState(hiveConf, username); + sessionState.setUserIpAddress(ipAddress); sessionState.setIsHiveServerQuery(true); SessionState.start(sessionState); + } + + @Override + public void initialize(Map<String, String> sessionConfMap) { + //process global init file: .hiverc + processGlobalInitFile(); + SessionState.setCurrentSessionState(sessionState); //set conf properties specified by user from client side if (sessionConfMap != null) { @@ -107,6 +121,53 @@ public class HiveSessionImpl implements } } + /** + * It is used for processing hiverc file from HiveServer2 side. + */ + private class GlobalHivercFileProcessor extends HiveFileProcessor { + @Override + protected BufferedReader loadFile(String fileName) throws IOException { + FileInputStream initStream = null; + BufferedReader bufferedReader = null; + initStream = new FileInputStream(fileName); + bufferedReader = new BufferedReader(new InputStreamReader(initStream)); + return bufferedReader; + } + + @Override + protected int processCmd(String cmd) { + int rc = 0; + String cmd_trimed = cmd.trim(); + try { + executeStatementInternal(cmd_trimed, null, false); + } catch (HiveSQLException e) { + rc = -1; + LOG.warn("Failed to execute HQL command in global .hiverc file.", e); + } + return rc; + } + } + + private void processGlobalInitFile() { + IHiveFileProcessor processor = new GlobalHivercFileProcessor(); + + try { + if (hiveConf.getVar(ConfVars.HIVE_GLOBAL_INIT_FILE_LOCATION) != null) { + String hiverc = hiveConf.getVar(ConfVars.HIVE_GLOBAL_INIT_FILE_LOCATION) + + File.separator + SessionManager.HIVERCFILE; + if (new File(hiverc).exists()) { + LOG.info("Running global init file: " + hiverc); + int rc = processor.processFile(hiverc); + if (rc != 0) { + LOG.warn("Failed on initializing global .hiverc file"); + } + } + } + } catch (IOException e) { + LOG.warn("Failed on initializing global .hiverc file", e); + } + } + private void configureSession(Map<String, String> sessionConfMap) { for (Map.Entry<String, String> entry : sessionConfMap.entrySet()) { String key = entry.getKey(); Modified: hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java URL: http://svn.apache.org/viewvc/hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java?rev=1613740&r1=1613739&r2=1613740&view=diff ============================================================================== --- hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java (original) +++ hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java Sat Jul 26 23:45:46 2014 @@ -44,9 +44,8 @@ public class HiveSessionImplwithUGI exte private HiveSession proxySession = null; public HiveSessionImplwithUGI(TProtocolVersion protocol, String username, String password, - HiveConf hiveConf, Map<String, String> sessionConf, String ipAddress, - String delegationToken) throws HiveSQLException { - super(protocol, username, password, hiveConf, sessionConf, ipAddress); + HiveConf hiveConf, String ipAddress, String delegationToken) throws HiveSQLException { + super(protocol, username, password, hiveConf, ipAddress); setSessionUGI(username); setDelegationToken(delegationToken); } Modified: hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/SessionManager.java URL: http://svn.apache.org/viewvc/hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/SessionManager.java?rev=1613740&r1=1613739&r2=1613740&view=diff ============================================================================== --- hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/SessionManager.java (original) +++ hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/SessionManager.java Sat Jul 26 23:45:46 2014 @@ -47,6 +47,7 @@ import org.apache.hive.service.cli.thrif public class SessionManager extends CompositeService { private static final Log LOG = LogFactory.getLog(CompositeService.class); + public static final String HIVERCFILE = ".hiverc"; private HiveConf hiveConf; private final Map<SessionHandle, HiveSession> handleToSession = new ConcurrentHashMap<SessionHandle, HiveSession>(); @@ -122,15 +123,16 @@ public class SessionManager extends Comp HiveSession session; if (withImpersonation) { HiveSessionImplwithUGI hiveSessionUgi = new HiveSessionImplwithUGI(protocol, username, password, - hiveConf, sessionConf, TSetIpAddressProcessor.getUserIpAddress(), delegationToken); + hiveConf, TSetIpAddressProcessor.getUserIpAddress(), delegationToken); session = HiveSessionProxy.getProxy(hiveSessionUgi, hiveSessionUgi.getSessionUgi()); hiveSessionUgi.setProxySession(session); } else { - session = new HiveSessionImpl(protocol, username, password, hiveConf, sessionConf, + session = new HiveSessionImpl(protocol, username, password, hiveConf, TSetIpAddressProcessor.getUserIpAddress()); } session.setSessionManager(this); session.setOperationManager(operationManager); + session.initialize(sessionConf); session.open(); handleToSession.put(session.getSessionHandle(), session); Modified: hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/EmbeddedThriftBinaryCLIService.java URL: http://svn.apache.org/viewvc/hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/EmbeddedThriftBinaryCLIService.java?rev=1613740&r1=1613739&r2=1613740&view=diff ============================================================================== --- hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/EmbeddedThriftBinaryCLIService.java (original) +++ hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/EmbeddedThriftBinaryCLIService.java Sat Jul 26 23:45:46 2014 @@ -32,6 +32,7 @@ public class EmbeddedThriftBinaryCLIServ public EmbeddedThriftBinaryCLIService() { super(new CLIService()); isEmbedded = true; + HiveConf.setLoadHiveServer2Config(true); cliService.init(new HiveConf()); cliService.start(); } Modified: hive/branches/spark/service/src/java/org/apache/hive/service/server/HiveServer2.java URL: http://svn.apache.org/viewvc/hive/branches/spark/service/src/java/org/apache/hive/service/server/HiveServer2.java?rev=1613740&r1=1613739&r2=1613740&view=diff ============================================================================== --- hive/branches/spark/service/src/java/org/apache/hive/service/server/HiveServer2.java (original) +++ hive/branches/spark/service/src/java/org/apache/hive/service/server/HiveServer2.java Sat Jul 26 23:45:46 2014 @@ -25,7 +25,6 @@ import org.apache.hadoop.hive.common.Log import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager; -import org.apache.hadoop.hive.ql.exec.tez.TezSessionState; import org.apache.hive.common.util.HiveStringUtils; import org.apache.hive.service.CompositeService; import org.apache.hive.service.cli.CLIService; @@ -45,6 +44,7 @@ public class HiveServer2 extends Composi public HiveServer2() { super("HiveServer2"); + HiveConf.setLoadHiveServer2Config(true); } @@ -142,7 +142,7 @@ public class HiveServer2 extends Composi // before any of the other core hive classes are loaded String initLog4jMessage = LogUtils.initHiveLog4j(); LOG.debug(initLog4jMessage); - + HiveStringUtils.startupShutdownMessage(HiveServer2.class, args, LOG); //log debug message from "oproc" after log4j initialize properly LOG.debug(oproc.getDebugMessage().toString()); Modified: hive/branches/spark/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java URL: http://svn.apache.org/viewvc/hive/branches/spark/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java?rev=1613740&r1=1613739&r2=1613740&view=diff ============================================================================== --- hive/branches/spark/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java (original) +++ hive/branches/spark/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java Sat Jul 26 23:45:46 2014 @@ -26,6 +26,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -67,6 +68,8 @@ import org.apache.hadoop.tools.HadoopArc import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.ToolRunner; +import com.google.common.primitives.Longs; + /** * Base implemention for shims against secure Hadoop 0.20.3/0.23. */ @@ -97,18 +100,12 @@ public abstract class HadoopShimsSecure _isShrinked = false; } - public InputSplitShim(CombineFileSplit old) throws IOException { - super(old.getJob(), old.getPaths(), old.getStartOffsets(), - old.getLengths(), dedup(old.getLocations())); + public InputSplitShim(JobConf conf, Path[] paths, long[] startOffsets, + long[] lengths, String[] locations) throws IOException { + super(conf, paths, startOffsets, lengths, dedup(locations)); _isShrinked = false; } - private static String[] dedup(String[] locations) { - Set<String> dedup = new HashSet<String>(); - Collections.addAll(dedup, locations); - return dedup.toArray(new String[dedup.size()]); - } - @Override public void shrinkSplit(long length) { _isShrinked = true; @@ -338,12 +335,22 @@ public abstract class HadoopShimsSecure InputSplit[] splits = (InputSplit[]) super.getSplits(job, numSplits); - InputSplitShim[] isplits = new InputSplitShim[splits.length]; + ArrayList<InputSplitShim> inputSplitShims = new ArrayList<InputSplitShim>(); for (int pos = 0; pos < splits.length; pos++) { - isplits[pos] = new InputSplitShim((CombineFileSplit)splits[pos]); + CombineFileSplit split = (CombineFileSplit) splits[pos]; + Set<Integer> dirIndices = getDirIndices(split.getPaths(), job); + if (dirIndices.size() != split.getPaths().length) { + List<Path> prunedPaths = prune(dirIndices, Arrays.asList(split.getPaths())); + List<Long> prunedStartOffsets = prune(dirIndices, Arrays.asList( + ArrayUtils.toObject(split.getStartOffsets()))); + List<Long> prunedLengths = prune(dirIndices, Arrays.asList( + ArrayUtils.toObject(split.getLengths()))); + inputSplitShims.add(new InputSplitShim(job, prunedPaths.toArray(new Path[prunedPaths.size()]), + Longs.toArray(prunedStartOffsets), + Longs.toArray(prunedLengths), split.getLocations())); + } } - - return isplits; + return inputSplitShims.toArray(new InputSplitShim[inputSplitShims.size()]); } public InputSplitShim getInputSplitShim() throws IOException { @@ -623,4 +630,37 @@ public abstract class HadoopShimsSecure int retval = shell.run(command); LOG.debug("Return value is :" + retval); } + + /** + * CombineFileInputFormat sometimes returns directories as splits, need to prune them. + */ + private static Set<Integer> getDirIndices(Path[] paths, JobConf conf) throws IOException { + Set<Integer> result = new HashSet<Integer>(); + for (int i = 0; i < paths.length; i++) { + FileSystem fs = paths[i].getFileSystem(conf); + if (!fs.isFile(paths[i])) { + result.add(i); + } + } + return result; + } + + private static <K> List<K> prune(Set<Integer> indicesToPrune, List<K> elms) { + List<K> result = new ArrayList<K>(); + int i = 0; + for (K elm : elms) { + if (indicesToPrune.contains(i)) { + continue; + } + result.add(elm); + i++; + } + return result; + } + + private static String[] dedup(String[] locations) throws IOException { + Set<String> dedup = new HashSet<String>(); + Collections.addAll(dedup, locations); + return dedup.toArray(new String[dedup.size()]); + } } Modified: hive/branches/spark/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java URL: http://svn.apache.org/viewvc/hive/branches/spark/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java?rev=1613740&r1=1613739&r2=1613740&view=diff ============================================================================== --- hive/branches/spark/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java (original) +++ hive/branches/spark/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java Sat Jul 26 23:45:46 2014 @@ -53,8 +53,8 @@ public class TestExecutor extends Thread private final BlockingQueue<Test> mTestQueue; private final PTest.Builder mPTestBuilder; private ExecutionContext mExecutionContext; - private boolean execute; + public TestExecutor(ExecutionContextConfiguration executionContextConfiguration, ExecutionContextProvider executionContextProvider, BlockingQueue<Test> testQueue, PTest.Builder pTestBuilder) { @@ -111,10 +111,12 @@ public class TestExecutor extends Thread testConfiguration.setPatch(startRequest.getPatchURL()); testConfiguration.setJiraName(startRequest.getJiraName()); testConfiguration.setClearLibraryCache(startRequest.isClearLibraryCache()); + LocalCommandFactory localCommandFactory = new LocalCommandFactory(logger); PTest ptest = mPTestBuilder.build(testConfiguration, mExecutionContext, test.getStartRequest().getTestHandle(), logDir, - new LocalCommandFactory(logger), new SSHCommandExecutor(logger), - new RSyncCommandExecutor(logger), logger); + localCommandFactory, new SSHCommandExecutor(logger), + new RSyncCommandExecutor(logger, mExecutionContextConfiguration.getMaxRsyncThreads(), + localCommandFactory), logger); int result = ptest.run(); if(result == Constants.EXIT_CODE_SUCCESS) { test.setStatus(Status.ok()); Modified: hive/branches/spark/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java URL: http://svn.apache.org/viewvc/hive/branches/spark/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java?rev=1613740&r1=1613739&r2=1613740&view=diff ============================================================================== --- hive/branches/spark/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java (original) +++ hive/branches/spark/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java Sat Jul 26 23:45:46 2014 @@ -345,9 +345,10 @@ public class PTest { executionContextProvider = executionContextConfiguration .getExecutionContextProvider(); executionContext = executionContextProvider.createExecutionContext(); + LocalCommandFactory localCommandFactory = new LocalCommandFactory(LOG); PTest ptest = new PTest(conf, executionContext, buildTag, logDir, - new LocalCommandFactory(LOG), new SSHCommandExecutor(LOG), - new RSyncCommandExecutor(LOG), LOG); + localCommandFactory, new SSHCommandExecutor(LOG), + new RSyncCommandExecutor(LOG, 10, localCommandFactory), LOG); exitCode = ptest.run(); } finally { if(executionContext != null) { Modified: hive/branches/spark/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/ExecutionContextConfiguration.java URL: http://svn.apache.org/viewvc/hive/branches/spark/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/ExecutionContextConfiguration.java?rev=1613740&r1=1613739&r2=1613740&view=diff ============================================================================== --- hive/branches/spark/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/ExecutionContextConfiguration.java (original) +++ hive/branches/spark/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/ExecutionContextConfiguration.java Sat Jul 26 23:45:46 2014 @@ -37,11 +37,14 @@ public class ExecutionContextConfigurati public static final String WORKING_DIRECTORY = "workingDirectory"; public static final String PROFILE_DIRECTORY = "profileDirectory"; public static final String MAX_LOG_DIRS_PER_PROFILE = "maxLogDirectoriesPerProfile"; + private static final String MAX_RSYNC_THREADS = "maxRsyncThreads"; + private static final int MAX_RSYNC_THREADS_DEFAULT = 10; private final ExecutionContextProvider mExecutionContextProvider; private final String mWorkingDirectory; private final String mGlobalLogDirectory; private final String mProfileDirectory; private final int mMaxLogDirectoriesPerProfile; + private final int mMaxRsyncThreads; @VisibleForTesting public ExecutionContextConfiguration(Context context) @@ -52,6 +55,7 @@ public class ExecutionContextConfigurati Preconditions.checkArgument(!mProfileDirectory.isEmpty(), PROFILE_DIRECTORY + " is required"); mGlobalLogDirectory = Dirs.create(new File(mWorkingDirectory, "logs")).getAbsolutePath(); mMaxLogDirectoriesPerProfile = context.getInteger(MAX_LOG_DIRS_PER_PROFILE, 10); + mMaxRsyncThreads = context.getInteger(MAX_RSYNC_THREADS, MAX_RSYNC_THREADS_DEFAULT); String executionContextProviderBuilder = context.getString("executionContextProvider", FixedExecutionContextProvider.Builder.class.getName()).trim(); try { @@ -66,6 +70,9 @@ public class ExecutionContextConfigurati throw Throwables.propagate(e); } } + public int getMaxRsyncThreads() { + return mMaxRsyncThreads; + } public int getMaxLogDirectoriesPerProfile() { return mMaxLogDirectoriesPerProfile; }
