Author: cws
Date: Fri Dec 2 08:14:10 2011
New Revision: 1209384
URL: http://svn.apache.org/viewvc?rev=1209384&view=rev
Log:
HIVE-2523. Add a new builtins subproject (John Sichi via cws)
Added:
hive/trunk/builtins/
hive/trunk/builtins/build-plugin.xml
hive/trunk/builtins/build.xml
hive/trunk/builtins/ivy.xml
hive/trunk/builtins/src/
hive/trunk/builtins/src/org/
hive/trunk/builtins/src/org/apache/
hive/trunk/builtins/src/org/apache/hive/
hive/trunk/builtins/src/org/apache/hive/builtins/
hive/trunk/builtins/src/org/apache/hive/builtins/BuiltinUtils.java
hive/trunk/builtins/src/org/apache/hive/builtins/UDAFUnionMap.java
hive/trunk/builtins/test/
hive/trunk/builtins/test/cleanup.sql
hive/trunk/builtins/test/iris.txt
hive/trunk/builtins/test/onerow.txt
hive/trunk/builtins/test/setup.sql
Modified:
hive/trunk/bin/hive
hive/trunk/build.xml
hive/trunk/eclipse-templates/.classpath
hive/trunk/pdk/scripts/build-plugin.xml
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out
Modified: hive/trunk/bin/hive
URL:
http://svn.apache.org/viewvc/hive/trunk/bin/hive?rev=1209384&r1=1209383&r2=1209384&view=diff
==============================================================================
--- hive/trunk/bin/hive (original)
+++ hive/trunk/bin/hive Fri Dec 2 08:14:10 2011
@@ -74,6 +74,11 @@ if [ ! -f ${HIVE_LIB}/hive-exec-*.jar ];
exit 1;
fi
+if [ ! -f ${HIVE_LIB}/hive-builtins-*.jar ]; then
+ echo "Missing Hive Builtins Jar: ${HIVE_LIB}/hive-builtins-*.jar"
+ exit 1;
+fi
+
if [ ! -f ${HIVE_LIB}/hive-metastore-*.jar ]; then
echo "Missing Hive MetaStore Jar"
exit 2;
Modified: hive/trunk/build.xml
URL:
http://svn.apache.org/viewvc/hive/trunk/build.xml?rev=1209384&r1=1209383&r2=1209384&view=diff
==============================================================================
--- hive/trunk/build.xml (original)
+++ hive/trunk/build.xml Fri Dec 2 08:14:10 2011
@@ -139,7 +139,7 @@
<subant target="@{target}">
<property name="build.dir.hive" location="${build.dir.hive}"/>
<property name="is-offline" value="${is-offline}"/>
- <filelist dir="."
files="ant/build.xml,shims/build.xml,common/build.xml,serde/build.xml,metastore/build.xml,ql/build.xml,contrib/build.xml,service/build.xml,cli/build.xml,jdbc/build.xml,hwi/build.xml,hbase-handler/build.xml,ant/build.xml,pdk/build.xml"/>
+ <filelist dir="."
files="ant/build.xml,shims/build.xml,common/build.xml,serde/build.xml,metastore/build.xml,ql/build.xml,contrib/build.xml,service/build.xml,cli/build.xml,jdbc/build.xml,hwi/build.xml,hbase-handler/build.xml,ant/build.xml,pdk/build.xml,builtins/build.xml"/>
</subant>
</sequential>
</macrodef>
@@ -150,7 +150,7 @@
<subant target="@{target}">
<property name="build.dir.hive" location="${build.dir.hive}"/>
<property name="is-offline" value="${is-offline}"/>
- <filelist dir="."
files="shims/build.xml,common/build.xml,serde/build.xml,metastore/build.xml,ql/build.xml,contrib/build.xml,service/build.xml,cli/build.xml,jdbc/build.xml,hwi/build.xml,hbase-handler/build.xml,pdk/build.xml"/>
+ <filelist dir="."
files="shims/build.xml,common/build.xml,serde/build.xml,metastore/build.xml,ql/build.xml,contrib/build.xml,service/build.xml,cli/build.xml,jdbc/build.xml,hwi/build.xml,hbase-handler/build.xml,pdk/build.xml,builtins/build.xml"/>
</subant>
</sequential>
</macrodef>
@@ -519,6 +519,13 @@
<symlink overwrite="true" link="${target.lib.dir}/libthrift.jar"
resource="libthrift-${libthrift.version}.jar"/>
<symlink overwrite="true" link="${target.lib.dir}/libfb303.jar"
resource="libfb303-${libfb303.version}.jar"/>
<symlink overwrite="true" link="${target.lib.dir}/hive_contrib.jar"
resource="hive-contrib-${version}.jar"/>
+ <!-- special case because builtins compilation depends on packaging
+ up everything else first -->
+ <ant antfile="${hive.root}/builtins/build.xml" target="package"
+ inheritAll="false" />
+ <copy todir="${target.lib.dir}" preservelastmodified="true" flatten="true">
+ <fileset file="${build.dir.hive}/builtins/hive-builtins-${version}.jar"/>
+ </copy>
</target>
@@ -665,6 +672,7 @@
<packageset dir="${build.dir.hive}/ql/gen/antlr/gen-java"/>
<packageset dir="shims/src/common/java"/>
<packageset dir="pdk/src/java"/>
+ <packageset dir="builtins/src"/>
<link href="${javadoc.link.java}"/>
@@ -902,6 +910,8 @@
todir="${mvn.jar.dir}" />
<copy file="${build.dir.hive}/pdk/hive-pdk-${version}.jar"
todir="${mvn.jar.dir}" />
+ <copy file="${build.dir.hive}/pdk/hive-builtins-${version}.jar"
+ todir="${mvn.jar.dir}" />
<!-- copy over maven pom files created using the make-pom target and
rename to maven convention -->
<copy file="${build.dir.hive}/anttasks/pom.xml"
@@ -930,6 +940,8 @@
tofile="${mvn.pom.dir}/hive-shims-${version}.pom" />
<copy file="${build.dir.hive}/pdk/pom.xml"
tofile="${mvn.pom.dir}/hive-pdk-${version}.pom" />
+ <copy file="${build.dir.hive}/builtins/pom.xml"
+ tofile="${mvn.pom.dir}/hive-builtins-${version}.pom" />
<!-- copy over licence -->
<copy file="${hive.root}/LICENSE" todir="${mvn.license.dir}" />
@@ -1019,6 +1031,9 @@
<param name="hive.project" value="pdk" />
</antcall>
<antcall target="maven-publish-artifact">
+ <param name="hive.project" value="builtins" />
+ </antcall>
+ <antcall target="maven-publish-artifact">
<param name="hive.project" value="jdbc" />
</antcall>
<antcall target="maven-publish-artifact">
@@ -1138,6 +1153,14 @@
output.file="${mvn.pom.dir}/hive-pdk-${version}.pom.asc"
gpg.passphrase="${gpg.passphrase}"/>
<sign-artifact
+ input.file="${mvn.jar.dir}/hive-builtins-${version}.jar"
+ output.file="${mvn.jar.dir}/hive-builtins-${version}.jar.asc"
+ gpg.passphrase="${gpg.passphrase}"/>
+ <sign-artifact
+ input.file="${mvn.pom.dir}/hive-builtins-${version}.pom"
+ output.file="${mvn.pom.dir}/hive-builtins-${version}.pom.asc"
+ gpg.passphrase="${gpg.passphrase}"/>
+ <sign-artifact
input.file="${mvn.jar.dir}/hive-serde-${version}.jar"
output.file="${mvn.jar.dir}/hive-serde-${version}.jar.asc"
gpg.passphrase="${gpg.passphrase}"/>
Added: hive/trunk/builtins/build-plugin.xml
URL:
http://svn.apache.org/viewvc/hive/trunk/builtins/build-plugin.xml?rev=1209384&view=auto
==============================================================================
--- hive/trunk/builtins/build-plugin.xml (added)
+++ hive/trunk/builtins/build-plugin.xml Fri Dec 2 08:14:10 2011
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project name="builtins-plugin" default="jar">
+ <dirname property="builtins.dir" file="${ant.file.builtins}"/>
+ <property name="hive.root" location="${builtins.dir}/.."/>
+ <property file="${hive.root}/build.properties"/>
+ <property name="plugin.libname" value="hive-builtins"/>
+ <property name="plugin.title" value="Hive Builtin Function Library"/>
+ <property name="plugin.version" value="${version}"/>
+ <property name="plugin.vendor" value="Apache Software Foundation"/>
+ <property name="function.sql.prefix" value=""/>
+ <property name="build.dir" location="${hive.root}/build/builtins"/>
+ <property name="install.dir" location="${hive.root}/build/dist"/>
+ <import file="${hive.root}/pdk/scripts/build-plugin.xml"/>
+</project>
Added: hive/trunk/builtins/build.xml
URL:
http://svn.apache.org/viewvc/hive/trunk/builtins/build.xml?rev=1209384&view=auto
==============================================================================
--- hive/trunk/builtins/build.xml (added)
+++ hive/trunk/builtins/build.xml Fri Dec 2 08:14:10 2011
@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project name="builtins" default="jar">
+
+ <property name="src.dir" location="${basedir}/src"/>
+ <import file="../build-common.xml"/>
+
+ <target name="compile" depends="init, setup">
+ <echo message="Project: ${ant.project.name}"/>
+ <!-- defer compilation until package phase -->
+ </target>
+
+ <target name="jar" depends="init">
+ <echo message="Project: ${ant.project.name}"/>
+ <!-- defer compilation until package phase -->
+ </target>
+
+ <target name="package">
+ <echo message="Project: ${ant.project.name}"/>
+ <ant antfile="build-plugin.xml" target="package" inheritAll="false">
+ <property name="hive.install.dir" value="${build.dir.hive}/dist"/>
+ <property name="hadoop.home" value="${hadoop.root}"/>
+ </ant>
+ </target>
+
+ <target name="test" unless="testcase">
+ <echo message="Project: ${ant.project.name}"/>
+ <ant antfile="build-plugin.xml" target="test" inheritAll="false">
+ <property name="hive.install.dir" value="${build.dir.hive}/dist"/>
+ <property name="hadoop.home" value="${hadoop.root}"/>
+ </ant>
+ </target>
+
+ <target name="clean">
+ <echo message="Project: ${ant.project.name}"/>
+ <ant antfile="build-plugin.xml" target="clean" inheritAll="false">
+ <property name="hive.install.dir" value="${build.dir.hive}/dist"/>
+ <property name="hadoop.home" value="${hadoop.root}"/>
+ </ant>
+ </target>
+</project>
Added: hive/trunk/builtins/ivy.xml
URL:
http://svn.apache.org/viewvc/hive/trunk/builtins/ivy.xml?rev=1209384&view=auto
==============================================================================
--- hive/trunk/builtins/ivy.xml (added)
+++ hive/trunk/builtins/ivy.xml Fri Dec 2 08:14:10 2011
@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<ivy-module version="2.0">
+ <info organisation="${hive.ivy.org}" module="hive-builtins"
revision="${version}">
+ <license name="The Apache Software License, Version 2.0"
url="http://www.apache.org/licenses/LICENSE-2.0.txt"; />
+ <description homepage="http://hive.apache.org";>
+ The Apache Hive (TM) data warehouse software facilitates querying and
managing large datasets residing in distributed storage.
+ https://cwiki.apache.org/confluence/display/Hive/Home
+ </description>
+ </info>
+ <dependencies>
+ </dependencies>
+</ivy-module>
Added: hive/trunk/builtins/src/org/apache/hive/builtins/BuiltinUtils.java
URL:
http://svn.apache.org/viewvc/hive/trunk/builtins/src/org/apache/hive/builtins/BuiltinUtils.java?rev=1209384&view=auto
==============================================================================
--- hive/trunk/builtins/src/org/apache/hive/builtins/BuiltinUtils.java (added)
+++ hive/trunk/builtins/src/org/apache/hive/builtins/BuiltinUtils.java Fri Dec
2 08:14:10 2011
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.builtins;
+
+/**
+ * Utility method class supporting builtin functions. We also
+ * use this class as a well-known name for loading metadata from
+ * the builtins jar.
+ */
+public abstract class BuiltinUtils
+{
+}
Added: hive/trunk/builtins/src/org/apache/hive/builtins/UDAFUnionMap.java
URL:
http://svn.apache.org/viewvc/hive/trunk/builtins/src/org/apache/hive/builtins/UDAFUnionMap.java?rev=1209384&view=auto
==============================================================================
--- hive/trunk/builtins/src/org/apache/hive/builtins/UDAFUnionMap.java (added)
+++ hive/trunk/builtins/src/org/apache/hive/builtins/UDAFUnionMap.java Fri Dec
2 08:14:10 2011
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.builtins;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hive.pdk.HivePdkUnitTest;
+import org.apache.hive.pdk.HivePdkUnitTests;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.udf.generic.AbstractGenericUDAFResolver;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
+import
org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationBuffer;
+import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+
+/**
+ * Aggregate all maps into a single map. If there are multiple values for
+ * the same key, result can contain any of those values.
+ * Because the mappers must keep all of the data in memory, if your data is
+ * non-trivially large you should set hive.map.aggr=false to ensure that
+ * UNION_MAP is only executed in the reduce phase.
+ */
+@HivePdkUnitTests(
+ setup = "",
+ cleanup = "",
+ cases = {
+ @HivePdkUnitTest(
+ query = "SELECT size(UNION_MAP(MAP(sepal_width, sepal_length))) "
+ +"FROM iris",
+ result = "23")
+ })
+@Description(
+ name = "union_map",
+ value = "_FUNC_(col) - aggregate given maps into a single map",
+ extended = "Aggregate maps, returns as a HashMap.")
+public class UDAFUnionMap extends AbstractGenericUDAFResolver {
+ @Override
+ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws
SemanticException {
+
+ // Next two validation calls are dependent on HIVE-2524, so
+ // leave them commented out for now.
+ //
+ // new LengthEquals(1).check(parameters.length);
+ // new IsMap().check(parameters[0], 0);
+
+ return new Evaluator();
+ }
+
+ public static class State implements AggregationBuffer {
+ HashMap<Object, Object> map = new HashMap<Object, Object>();
+ }
+
+ public static class Evaluator extends GenericUDAFEvaluator {
+ ObjectInspector inputOI;
+ MapObjectInspector internalMergeOI;
+
+ @Override
+ public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws
HiveException {
+ super.init(m, parameters);
+ if (m == Mode.COMPLETE || m == Mode.PARTIAL1) {
+ inputOI = (MapObjectInspector) parameters[0];
+ } else {
+ internalMergeOI = (MapObjectInspector) parameters[0];
+ }
+ return ObjectInspectorUtils.getStandardObjectInspector(parameters[0]);
+ }
+
+ @Override
+ public AggregationBuffer getNewAggregationBuffer() throws HiveException {
+ return new State();
+ }
+
+ @Override
+ public void iterate(AggregationBuffer agg, Object[] input) throws
HiveException {
+ if (input[0] != null) {
+ State state = (State) agg;
+
state.map.putAll((Map<?,?>)ObjectInspectorUtils.copyToStandardObject(input[0],
inputOI));
+ }
+ }
+
+ @Override
+ public void merge(AggregationBuffer agg, Object partial) throws
HiveException {
+ if (partial != null) {
+ State state = (State) agg;
+ Map<?,?> pset =
(Map<?,?>)ObjectInspectorUtils.copyToStandardObject(partial, internalMergeOI);
+ state.map.putAll(pset);
+ }
+ }
+
+ @Override
+ public void reset(AggregationBuffer agg) throws HiveException {
+ ((State) agg).map.clear();
+ }
+
+ @Override
+ public Object terminate(AggregationBuffer agg) throws HiveException {
+ return ((State) agg).map;
+ }
+
+ @Override
+ public Object terminatePartial(AggregationBuffer agg) throws HiveException
{
+ return ((State) agg).map;
+ }
+ }
+}
Added: hive/trunk/builtins/test/cleanup.sql
URL:
http://svn.apache.org/viewvc/hive/trunk/builtins/test/cleanup.sql?rev=1209384&view=auto
==============================================================================
--- hive/trunk/builtins/test/cleanup.sql (added)
+++ hive/trunk/builtins/test/cleanup.sql Fri Dec 2 08:14:10 2011
@@ -0,0 +1,2 @@
+drop table if exists onerow;
+drop table if exists iris;
Added: hive/trunk/builtins/test/iris.txt
URL:
http://svn.apache.org/viewvc/hive/trunk/builtins/test/iris.txt?rev=1209384&view=auto
==============================================================================
--- hive/trunk/builtins/test/iris.txt (added)
+++ hive/trunk/builtins/test/iris.txt Fri Dec 2 08:14:10 2011
@@ -0,0 +1,150 @@
+6 2.9 4.5 1.5 versicolor
+6.9 3.1 5.1 2.3 virginica
+5.4 3.4 1.5 0.4 setosa
+5.5 3.5 1.3 0.2 setosa
+4.6 3.1 1.5 0.2 setosa
+4.4 2.9 1.4 0.2 setosa
+5.9 3 4.2 1.5 versicolor
+5.1 3.8 1.6 0.2 setosa
+5 3.6 1.4 0.2 setosa
+6.7 3.1 5.6 2.4 virginica
+4.9 3.6 1.4 0.1 setosa
+4.9 2.5 4.5 1.7 virginica
+5.8 2.8 5.1 2.4 virginica
+4.5 2.3 1.3 0.3 setosa
+6.7 3 5.2 2.3 virginica
+6 3 4.8 1.8 virginica
+6.4 3.2 4.5 1.5 versicolor
+7.7 3.8 6.7 2.2 virginica
+5.7 2.6 3.5 1 versicolor
+5.7 2.9 4.2 1.3 versicolor
+5.4 3.9 1.7 0.4 setosa
+5 3.5 1.3 0.3 setosa
+6 2.2 5 1.5 virginica
+6.6 2.9 4.6 1.3 versicolor
+5.9 3 5.1 1.8 virginica
+4.7 3.2 1.6 0.2 setosa
+5 3.4 1.5 0.2 setosa
+6.9 3.2 5.7 2.3 virginica
+6.5 3 5.8 2.2 virginica
+5.5 4.2 1.4 0.2 setosa
+7.1 3 5.9 2.1 virginica
+6.2 2.9 4.3 1.3 versicolor
+5.7 4.4 1.5 0.4 setosa
+4.9 2.4 3.3 1 versicolor
+5.4 3.9 1.3 0.4 setosa
+6.4 2.8 5.6 2.1 virginica
+6.9 3.1 4.9 1.5 versicolor
+5.9 3.2 4.8 1.8 versicolor
+6.1 3 4.9 1.8 virginica
+5.6 3 4.5 1.5 versicolor
+5.3 3.7 1.5 0.2 setosa
+6.4 2.9 4.3 1.3 versicolor
+6.6 3 4.4 1.4 versicolor
+6.4 3.1 5.5 1.8 virginica
+6.3 2.5 4.9 1.5 versicolor
+6.7 3.3 5.7 2.5 virginica
+6.4 2.7 5.3 1.9 virginica
+6.5 3 5.5 1.8 virginica
+6.7 3.1 4.4 1.4 versicolor
+4.9 3 1.4 0.2 setosa
+6.2 2.8 4.8 1.8 virginica
+7.6 3 6.6 2.1 virginica
+5.2 3.5 1.5 0.2 setosa
+7.7 2.6 6.9 2.3 virginica
+6.5 3.2 5.1 2 virginica
+5.8 2.6 4 1.2 versicolor
+6.3 2.9 5.6 1.8 virginica
+5.8 4 1.2 0.2 setosa
+6.5 2.8 4.6 1.5 versicolor
+5 3.2 1.2 0.2 setosa
+7.4 2.8 6.1 1.9 virginica
+7 3.2 4.7 1.4 versicolor
+4.8 3.4 1.6 0.2 setosa
+6.4 2.8 5.6 2.2 virginica
+5.4 3.7 1.5 0.2 setosa
+5.7 3.8 1.7 0.3 setosa
+5.2 2.7 3.9 1.4 versicolor
+6.3 3.3 6 2.5 virginica
+5.1 2.5 3 1.1 versicolor
+7.7 2.8 6.7 2 virginica
+5.5 2.4 3.7 1 versicolor
+7.3 2.9 6.3 1.8 virginica
+4.3 3 1.1 0.1 setosa
+5.6 2.7 4.2 1.3 versicolor
+7.9 3.8 6.4 2 virginica
+5.6 2.8 4.9 2 virginica
+5 3 1.6 0.2 setosa
+6.1 3 4.6 1.4 versicolor
+5.1 3.5 1.4 0.2 setosa
+5.1 3.4 1.5 0.2 setosa
+4.8 3.4 1.9 0.2 setosa
+4.6 3.4 1.4 0.3 setosa
+6.5 3 5.2 2 virginica
+5.1 3.8 1.9 0.4 setosa
+5.6 2.9 3.6 1.3 versicolor
+6.7 3 5 1.7 versicolor
+7.2 3.6 6.1 2.5 virginica
+4.9 3.1 1.5 0.1 setosa
+6.2 3.4 5.4 2.3 virginica
+5.8 2.7 5.1 1.9 virginica
+5.6 2.5 3.9 1.1 versicolor
+6.3 2.5 5 1.9 virginica
+4.6 3.2 1.4 0.2 setosa
+5.8 2.7 5.1 1.9 virginica
+5.5 2.3 4 1.3 versicolor
+6.7 3.3 5.7 2.1 virginica
+6 2.2 4 1 versicolor
+5.1 3.7 1.5 0.4 setosa
+5 3.5 1.6 0.6 setosa
+5.8 2.7 4.1 1 versicolor
+5.7 2.8 4.1 1.3 versicolor
+6 3.4 4.5 1.6 versicolor
+6.1 2.9 4.7 1.4 versicolor
+5.2 3.4 1.4 0.2 setosa
+5.6 3 4.1 1.3 versicolor
+6.8 2.8 4.8 1.4 versicolor
+5.8 2.7 3.9 1.2 versicolor
+4.8 3.1 1.6 0.2 setosa
+5.1 3.5 1.4 0.3 setosa
+5 3.3 1.4 0.2 setosa
+7.2 3.2 6 1.8 virginica
+6.7 3.1 4.7 1.5 versicolor
+6.8 3.2 5.9 2.3 virginica
+4.6 3.6 1 0.2 setosa
+5.4 3 4.5 1.5 versicolor
+6.4 3.2 5.3 2.3 virginica
+4.8 3 1.4 0.1 setosa
+5.5 2.5 4 1.3 versicolor
+6.9 3.1 5.4 2.1 virginica
+6.8 3 5.5 2.1 virginica
+4.8 3 1.4 0.3 setosa
+6.3 2.7 4.9 1.8 virginica
+6 2.7 5.1 1.6 versicolor
+6.1 2.6 5.6 1.4 virginica
+7.2 3 5.8 1.6 virginica
+5 2.3 3.3 1 versicolor
+6.3 3.3 4.7 1.6 versicolor
+6.3 2.8 5.1 1.5 virginica
+4.7 3.2 1.3 0.2 setosa
+6.3 3.4 5.6 2.4 virginica
+5.7 2.8 4.5 1.3 versicolor
+5.7 3 4.2 1.2 versicolor
+4.9 3.1 1.5 0.2 setosa
+5 2 3.5 1 versicolor
+7.7 3 6.1 2.3 virginica
+5.7 2.5 5 2 virginica
+5.5 2.4 3.8 1.1 versicolor
+5.1 3.3 1.7 0.5 setosa
+6.3 2.3 4.4 1.3 versicolor
+4.4 3 1.3 0.2 setosa
+6.1 2.8 4 1.3 versicolor
+6.1 2.8 4.7 1.2 versicolor
+5.5 2.6 4.4 1.2 versicolor
+5.1 3.8 1.5 0.3 setosa
+4.4 3.2 1.3 0.2 setosa
+5 3.4 1.6 0.4 setosa
+6.7 2.5 5.8 1.8 virginica
+5.2 4.1 1.5 0.1 setosa
+6.2 2.2 4.5 1.5 versicolor
+5.4 3.4 1.7 0.2 setosa
Added: hive/trunk/builtins/test/onerow.txt
URL:
http://svn.apache.org/viewvc/hive/trunk/builtins/test/onerow.txt?rev=1209384&view=auto
==============================================================================
--- hive/trunk/builtins/test/onerow.txt (added)
+++ hive/trunk/builtins/test/onerow.txt Fri Dec 2 08:14:10 2011
@@ -0,0 +1 @@
+plugh
Added: hive/trunk/builtins/test/setup.sql
URL:
http://svn.apache.org/viewvc/hive/trunk/builtins/test/setup.sql?rev=1209384&view=auto
==============================================================================
--- hive/trunk/builtins/test/setup.sql (added)
+++ hive/trunk/builtins/test/setup.sql Fri Dec 2 08:14:10 2011
@@ -0,0 +1,10 @@
+create table onerow(s string);
+load data local inpath '${env:HIVE_PLUGIN_ROOT_DIR}/test/onerow.txt'
+overwrite into table onerow;
+create table iris(
+sepal_length string, sepal_width string,
+petal_length string, petal_width string,
+species string)
+row format delimited fields terminated by '\t' stored as textfile;
+load data local inpath '${env:HIVE_PLUGIN_ROOT_DIR}/test/iris.txt'
+overwrite into table iris;
Modified: hive/trunk/eclipse-templates/.classpath
URL:
http://svn.apache.org/viewvc/hive/trunk/eclipse-templates/.classpath?rev=1209384&r1=1209383&r2=1209384&view=diff
==============================================================================
--- hive/trunk/eclipse-templates/.classpath (original)
+++ hive/trunk/eclipse-templates/.classpath Fri Dec 2 08:14:10 2011
@@ -69,6 +69,7 @@
<classpathentry excluding="queries/|results/|templates/" kind="src"
path="hbase-handler/src/test"/>
<classpathentry kind="src" path="pdk/src/java"/>
<classpathentry kind="src" path="pdk/test-plugin/src"/>
+ <classpathentry kind="src" path="builtins/src/java"/>
<classpathentry kind="src" path="cli/src/test"/>
<classpathentry kind="output" path="build/eclipse-classes"/>
</classpath>
Modified: hive/trunk/pdk/scripts/build-plugin.xml
URL:
http://svn.apache.org/viewvc/hive/trunk/pdk/scripts/build-plugin.xml?rev=1209384&r1=1209383&r2=1209384&view=diff
==============================================================================
--- hive/trunk/pdk/scripts/build-plugin.xml (original)
+++ hive/trunk/pdk/scripts/build-plugin.xml Fri Dec 2 08:14:10 2011
@@ -59,10 +59,14 @@
</javac>
</target>
- <target name="jar" depends="compile">
+ <target name="jar" depends="extract-functions">
<jar
jarfile="${build.dir}/${plugin.jar.basename}"
basedir="${build.classes}">
+ <metainf dir="${build.metadata}">
+ <include name="class-info.xml"/>
+ <include name="class-registration.sql"/>
+ </metainf>
<manifest>
<attribute name="Implementation-Title" value="${plugin.title}"/>
<attribute name="Implementation-Version" value="${plugin.version}"/>
@@ -111,7 +115,7 @@
<delete quiet="true" dir="${build.dir}"/>
</target>
- <target name="package" depends="jar,extract-functions"/>
+ <target name="package" depends="jar"/>
<target name="test" depends="package" unless="testcase">
<junit fork="true" printsummary="on" haltonfailure="true"
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1209384&r1=1209383&r2=1209384&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
Fri Dec 2 08:14:10 2011
@@ -18,7 +18,9 @@
package org.apache.hadoop.hive.ql.exec;
+import java.io.InputStream;
import java.lang.reflect.Method;
+import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -30,6 +32,9 @@ import java.util.TreeSet;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -212,8 +217,13 @@ import org.apache.hadoop.hive.serde2.typ
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.ReflectionUtils;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
/**
* FunctionRegistry.
@@ -1130,6 +1140,82 @@ public final class FunctionRegistry {
return UDFOPPositive.class == udfClass;
}
+ /**
+ * Registers the appropriate kind of temporary function based on a class's
+ * type.
+ *
+ * @param functionName name under which to register function
+ *
+ * @param udfClass class implementing UD[A|T]F
+ *
+ * @return true if udfClass's type was recognized (so registration
+ * succeeded); false otherwise
+ */
+ public static boolean registerTemporaryFunction(
+ String functionName, Class<?> udfClass) {
+
+ if (UDF.class.isAssignableFrom(udfClass)) {
+ FunctionRegistry.registerTemporaryUDF(
+ functionName, (Class<? extends UDF>) udfClass, false);
+ } else if (GenericUDF.class.isAssignableFrom(udfClass)) {
+ FunctionRegistry.registerTemporaryGenericUDF(
+ functionName, (Class<? extends GenericUDF>) udfClass);
+ } else if (GenericUDTF.class.isAssignableFrom(udfClass)) {
+ FunctionRegistry.registerTemporaryGenericUDTF(
+ functionName, (Class<? extends GenericUDTF>) udfClass);
+ } else if (UDAF.class.isAssignableFrom(udfClass)) {
+ FunctionRegistry.registerTemporaryUDAF(
+ functionName, (Class<? extends UDAF>) udfClass);
+ } else if (GenericUDAFResolver.class.isAssignableFrom(udfClass)) {
+ FunctionRegistry.registerTemporaryGenericUDAF(
+ functionName, (GenericUDAFResolver)
+ ReflectionUtils.newInstance(udfClass, null));
+ } else {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Registers Hive functions from a plugin jar, using metadata from
+ * the jar's META-INF/class-info.xml.
+ *
+ * @param jarLocation URL for reading jar file
+ *
+ * @param classLoader classloader to use for loading function classes
+ */
+ public static void registerFunctionsFromPluginJar(
+ URL jarLocation,
+ ClassLoader classLoader) throws Exception {
+
+ URL url = new URL("jar:" + jarLocation + "!/META-INF/class-info.xml");
+ InputStream inputStream = null;
+ try {
+ inputStream = url.openStream();
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ DocumentBuilder docBuilder = dbf.newDocumentBuilder();
+ Document doc = docBuilder.parse(inputStream);
+ Element root = doc.getDocumentElement();
+ if (!root.getTagName().equals("ClassList")) {
+ return;
+ }
+ NodeList children = root.getElementsByTagName("Class");
+ for (int i = 0; i < children.getLength(); ++i) {
+ Element child = (Element) children.item(i);
+ String javaName = child.getAttribute("javaname");
+ String sqlName = child.getAttribute("sqlname");
+ Class<?> udfClass = Class.forName(javaName, true, classLoader);
+ boolean registered = registerTemporaryFunction(sqlName, udfClass);
+ if (!registered) {
+ throw new RuntimeException(
+ "Class " + udfClass + " is not a Hive function implementation");
+ }
+ }
+ } finally {
+ IOUtils.closeStream(inputStream);
+ }
+ }
+
private FunctionRegistry() {
// prevent instantiation
}
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java?rev=1209384&r1=1209383&r2=1209384&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java Fri
Dec 2 08:14:10 2011
@@ -73,32 +73,15 @@ public class FunctionTask extends Task<F
private int createFunction(CreateFunctionDesc createFunctionDesc) {
try {
Class<?> udfClass = getUdfClass(createFunctionDesc);
- if (UDF.class.isAssignableFrom(udfClass)) {
- FunctionRegistry.registerTemporaryUDF(createFunctionDesc
- .getFunctionName(), (Class<? extends UDF>) udfClass, false);
- return 0;
- } else if (GenericUDF.class.isAssignableFrom(udfClass)) {
- FunctionRegistry.registerTemporaryGenericUDF(createFunctionDesc
- .getFunctionName(), (Class<? extends GenericUDF>) udfClass);
- return 0;
- } else if (GenericUDTF.class.isAssignableFrom(udfClass)) {
- FunctionRegistry.registerTemporaryGenericUDTF(createFunctionDesc
- .getFunctionName(), (Class<? extends GenericUDTF>) udfClass);
- return 0;
- } else if (UDAF.class.isAssignableFrom(udfClass)) {
- FunctionRegistry.registerTemporaryUDAF(createFunctionDesc
- .getFunctionName(), (Class<? extends UDAF>) udfClass);
- return 0;
- } else if (GenericUDAFResolver.class.isAssignableFrom(udfClass)) {
- FunctionRegistry.registerTemporaryGenericUDAF(createFunctionDesc
- .getFunctionName(), (GenericUDAFResolver) ReflectionUtils
- .newInstance(udfClass, null));
+ boolean registered = FunctionRegistry.registerTemporaryFunction(
+ createFunctionDesc.getFunctionName(),
+ udfClass);
+ if (registered) {
return 0;
}
console.printError("FAILED: Class " + createFunctionDesc.getClassName()
+ " does not implement UDF, GenericUDF, or UDAF");
return 1;
-
} catch (ClassNotFoundException e) {
console.printError("FAILED: Class " + createFunctionDesc.getClassName()
+ " not found");
LOG.info("create function: " + StringUtils.stringifyException(e));
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1209384&r1=1209383&r2=1209384&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Fri
Dec 2 08:14:10 2011
@@ -2275,4 +2275,8 @@ public final class Utilities {
return sb.toString();
}
+
+ public static Class getBuiltinUtilsClass() throws ClassNotFoundException {
+ return Class.forName("org.apache.hive.builtins.BuiltinUtils");
+ }
}
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1209384&r1=1209383&r2=1209384&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
Fri Dec 2 08:14:10 2011
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.net.URI;
+import java.net.URL;
import java.util.Calendar;
import java.util.GregorianCalendar;
import java.util.HashMap;
@@ -39,6 +40,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.MapRedStats;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.history.HiveHistory;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -174,6 +176,19 @@ public class SessionState {
this.conf = conf;
isSilent = conf.getBoolVar(HiveConf.ConfVars.HIVESESSIONSILENT);
ls = new LineageState();
+
+ // Register the Hive builtins jar and all of its functions
+ try {
+ Class<?> pluginClass = Utilities.getBuiltinUtilsClass();
+ URL jarLocation = pluginClass.getProtectionDomain().getCodeSource()
+ .getLocation();
+ add_builtin_resource(
+ ResourceType.JAR, jarLocation.toString());
+ FunctionRegistry.registerFunctionsFromPluginJar(
+ jarLocation, pluginClass.getClassLoader());
+ } catch (Exception ex) {
+ throw new RuntimeException("Failed to load Hive builtin functions", ex);
+ }
}
public void setCmd(String cmdString) {
@@ -509,8 +524,8 @@ public class SessionState {
return null;
}
- private final HashMap<ResourceType, HashSet<String>> resource_map =
- new HashMap<ResourceType, HashSet<String>>();
+ private final HashMap<ResourceType, Set<String>> resource_map =
+ new HashMap<ResourceType, Set<String>>();
public String add_resource(ResourceType t, String value) {
// By default don't convert to unix
@@ -525,23 +540,34 @@ public class SessionState {
return null;
}
- if (resource_map.get(t) == null) {
- resource_map.put(t, new HashSet<String>());
- }
+ Set<String> resourceMap = getResourceMap(t);
String fnlVal = value;
if (t.hook != null) {
- fnlVal = t.hook.preHook(resource_map.get(t), value);
+ fnlVal = t.hook.preHook(resourceMap, value);
if (fnlVal == null) {
return fnlVal;
}
}
getConsole().printInfo("Added resource: " + fnlVal);
- resource_map.get(t).add(fnlVal);
+ resourceMap.add(fnlVal);
return fnlVal;
}
+ public void add_builtin_resource(ResourceType t, String value) {
+ getResourceMap(t).add(value);
+ }
+
+ private Set<String> getResourceMap(ResourceType t) {
+ Set<String> result = resource_map.get(t);
+ if (result == null) {
+ result = new HashSet<String>();
+ resource_map.put(t, result);
+ }
+ return result;
+ }
+
/**
* Returns the list of filesystem schemas as regex which
* are permissible for download as a resource.
Modified: hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out?rev=1209384&r1=1209383&r2=1209384&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out
(original)
+++ hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out Fri Dec
2 08:14:10 2011
@@ -155,6 +155,7 @@ to_utc_timestamp
trim
ucase
unhex
+union_map
unix_timestamp
upper
var_pop
