add ddl definitions
Project: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/commit/be1ea37a Tree: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/tree/be1ea37a Diff: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/diff/be1ea37a Branch: refs/heads/JIRA-22/pr-385 Commit: be1ea37a0f5048cde4284107c04e109f0f526b42 Parents: ad81b3a Author: amaya <[email protected]> Authored: Tue Sep 20 18:00:49 2016 +0900 Committer: amaya <[email protected]> Committed: Tue Sep 20 18:38:01 2016 +0900 ---------------------------------------------------------------------- resources/ddl/define-all-as-permanent.hive | 20 ++++++++++++++++++++ resources/ddl/define-all.hive | 20 ++++++++++++++++++++ resources/ddl/define-all.spark | 20 ++++++++++++++++++++ resources/ddl/define-udfs.td.hql | 4 ++++ 4 files changed, 64 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/be1ea37a/resources/ddl/define-all-as-permanent.hive ---------------------------------------------------------------------- diff --git a/resources/ddl/define-all-as-permanent.hive b/resources/ddl/define-all-as-permanent.hive index bab5a29..52b73a0 100644 --- a/resources/ddl/define-all-as-permanent.hive +++ b/resources/ddl/define-all-as-permanent.hive @@ -202,6 +202,13 @@ CREATE FUNCTION zscore as 'hivemall.ftvec.scaling.ZScoreUDF' USING JAR '${hivema DROP FUNCTION IF EXISTS l2_normalize; CREATE FUNCTION l2_normalize as 'hivemall.ftvec.scaling.L2NormalizationUDF' USING JAR '${hivemall_jar}'; +------------------------- +-- selection functions -- +------------------------- + +DROP FUNCTION IF EXISTS chi_square; +CREATE FUNCTION chi_square as 'hivemall.ftvec.selection.ChiSquareUDF' USING JAR '${hivemall_jar}'; + -------------------- -- misc functions -- -------------------- @@ -364,6 +371,9 @@ CREATE FUNCTION subarray_endwith as 'hivemall.tools.array.SubarrayEndWithUDF' US DROP FUNCTION IF EXISTS subarray_startwith; CREATE FUNCTION subarray_startwith as 'hivemall.tools.array.SubarrayStartWithUDF' USING JAR '${hivemall_jar}'; +DROP FUNCTION IF EXISTS subarray_by_indices; +CREATE FUNCTION subarray_by_indices as 'hivemall.tools.array.SubarrayByIndicesUDF' USING JAR '${hivemall_jar}'; + DROP FUNCTION IF EXISTS array_concat; CREATE FUNCTION array_concat as 'hivemall.tools.array.ArrayConcatUDF' USING JAR '${hivemall_jar}'; @@ -380,6 +390,9 @@ CREATE FUNCTION array_avg as 'hivemall.tools.array.ArrayAvgGenericUDAF' USING JA DROP FUNCTION IF EXISTS array_sum; CREATE FUNCTION array_sum as 'hivemall.tools.array.ArraySumUDAF' USING JAR '${hivemall_jar}'; +DROP FUNCTION array_top_k_indices; +CREATE FUNCTION array_top_k_indices as 'hivemall.tools.array.ArrayTopKIndicesUDF' USING JAR '${hivemall_jar}'; + DROP FUNCTION IF EXISTS to_string_array; CREATE FUNCTION to_string_array as 'hivemall.tools.array.ToStringArrayUDF' USING JAR '${hivemall_jar}'; @@ -436,6 +449,13 @@ DROP FUNCTION IF EXISTS sigmoid; CREATE FUNCTION sigmoid as 'hivemall.tools.math.SigmoidGenericUDF' USING JAR '${hivemall_jar}'; ---------------------- +-- Matrix functions -- +---------------------- + +DROP FUNCTION IF EXISTS transpose_and_dot; +CREATE FUNCTION transpose_and_dot as 'hivemall.tools.matrix.TransposeAndDotUDAF' USING JAR '${hivemall_jar}'; + +---------------------- -- mapred functions -- ---------------------- http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/be1ea37a/resources/ddl/define-all.hive ---------------------------------------------------------------------- diff --git a/resources/ddl/define-all.hive b/resources/ddl/define-all.hive index 315b4d2..a70ae0f 100644 --- a/resources/ddl/define-all.hive +++ b/resources/ddl/define-all.hive @@ -198,6 +198,13 @@ create temporary function zscore as 'hivemall.ftvec.scaling.ZScoreUDF'; drop temporary function l2_normalize; create temporary function l2_normalize as 'hivemall.ftvec.scaling.L2NormalizationUDF'; +------------------------- +-- selection functions -- +------------------------- + +drop temporary function chi_square; +create temporary function chi_square as 'hivemall.ftvec.selection.ChiSquareUDF'; + ----------------------------------- -- Feature engineering functions -- ----------------------------------- @@ -360,6 +367,9 @@ create temporary function subarray_endwith as 'hivemall.tools.array.SubarrayEndW drop temporary function subarray_startwith; create temporary function subarray_startwith as 'hivemall.tools.array.SubarrayStartWithUDF'; +drop temporary function subarray_by_indices; +create temporary function subarray_by_indices as 'hivemall.tools.array.SubarrayByIndicesUDF'; + drop temporary function array_concat; create temporary function array_concat as 'hivemall.tools.array.ArrayConcatUDF'; @@ -376,6 +386,9 @@ create temporary function array_avg as 'hivemall.tools.array.ArrayAvgGenericUDAF drop temporary function array_sum; create temporary function array_sum as 'hivemall.tools.array.ArraySumUDAF'; +drop temporary function array_top_k_indices; +create temporary function array_top_k_indices as 'hivemall.tools.array.ArrayTopKIndicesUDF'; + drop temporary function to_string_array; create temporary function to_string_array as 'hivemall.tools.array.ToStringArrayUDF'; @@ -432,6 +445,13 @@ drop temporary function sigmoid; create temporary function sigmoid as 'hivemall.tools.math.SigmoidGenericUDF'; ---------------------- +-- Matrix functions -- +---------------------- + +drop temporary function transpose_and_dot; +create temporary function transpose_and_dot as 'hivemall.tools.matrix.TransposeAndDotUDAF'; + +---------------------- -- mapred functions -- ---------------------- http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/be1ea37a/resources/ddl/define-all.spark ---------------------------------------------------------------------- diff --git a/resources/ddl/define-all.spark b/resources/ddl/define-all.spark index 4aed65b..e009511 100644 --- a/resources/ddl/define-all.spark +++ b/resources/ddl/define-all.spark @@ -184,6 +184,13 @@ sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS normalize") sqlContext.sql("CREATE TEMPORARY FUNCTION normalize AS 'hivemall.ftvec.scaling.L2NormalizationUDF'") /** + * selection functions + */ + +sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS chi_square") +sqlContext.sql("CREATE TEMPORARY FUNCTION chi_square AS 'hivemall.ftvec.selection.ChiSquareUDF'") + +/** * misc functions */ @@ -309,6 +316,9 @@ sqlContext.sql("CREATE TEMPORARY FUNCTION subarray_endwith AS 'hivemall.tools.ar sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS subarray_startwith") sqlContext.sql("CREATE TEMPORARY FUNCTION subarray_startwith AS 'hivemall.tools.array.SubarrayStartWithUDF'") +sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS subarray_by_indices") +sqlContext.sql("CREATE TEMPORARY FUNCTION subarray_by_indices AS 'hivemall.tools.array.SubarrayByIndicesUDF'") + sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS collect_all") sqlContext.sql("CREATE TEMPORARY FUNCTION collect_all AS 'hivemall.tools.array.CollectAllUDAF'") @@ -321,6 +331,9 @@ sqlContext.sql("CREATE TEMPORARY FUNCTION subarray AS 'hivemall.tools.array.Suba sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS array_avg") sqlContext.sql("CREATE TEMPORARY FUNCTION array_avg AS 'hivemall.tools.array.ArrayAvgGenericUDAF'") +sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS array_top_k_indices") +sqlContext.sql("CREATE TEMPORARY FUNCTION array_top_k_indices AS 'hivemall.tools.array.ArrayTopKIndicesUDF'") + /** * compression functions */ @@ -355,6 +368,13 @@ sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS sigmoid") sqlContext.sql("CREATE TEMPORARY FUNCTION sigmoid AS 'hivemall.tools.math.SigmoidGenericUDF'") /** + * Matrix functions + */ + +sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS transpose_and_dot") +sqlContext.sql("CREATE TEMPORARY FUNCTION transpose_and_dot AS 'hivemall.tools.matrix.TransposeAndDotUDAF'") + +/** * mapred functions */ http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/be1ea37a/resources/ddl/define-udfs.td.hql ---------------------------------------------------------------------- diff --git a/resources/ddl/define-udfs.td.hql b/resources/ddl/define-udfs.td.hql index 18500aa..92e4003 100644 --- a/resources/ddl/define-udfs.td.hql +++ b/resources/ddl/define-udfs.td.hql @@ -50,6 +50,7 @@ create temporary function powered_features as 'hivemall.ftvec.pairing.PoweredFea create temporary function rescale as 'hivemall.ftvec.scaling.RescaleUDF'; create temporary function zscore as 'hivemall.ftvec.scaling.ZScoreUDF'; create temporary function l2_normalize as 'hivemall.ftvec.scaling.L2NormalizationUDF'; +create temporary function chi_square as 'hivemall.ftvec.selection.ChiSquareUDF'; create temporary function amplify as 'hivemall.ftvec.amplify.AmplifierUDTF'; create temporary function rand_amplify as 'hivemall.ftvec.amplify.RandomAmplifierUDTF'; create temporary function add_bias as 'hivemall.ftvec.AddBiasUDF'; @@ -94,10 +95,12 @@ create temporary function array_remove as 'hivemall.tools.array.ArrayRemoveUDF'; create temporary function sort_and_uniq_array as 'hivemall.tools.array.SortAndUniqArrayUDF'; create temporary function subarray_endwith as 'hivemall.tools.array.SubarrayEndWithUDF'; create temporary function subarray_startwith as 'hivemall.tools.array.SubarrayStartWithUDF'; +create temporary function subarray_by_indices as 'hivemall.tools.array.SubarrayByIndicesUDF'; create temporary function array_concat as 'hivemall.tools.array.ArrayConcatUDF'; create temporary function subarray as 'hivemall.tools.array.SubarrayUDF'; create temporary function array_avg as 'hivemall.tools.array.ArrayAvgGenericUDAF'; create temporary function array_sum as 'hivemall.tools.array.ArraySumUDAF'; +create temporary function array_top_k_indices as 'hivemall.tools.array.ArrayTopKIndicesUDF'; create temporary function to_string_array as 'hivemall.tools.array.ToStringArrayUDF'; create temporary function array_intersect as 'hivemall.tools.array.ArrayIntersectUDF'; create temporary function bits_collect as 'hivemall.tools.bits.BitsCollectUDAF'; @@ -111,6 +114,7 @@ create temporary function map_tail_n as 'hivemall.tools.map.MapTailNUDF'; create temporary function to_map as 'hivemall.tools.map.UDAFToMap'; create temporary function to_ordered_map as 'hivemall.tools.map.UDAFToOrderedMap'; create temporary function sigmoid as 'hivemall.tools.math.SigmoidGenericUDF'; +create temporary function transpose_and_dot as 'hivemall.tools.matrix.TransposeAndDotUDAF'; create temporary function taskid as 'hivemall.tools.mapred.TaskIdUDF'; create temporary function jobid as 'hivemall.tools.mapred.JobIdUDF'; create temporary function rowid as 'hivemall.tools.mapred.RowIdUDF';
