http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/facts/testfact6_base.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/facts/testfact6_base.xml b/lens-cube/src/test/resources/schema/facts/testfact6_base.xml new file mode 100644 index 0000000..42715e9 --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/testfact6_base.xml @@ -0,0 +1,117 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<x_fact_table name="testfact6_base" cube_name="baseCube" weight="150.0" xmlns="uri:lens:cube:0.1"> + <columns> + <column name="msr13" _type="double" comment="third measure"/> + <column name="msr14" _type="bigint" comment="fourth measure"/> + <column name="d_time" _type="timestamp" comment="event time"/> + <column name="processing_time" _type="timestamp" comment="processing time"/> + <column name="dim1" _type="string" comment="base dim"/> + <column name="booleancut" _type="boolean" comment="expr dim"/> + </columns> + <properties> + <property name="cube.fact.testfact6_base.c3.updateperiods" value="MONTHLY,HOURLY,YEARLY,QUARTERLY,MINUTELY,DAILY"/> + <property name="cube.fact.relative.start.time" value="now.year - 90 days"/> + <property name="cube.fact.testfact6_base.cubename" value="baseCube"/> + <property name="cube.fact.testfact6_base.c1.updateperiods" value="MONTHLY,HOURLY,YEARLY,QUARTERLY,MINUTELY,DAILY"/> + <property name="cube.fact.testfact6_base.c4.updateperiods" value="MONTHLY,HOURLY,YEARLY,QUARTERLY,MINUTELY,DAILY"/> + <property name="cube.fact.testfact6_base.c2.updateperiods" value="MONTHLY,HOURLY,YEARLY,QUARTERLY,MINUTELY,DAILY"/> + <property name="cube.fact.testfact6_base.storages" value="C3,C4,C1,C2"/> + <property name="cube.table.testfact6_base.weight" value="150.0"/> + </properties> + <storage_tables> + <storage_table> + <update_periods> + <update_period>MINUTELY</update_period> + <update_period>HOURLY</update_period> + <update_period>DAILY</update_period> + <update_period>MONTHLY</update_period> + <update_period>QUARTERLY</update_period> + <update_period>YEARLY</update_period> + </update_periods> + <storage_name>C3</storage_name> + <table_desc external="false"> + <part_cols> + <column name="dt" _type="string" comment="date partition"/> + </part_cols> + <table_parameters> + <property name="cube.storagetable.time.partcols" value="dt"/> + </table_parameters> + <serde_parameters> + <property name="serialization.format" value="1"/> + </serde_parameters> + <time_part_cols>dt</time_part_cols> + </table_desc> + </storage_table> + <storage_table> + <update_periods> + <update_period>MINUTELY</update_period> + <update_period>HOURLY</update_period> + <update_period>DAILY</update_period> + <update_period>MONTHLY</update_period> + <update_period>QUARTERLY</update_period> + <update_period>YEARLY</update_period> + </update_periods> + <storage_name>C4</storage_name> + <table_desc external="false"> + <part_cols> + <column name="ttd" _type="string" comment="test date partition"/> + <column name="ttd2" _type="string" comment="test date partition"/> + </part_cols> + <table_parameters> + <property name="cube.storagetable.time.partcols" value="ttd,ttd2"/> + </table_parameters> + <serde_parameters> + <property name="serialization.format" value="1"/> + </serde_parameters> + <time_part_cols>ttd</time_part_cols> + <time_part_cols>ttd2</time_part_cols> + </table_desc> + </storage_table> + <storage_table> + <update_periods> + <update_period>MINUTELY</update_period> + <update_period>HOURLY</update_period> + <update_period>DAILY</update_period> + <update_period>MONTHLY</update_period> + <update_period>QUARTERLY</update_period> + <update_period>YEARLY</update_period> + </update_periods> + <storage_name>C1</storage_name> + <table_desc external="false"> + <part_cols> + <column name="dt" _type="string" comment="date partition"/> + </part_cols> + <table_parameters> + <property name="cube.storagetable.time.partcols" value="dt"/> + </table_parameters> + <serde_parameters> + <property name="serialization.format" value="1"/> + </serde_parameters> + <time_part_cols>dt</time_part_cols> + </table_desc> + </storage_table> + <storage_table> + <update_periods> + <update_period>MINUTELY</update_period> + <update_period>HOURLY</update_period> + <update_period>DAILY</update_period> + <update_period>MONTHLY</update_period> + <update_period>QUARTERLY</update_period> + <update_period>YEARLY</update_period> + </update_periods> + <storage_name>C2</storage_name> + <table_desc external="false"> + <part_cols> + <column name="dt" _type="string" comment="date partition"/> + </part_cols> + <table_parameters> + <property name="cube.storagetable.time.partcols" value="dt"/> + </table_parameters> + <serde_parameters> + <property name="serialization.format" value="1"/> + </serde_parameters> + <time_part_cols>dt</time_part_cols> + </table_desc> + </storage_table> + </storage_tables> +</x_fact_table> \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/facts/testfact_continuous.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/facts/testfact_continuous.xml b/lens-cube/src/test/resources/schema/facts/testfact_continuous.xml new file mode 100644 index 0000000..94fb68a --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/testfact_continuous.xml @@ -0,0 +1,38 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<x_fact_table name="testfact_continuous" cube_name="testCube" weight="100.0" xmlns="uri:lens:cube:0.1"> + <columns> + <column name="msr11" _type="double" comment="third measure"/> + <column name="msr15" _type="int" comment="fifteenth measure"/> + <column name="d_time" _type="timestamp" comment="event time"/> + <column name="processing_time" _type="timestamp" comment="processing time"/> + <column name="dim1" _type="string" comment="base dim"/> + <column name="dim11" _type="string" comment="base dim"/> + <column name="dim12" _type="string" comment="base dim"/> + </columns> + <properties> + <property name="cube.fact.relative.start.time" value="now.year - 90 days"/> + <property name="cube.table.testfact_continuous.weight" value="100.0"/> + <property name="cube.fact.testfact_continuous.storages" value="C0"/> + <property name="cube.fact.absolute.start.time" value="$absolute{now.day-3days}"/> + <property name="cube.fact.testfact_continuous.c0.updateperiods" value="CONTINUOUS"/> + <property name="cube.fact.testfact_continuous.cubename" value="testCube"/> + </properties> + <storage_tables> + <storage_table> + <update_periods> + <update_period>CONTINUOUS</update_period> + </update_periods> + <storage_name>C0</storage_name> + <table_desc external="false"> + <part_cols/> + <table_parameters> + <property name="totalSize" value="0"/> + <property name="numFiles" value="0"/> + </table_parameters> + <serde_parameters> + <property name="serialization.format" value="1"/> + </serde_parameters> + </table_desc> + </storage_table> + </storage_tables> +</x_fact_table> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/facts/testfact_deprecated.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/facts/testfact_deprecated.xml b/lens-cube/src/test/resources/schema/facts/testfact_deprecated.xml new file mode 100644 index 0000000..f14395e --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/testfact_deprecated.xml @@ -0,0 +1,126 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<x_fact_table name="testfact_deprecated" cube_name="baseCube" weight="5.0" xmlns="uri:lens:cube:0.1"> + <columns> + <column name="msr12" _type="float" comment="second measure"/> + <column name="d_time" _type="timestamp" comment="event time"/> + <column name="processing_time" _type="timestamp" comment="processing time"/> + <column name="dim1" _type="string" comment="base dim"/> + <column name="dim11" _type="string" comment="base dim"/> + <column name="dim2" _type="int" comment="dim2 id"/> + <column name="userid" _type="int" comment="user id"/> + <column name="xuserid" _type="int" comment="user id"/> + <column name="yuserid" _type="int" comment="user id"/> + </columns> + <properties> + <property name="cube.fact.testfact_deprecated.c2.updateperiods" + value="MONTHLY,HOURLY,YEARLY,QUARTERLY,MINUTELY,DAILY"/> + <property name="cube.fact.testfact_deprecated.cubename" value="baseCube"/> + <property name="cube.fact.testfact_deprecated.c3.updateperiods" + value="MONTHLY,HOURLY,YEARLY,QUARTERLY,MINUTELY,DAILY"/> + <property name="cube.fact.absolute.start.time" value="$absolute{now.day-3days}"/> + <property name="cube.fact.testfact_deprecated.c4.updateperiods" + value="MONTHLY,HOURLY,YEARLY,QUARTERLY,MINUTELY,DAILY"/> + <property name="cube.fact.absolute.end.time" value="$absolute{now.day-2days}"/> + <property name="cube.fact.relative.start.time" value="now.year - 90 days"/> + <property name="cube.table.testfact_deprecated.weight" value="5.0"/> + <property name="cube.fact.testfact_deprecated.storages" value="C3,C4,C1,C2"/> + <property name="cube.fact.testfact_deprecated.c1.updateperiods" + value="MONTHLY,HOURLY,YEARLY,QUARTERLY,MINUTELY,DAILY"/> + </properties> + <storage_tables> + <storage_table> + <update_periods> + <update_period>MINUTELY</update_period> + <update_period>HOURLY</update_period> + <update_period>DAILY</update_period> + <update_period>MONTHLY</update_period> + <update_period>QUARTERLY</update_period> + <update_period>YEARLY</update_period> + </update_periods> + <storage_name>C3</storage_name> + <table_desc external="false"> + <part_cols> + <column name="dt" _type="string" comment="date partition"/> + </part_cols> + <table_parameters> + <property name="cube.storagetable.time.partcols" value="dt"/> + </table_parameters> + <serde_parameters> + <property name="serialization.format" value="1"/> + </serde_parameters> + <time_part_cols>dt</time_part_cols> + </table_desc> + </storage_table> + <storage_table> + <update_periods> + <update_period>MINUTELY</update_period> + <update_period>HOURLY</update_period> + <update_period>DAILY</update_period> + <update_period>MONTHLY</update_period> + <update_period>QUARTERLY</update_period> + <update_period>YEARLY</update_period> + </update_periods> + <storage_name>C4</storage_name> + <table_desc external="false"> + <part_cols> + <column name="ttd" _type="string" comment="test date partition"/> + <column name="ttd2" _type="string" comment="test date partition"/> + </part_cols> + <table_parameters> + <property name="cube.storagetable.time.partcols" value="ttd,ttd2"/> + </table_parameters> + <serde_parameters> + <property name="serialization.format" value="1"/> + </serde_parameters> + <time_part_cols>ttd</time_part_cols> + <time_part_cols>ttd2</time_part_cols> + </table_desc> + </storage_table> + <storage_table> + <update_periods> + <update_period>MINUTELY</update_period> + <update_period>HOURLY</update_period> + <update_period>DAILY</update_period> + <update_period>MONTHLY</update_period> + <update_period>QUARTERLY</update_period> + <update_period>YEARLY</update_period> + </update_periods> + <storage_name>C1</storage_name> + <table_desc external="false"> + <part_cols> + <column name="dt" _type="string" comment="date partition"/> + </part_cols> + <table_parameters> + <property name="cube.storagetable.time.partcols" value="dt"/> + </table_parameters> + <serde_parameters> + <property name="serialization.format" value="1"/> + </serde_parameters> + <time_part_cols>dt</time_part_cols> + </table_desc> + </storage_table> + <storage_table> + <update_periods> + <update_period>MINUTELY</update_period> + <update_period>HOURLY</update_period> + <update_period>DAILY</update_period> + <update_period>MONTHLY</update_period> + <update_period>QUARTERLY</update_period> + <update_period>YEARLY</update_period> + </update_periods> + <storage_name>C2</storage_name> + <table_desc external="false"> + <part_cols> + <column name="dt" _type="string" comment="date partition"/> + </part_cols> + <table_parameters> + <property name="cube.storagetable.time.partcols" value="dt"/> + </table_parameters> + <serde_parameters> + <property name="serialization.format" value="1"/> + </serde_parameters> + <time_part_cols>dt</time_part_cols> + </table_desc> + </storage_table> + </storage_tables> +</x_fact_table> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/facts/testfactmonthly.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/facts/testfactmonthly.xml b/lens-cube/src/test/resources/schema/facts/testfactmonthly.xml new file mode 100644 index 0000000..8237ba0 --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/testfactmonthly.xml @@ -0,0 +1,46 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<x_fact_table name="testfactmonthly" cube_name="testCube" weight="5.0" xmlns="uri:lens:cube:0.1"> + <columns> + <column name="msr21" _type="float" comment="second measure"/> + <column name="msr4" _type="bigint" comment="fourth measure"/> + <column name="msr15" _type="int" comment="fifteenth measure"/> + <column name="union_join_ctx_msr3" _type="int" comment="union_join_ctx_third measure"/> + <column name="newmeasure" _type="bigint" comment="measure available from now"/> + <column name="union_join_ctx_msr2" _type="int" comment="union_join_ctx_second measure"/> + <column name="msr2" _type="float" comment="second measure"/> + <column name="msr3" _type="double" comment="third measure"/> + <column name="msr22" _type="float" comment="second measure"/> + <column name="msr9" _type="bigint" comment="ninth measure"/> + <column name="msr1" _type="int" comment="first measure"/> + <column name="noaggrmsr" _type="bigint" comment="measure without a default aggregate"/> + <column name="union_join_ctx_msr1" _type="int" comment="union_join_ctx_first measure"/> + <column name="countryid" _type="int" comment="country id"/> + </columns> + <properties> + <property name="cube.fact.relative.start.time" value="now.year - 90 days"/> + <property name="cube.fact.testfactmonthly.storages" value="C2"/> + <property name="cube.fact.testfactmonthly.cubename" value="testCube"/> + <property name="cube.table.testfactmonthly.weight" value="5.0"/> + <property name="cube.fact.testfactmonthly.c2.updateperiods" value="MONTHLY"/> + </properties> + <storage_tables> + <storage_table> + <update_periods> + <update_period>MONTHLY</update_period> + </update_periods> + <storage_name>C2</storage_name> + <table_desc external="false"> + <part_cols> + <column name="dt" _type="string" comment="date partition"/> + </part_cols> + <table_parameters> + <property name="cube.storagetable.time.partcols" value="dt"/> + </table_parameters> + <serde_parameters> + <property name="serialization.format" value="1"/> + </serde_parameters> + <time_part_cols>dt</time_part_cols> + </table_desc> + </storage_table> + </storage_tables> +</x_fact_table> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact1.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact1.xml b/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact1.xml new file mode 100644 index 0000000..d07393d --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact1.xml @@ -0,0 +1,39 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<x_fact_table name="union_join_ctx_fact1" cube_name="baseCube" weight="5.0" xmlns="uri:lens:cube:0.1"> + <columns> + <column name="union_join_ctx_msr1" _type="int" comment="first measure"/> + <column name="d_time" _type="timestamp" comment="event time"/> + <column name="union_join_ctx_zipcode" _type="int" comment="zip"/> + <column name="union_join_ctx_cityid" _type="int" comment="city id"/> + </columns> + <properties> + <property name="cube.fact.relative.start.time" value="now.year - 90 days"/> + <property name="cube.fact.union_join_ctx_fact1.cubename" value="baseCube"/> + <property name="cube.fact.absolute.start.time" value="$absolute{now.day - 90 days}"/> + <property name="cube.fact.union_join_ctx_fact1.storages" value="C1"/> + <property name="cube.table.union_join_ctx_fact1.weight" value="5.0"/> + <property name="cube.fact.absolute.end.time" value="$absolute{now.day - 30 days}"/> + <property name="cube.fact.is.aggregated" value="false"/> + <property name="cube.fact.union_join_ctx_fact1.c1.updateperiods" value="DAILY"/> + </properties> + <storage_tables> + <storage_table> + <update_periods> + <update_period>DAILY</update_period> + </update_periods> + <storage_name>C1</storage_name> + <table_desc external="false"> + <part_cols> + <column name="dt" _type="string" comment="date partition"/> + </part_cols> + <table_parameters> + <property name="cube.storagetable.time.partcols" value="dt"/> + </table_parameters> + <serde_parameters> + <property name="serialization.format" value="1"/> + </serde_parameters> + <time_part_cols>dt</time_part_cols> + </table_desc> + </storage_table> + </storage_tables> +</x_fact_table> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact2.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact2.xml b/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact2.xml new file mode 100644 index 0000000..9145dcc --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact2.xml @@ -0,0 +1,38 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<x_fact_table name="union_join_ctx_fact2" cube_name="baseCube" weight="5.0" xmlns="uri:lens:cube:0.1"> + <columns> + <column name="union_join_ctx_msr1" _type="int" comment="first measure"/> + <column name="d_time" _type="timestamp" comment="event time"/> + <column name="union_join_ctx_zipcode" _type="int" comment="zip"/> + <column name="union_join_ctx_cityid" _type="int" comment="city id"/> + </columns> + <properties> + <property name="cube.fact.union_join_ctx_fact2.storages" value="C1"/> + <property name="cube.fact.absolute.start.time" value="$absolute{now.day - 31 days}"/> + <property name="cube.fact.union_join_ctx_fact2.cubename" value="baseCube"/> + <property name="cube.fact.absolute.end.time" value="$absolute{now.day + 7 days}"/> + <property name="cube.fact.is.aggregated" value="false"/> + <property name="cube.fact.union_join_ctx_fact2.c1.updateperiods" value="DAILY"/> + <property name="cube.table.union_join_ctx_fact2.weight" value="5.0"/> + </properties> + <storage_tables> + <storage_table> + <update_periods> + <update_period>DAILY</update_period> + </update_periods> + <storage_name>C1</storage_name> + <table_desc external="false"> + <part_cols> + <column name="dt" _type="string" comment="date partition"/> + </part_cols> + <table_parameters> + <property name="cube.storagetable.time.partcols" value="dt"/> + </table_parameters> + <serde_parameters> + <property name="serialization.format" value="1"/> + </serde_parameters> + <time_part_cols>dt</time_part_cols> + </table_desc> + </storage_table> + </storage_tables> +</x_fact_table> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact3.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact3.xml b/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact3.xml new file mode 100644 index 0000000..db091b7 --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact3.xml @@ -0,0 +1,38 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<x_fact_table name="union_join_ctx_fact3" cube_name="baseCube" weight="5.0" xmlns="uri:lens:cube:0.1"> + <columns> + <column name="union_join_ctx_msr2" _type="int" comment="second measure"/> + <column name="d_time" _type="timestamp" comment="event time"/> + <column name="union_join_ctx_zipcode" _type="int" comment="zip"/> + <column name="union_join_ctx_cityid" _type="int" comment="city id"/> + </columns> + <properties> + <property name="cube.fact.union_join_ctx_fact3.cubename" value="baseCube"/> + <property name="cube.fact.absolute.start.time" value="$absolute{now.day - 90 days}"/> + <property name="cube.fact.union_join_ctx_fact3.c1.updateperiods" value="DAILY"/> + <property name="cube.fact.absolute.end.time" value="$absolute{now.day + 7 days}"/> + <property name="cube.fact.is.aggregated" value="false"/> + <property name="cube.fact.union_join_ctx_fact3.storages" value="C1"/> + <property name="cube.table.union_join_ctx_fact3.weight" value="5.0"/> + </properties> + <storage_tables> + <storage_table> + <update_periods> + <update_period>DAILY</update_period> + </update_periods> + <storage_name>C1</storage_name> + <table_desc external="false"> + <part_cols> + <column name="dt" _type="string" comment="date partition"/> + </part_cols> + <table_parameters> + <property name="cube.storagetable.time.partcols" value="dt"/> + </table_parameters> + <serde_parameters> + <property name="serialization.format" value="1"/> + </serde_parameters> + <time_part_cols>dt</time_part_cols> + </table_desc> + </storage_table> + </storage_tables> +</x_fact_table> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact5.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact5.xml b/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact5.xml new file mode 100644 index 0000000..e1fbad6 --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact5.xml @@ -0,0 +1,38 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<x_fact_table name="union_join_ctx_fact5" cube_name="baseCube" weight="5.0" xmlns="uri:lens:cube:0.1"> + <columns> + <column name="d_time" _type="timestamp" comment="event time"/> + <column name="union_join_ctx_zipcode" _type="int" comment="zip"/> + <column name="union_join_ctx_cityid" _type="int" comment="city id"/> + <column name="union_join_ctx_msr3" _type="int" comment="third measure"/> + </columns> + <properties> + <property name="cube.fact.union_join_ctx_fact5.c1.updateperiods" value="DAILY"/> + <property name="cube.fact.absolute.start.time" value="$absolute{now.day - 90 days}"/> + <property name="cube.fact.union_join_ctx_fact5.cubename" value="baseCube"/> + <property name="cube.fact.absolute.end.time" value="$absolute{now.day -30 days}"/> + <property name="cube.fact.union_join_ctx_fact5.storages" value="C1"/> + <property name="cube.fact.is.aggregated" value="false"/> + <property name="cube.table.union_join_ctx_fact5.weight" value="5.0"/> + </properties> + <storage_tables> + <storage_table> + <update_periods> + <update_period>DAILY</update_period> + </update_periods> + <storage_name>C1</storage_name> + <table_desc external="false"> + <part_cols> + <column name="dt" _type="string" comment="date partition"/> + </part_cols> + <table_parameters> + <property name="cube.storagetable.time.partcols" value="dt"/> + </table_parameters> + <serde_parameters> + <property name="serialization.format" value="1"/> + </serde_parameters> + <time_part_cols>dt</time_part_cols> + </table_desc> + </storage_table> + </storage_tables> +</x_fact_table> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact6.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact6.xml b/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact6.xml new file mode 100644 index 0000000..0af6a13 --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact6.xml @@ -0,0 +1,38 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<x_fact_table name="union_join_ctx_fact6" cube_name="baseCube" weight="5.0" xmlns="uri:lens:cube:0.1"> + <columns> + <column name="d_time" _type="timestamp" comment="event time"/> + <column name="union_join_ctx_zipcode" _type="int" comment="zip"/> + <column name="union_join_ctx_cityid" _type="int" comment="city id"/> + <column name="union_join_ctx_msr3" _type="int" comment="third measure"/> + </columns> + <properties> + <property name="cube.fact.union_join_ctx_fact6.c1.updateperiods" value="DAILY"/> + <property name="cube.fact.union_join_ctx_fact6.cubename" value="baseCube"/> + <property name="cube.fact.absolute.start.time" value="$absolute{now.day -31 days}"/> + <property name="cube.fact.union_join_ctx_fact6.storages" value="C1"/> + <property name="cube.fact.absolute.end.time" value="$absolute{now.day + 7 days}"/> + <property name="cube.fact.is.aggregated" value="false"/> + <property name="cube.table.union_join_ctx_fact6.weight" value="5.0"/> + </properties> + <storage_tables> + <storage_table> + <update_periods> + <update_period>DAILY</update_period> + </update_periods> + <storage_name>C1</storage_name> + <table_desc external="false"> + <part_cols> + <column name="dt" _type="string" comment="date partition"/> + </part_cols> + <table_parameters> + <property name="cube.storagetable.time.partcols" value="dt"/> + </table_parameters> + <serde_parameters> + <property name="serialization.format" value="1"/> + </serde_parameters> + <time_part_cols>dt</time_part_cols> + </table_desc> + </storage_table> + </storage_tables> +</x_fact_table> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/segmentations/seg1.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/segmentations/seg1.xml b/lens-cube/src/test/resources/schema/segmentations/seg1.xml new file mode 100644 index 0000000..7ed48a1 --- /dev/null +++ b/lens-cube/src/test/resources/schema/segmentations/seg1.xml @@ -0,0 +1,45 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<x_segmentation cube_name="testcube" name="seg1" weight="100.0" xmlns="uri:lens:cube:0.1" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd "> + <properties> + <property name="seg1.prop" value="s1"/> + <property name="cube.segmentation.relative.start.time" value="now -10days"/> + </properties> + <segements> + <segment cube_name="cube11"> + <segment_parameters> + <property name="lens.metastore.cube.column.mapping" value="foo=bar"/> + </segment_parameters> + </segment> + <segment cube_name="cube22"> + <segment_parameters> + <property name="lens.metastore.cube.column.mapping" value="foo1=bar1"/> + </segment_parameters> + </segment> + <segment cube_name="cube33"> + <segment_parameters> + <property name="lens.metastore.cube.column.mapping" value="foo2=bar2"/> + </segment_parameters> + </segment> + </segements> +</x_segmentation> http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/storages/c0.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/storages/c0.xml b/lens-cube/src/test/resources/schema/storages/c0.xml new file mode 100644 index 0000000..de432a1 --- /dev/null +++ b/lens-cube/src/test/resources/schema/storages/c0.xml @@ -0,0 +1,6 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<x_storage name="c0" classname="org.apache.lens.cube.metadata.HDFSStorage" xmlns="uri:lens:cube:0.1"> + <properties> + <property name="cube.table.c0.weight" value="0.0"/> + </properties> +</x_storage> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/storages/c1.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/storages/c1.xml b/lens-cube/src/test/resources/schema/storages/c1.xml new file mode 100644 index 0000000..a0f0886 --- /dev/null +++ b/lens-cube/src/test/resources/schema/storages/c1.xml @@ -0,0 +1,6 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<x_storage name="c1" classname="org.apache.lens.cube.metadata.HDFSStorage" xmlns="uri:lens:cube:0.1"> + <properties> + <property name="cube.table.c1.weight" value="0.0"/> + </properties> +</x_storage> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/storages/c2.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/storages/c2.xml b/lens-cube/src/test/resources/schema/storages/c2.xml new file mode 100644 index 0000000..eb670af --- /dev/null +++ b/lens-cube/src/test/resources/schema/storages/c2.xml @@ -0,0 +1,6 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<x_storage name="c2" classname="org.apache.lens.cube.metadata.HDFSStorage" xmlns="uri:lens:cube:0.1"> + <properties> + <property name="cube.table.c2.weight" value="0.0"/> + </properties> +</x_storage> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/storages/c3.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/storages/c3.xml b/lens-cube/src/test/resources/schema/storages/c3.xml new file mode 100644 index 0000000..4b78cdb --- /dev/null +++ b/lens-cube/src/test/resources/schema/storages/c3.xml @@ -0,0 +1,6 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<x_storage name="c3" classname="org.apache.lens.cube.metadata.HDFSStorage" xmlns="uri:lens:cube:0.1"> + <properties> + <property name="cube.table.c3.weight" value="0.0"/> + </properties> +</x_storage> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/storages/c4.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/storages/c4.xml b/lens-cube/src/test/resources/schema/storages/c4.xml new file mode 100644 index 0000000..9ed2d52 --- /dev/null +++ b/lens-cube/src/test/resources/schema/storages/c4.xml @@ -0,0 +1,6 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<x_storage name="c4" classname="org.apache.lens.cube.metadata.HDFSStorage" xmlns="uri:lens:cube:0.1"> + <properties> + <property name="cube.table.c4.weight" value="0.0"/> + </properties> +</x_storage> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/storages/c5.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/storages/c5.xml b/lens-cube/src/test/resources/schema/storages/c5.xml new file mode 100644 index 0000000..8ebdf3c --- /dev/null +++ b/lens-cube/src/test/resources/schema/storages/c5.xml @@ -0,0 +1,6 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<x_storage name="c5" classname="org.apache.lens.cube.metadata.HDFSStorage" xmlns="uri:lens:cube:0.1"> + <properties> + <property name="cube.table.c5.weight" value="0.0"/> + </properties> +</x_storage> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/storages/c99.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/storages/c99.xml b/lens-cube/src/test/resources/schema/storages/c99.xml new file mode 100644 index 0000000..d87db78 --- /dev/null +++ b/lens-cube/src/test/resources/schema/storages/c99.xml @@ -0,0 +1,6 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<x_storage name="c99" classname="org.apache.lens.cube.metadata.HDFSStorage" xmlns="uri:lens:cube:0.1"> + <properties> + <property name="cube.table.c99.weight" value="0.0"/> + </properties> +</x_storage> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java index 24660e1..abaae5b 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java @@ -18,7 +18,7 @@ */ package org.apache.lens.server.metastore; -import static org.apache.lens.server.metastore.JAXBUtils.*; +import static org.apache.lens.cube.metadata.JAXBUtils.*; import java.util.*; import java.util.Date; @@ -168,10 +168,7 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet @Override public void createCube(LensSessionHandle sessionid, XCube cube) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - CubeMetastoreClient msClient = getClient(sessionid); - Cube parent = cube instanceof XDerivedCube ? (Cube) msClient.getCube( - ((XDerivedCube) cube).getParent()) : null; - msClient.createCube(JAXBUtils.hiveCubeFromXCube(cube, parent)); + getClient(sessionid).createCube(cube); log.info("Created cube " + cube.getName()); } } @@ -200,7 +197,7 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet * @param cubeName cube name */ public void dropCube(LensSessionHandle sessionid, String cubeName) throws LensException { - try(SessionContext ignored = new SessionContext(sessionid)) { + try (SessionContext ignored = new SessionContext(sessionid)) { getClient(sessionid).dropCube(cubeName); } } @@ -214,10 +211,7 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet @Override public void updateCube(LensSessionHandle sessionid, XCube cube) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - CubeMetastoreClient msClient = getClient(sessionid); - Cube parent = cube instanceof XDerivedCube ? (Cube) msClient.getCube( - ((XDerivedCube) cube).getParent()) : null; - msClient.alterCube(cube.getName(), JAXBUtils.hiveCubeFromXCube(cube, parent)); + getClient(sessionid).alterCube(cube); log.info("Cube updated " + cube.getName()); } catch (HiveException e) { throw new LensException(e); @@ -233,23 +227,9 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet @Override public void createDimensionTable(LensSessionHandle sessionid, XDimensionTable xDimTable) throws LensException { String dimTblName = xDimTable.getTableName(); - List<FieldSchema> columns = JAXBUtils.fieldSchemaListFromColumns(xDimTable.getColumns()); - Map<String, UpdatePeriod> updatePeriodMap = - JAXBUtils.dumpPeriodsFromStorageTables(xDimTable.getStorageTables()); - - Map<String, String> properties = JAXBUtils.mapFromXProperties(xDimTable.getProperties()); - Map<String, StorageTableDesc> storageDesc = JAXBUtils.tableDescPrefixMapFromXStorageTables( - xDimTable.getStorageTables()); try (SessionContext ignored = new SessionContext(sessionid)){ - log.info("# Columns: " + columns); - getClient(sessionid).createCubeDimensionTable(xDimTable.getDimensionName(), - dimTblName, - columns, - xDimTable.getWeight(), - updatePeriodMap, - properties, - storageDesc); + getClient(sessionid).createCubeDimensionTable(xDimTable); log.info("Dimension Table created " + xDimTable.getTableName()); } } @@ -265,31 +245,14 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet @Override public XDimensionTable getDimensionTable(LensSessionHandle sessionid, String dimTblName) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - CubeMetastoreClient msClient = getClient(sessionid); - CubeDimensionTable dimTable = msClient.getDimensionTable(dimTblName); - XDimensionTable dt = JAXBUtils.dimTableFromCubeDimTable(dimTable); - if (dimTable.getStorages() != null && !dimTable.getStorages().isEmpty()) { - for (String storageName : dimTable.getStorages()) { - XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( - msClient.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(dimTblName, storageName))); - tblElement.setStorageName(storageName); - UpdatePeriod p = dimTable.getSnapshotDumpPeriods().get(storageName); - if (p != null) { - tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); - } - dt.getStorageTables().getStorageTable().add(tblElement); - } - } - return dt; + return getClient(sessionid).getXDimensionTable(dimTblName); } } @Override public void updateDimensionTable(LensSessionHandle sessionid, XDimensionTable dimensionTable) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - getClient(sessionid).alterCubeDimensionTable(dimensionTable.getTableName(), - JAXBUtils.cubeDimTableFromDimTable(dimensionTable), - JAXBUtils.tableDescPrefixMapFromXStorageTables(dimensionTable.getStorageTables())); + getClient(sessionid).alterCubeDimensionTable(dimensionTable); log.info("Updated dimension table " + dimensionTable.getTableName()); } catch (HiveException exc) { throw new LensException(exc); @@ -395,43 +358,7 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet @Override public XFactTable getFactTable(LensSessionHandle sessionid, String fact) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - CubeMetastoreClient msClient = getClient(sessionid); - CubeFactTable cft = msClient.getFactTable(fact); - XFactTable factTable = JAXBUtils.factTableFromCubeFactTable(cft); - Map<String, Map<UpdatePeriod, String>> storageMap = cft.getStoragePrefixUpdatePeriodMap(); - for (String storageName : cft.getStorages()) { - Set<UpdatePeriod> updatePeriods = cft.getUpdatePeriods().get(storageName); - // This map tells if there are different tables for different update period. - Map<UpdatePeriod, String> updatePeriodToTableMap = storageMap.get(storageName); - Set<String> tableNames = new HashSet<>(); - for (UpdatePeriod updatePeriod : updatePeriods) { - tableNames.add(updatePeriodToTableMap.get(updatePeriod)); - } - if (tableNames.size() <= 1) { - XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( - msClient.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(fact, storageName))); - tblElement.setStorageName(storageName); - for (UpdatePeriod p : updatePeriods) { - tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); - } - factTable.getStorageTables().getStorageTable().add(tblElement); - } else { - // Multiple storage tables. - XStorageTableElement tblElement = new XStorageTableElement(); - tblElement.setStorageName(storageName); - XUpdatePeriods xUpdatePeriods = new XUpdatePeriods(); - tblElement.setUpdatePeriods(xUpdatePeriods); - for (Map.Entry entry : updatePeriodToTableMap.entrySet()) { - XUpdatePeriodTableDescriptor updatePeriodTableDescriptor = new XUpdatePeriodTableDescriptor(); - updatePeriodTableDescriptor.setTableDesc(getStorageTableDescFromHiveTable( - msClient.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(fact, (String) entry.getValue())))); - updatePeriodTableDescriptor.setUpdatePeriod(XUpdatePeriod.valueOf(((UpdatePeriod)entry.getKey()).name())); - xUpdatePeriods.getUpdatePeriodTableDescriptor().add(updatePeriodTableDescriptor); - } - factTable.getStorageTables().getStorageTable().add(tblElement); - } - } - return factTable; + return getClient(sessionid).getXFactTable(fact); } } @@ -448,54 +375,34 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet @Override public void createFactTable(LensSessionHandle sessionid, XFactTable fact) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - getClient(sessionid).createCubeFactTable(fact.getCubeName(), - fact.getName(), - JAXBUtils.fieldSchemaListFromColumns(fact.getColumns()), - JAXBUtils.getFactUpdatePeriodsFromStorageTables(fact.getStorageTables()), - fact.getWeight(), - addFactColStartTimePropertyToFactProperties(fact), - JAXBUtils.tableDescPrefixMapFromXStorageTables(fact.getStorageTables()), - JAXBUtils.storageTablePrefixMapOfStorage(fact.getStorageTables())); + getClient(sessionid).createCubeFactTable(fact); log.info("Created fact table " + fact.getName()); } } - public Map<String, String> addFactColStartTimePropertyToFactProperties(XFactTable fact) { - Map<String, String> props = new HashMap<String, String>(); - props.putAll(JAXBUtils.mapFromXProperties(fact.getProperties())); - props.putAll(JAXBUtils.columnStartAndEndTimeFromXColumns(fact.getColumns())); - return props; - } @Override - public void createSegmentation(LensSessionHandle sessionid, XSegmentation cubeSeg) throws LensException { + public void updateFactTable(LensSessionHandle sessionid, XFactTable fact) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - getClient(sessionid).createSegmentation( - cubeSeg.getCubeName(), - cubeSeg.getName(), - JAXBUtils.segmentsFromXSegments(cubeSeg.getSegements()), - cubeSeg.getWeight(), - JAXBUtils.mapFromXProperties(cubeSeg.getProperties())); - log.info("Created segmentation " + cubeSeg.getName()); + getClient(sessionid).alterCubeFactTable(fact); + log.info("Updated fact table " + fact.getName()); + } catch (HiveException e) { + throw new LensException(e); } } @Override - public void updateFactTable(LensSessionHandle sessionid, XFactTable fact) throws LensException { + public void createSegmentation(LensSessionHandle sessionid, XSegmentation cubeSeg) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - getClient(sessionid).alterCubeFactTable(fact.getName(), JAXBUtils.cubeFactFromFactTable(fact), - JAXBUtils.tableDescPrefixMapFromXStorageTables(fact.getStorageTables()), - JAXBUtils.columnStartAndEndTimeFromXColumns(fact.getColumns())); - log.info("Updated fact table " + fact.getName()); - } catch (HiveException e) { - throw new LensException(e); + getClient(sessionid).createSegmentation(cubeSeg); + log.info("Created segmentation " + cubeSeg.getName()); } } @Override public void updateSegmentation(LensSessionHandle sessionid, XSegmentation cubeSeg) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - getClient(sessionid).alterSegmentation(cubeSeg.getName(), segmentationFromXSegmentation(cubeSeg)); + getClient(sessionid).alterSegmentation(cubeSeg); log.info("Updated segmentation " + cubeSeg.getName()); } catch (HiveException e) { throw new LensException(e); @@ -906,7 +813,7 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet public void createStorage(LensSessionHandle sessionid, XStorage storage) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - getClient(sessionid).createStorage(JAXBUtils.storageFromXStorage(storage)); + getClient(sessionid).createStorage(storage); log.info("Created storage " + storage.getName()); } @@ -925,8 +832,7 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet public void alterStorage(LensSessionHandle sessionid, String storageName, XStorage storage) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - getClient(sessionid).alterStorage(storageName, - JAXBUtils.storageFromXStorage(storage)); + getClient(sessionid).alterStorage(storage); log.info("Altered storage " + storageName); } catch (HiveException e) { throw new LensException(e); @@ -1015,7 +921,7 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet public void createDimension(LensSessionHandle sessionid, XDimension dimension) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - getClient(sessionid).createDimension(JAXBUtils.dimensionFromXDimension(dimension)); + getClient(sessionid).createDimension(dimension); log.info("Created dimension " + dimension.getName()); } } @@ -1041,8 +947,7 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet public void updateDimension(LensSessionHandle sessionid, String dimName, XDimension dimension) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - getClient(sessionid).alterDimension(dimName, - JAXBUtils.dimensionFromXDimension(dimension)); + getClient(sessionid).alterDimension(dimension); log.info("Altered dimension " + dimName); } catch (HiveException e) { throw new LensException(e);
