LENS-484 : NPE when weight is not set in XFactTable object while creating fact table (Yash Sharma via Raghavendra Singh)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/1584f001 Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/1584f001 Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/1584f001 Branch: refs/heads/current-release-line Commit: 1584f001fa3a3e5bcbf8483bd1350587e3fff653 Parents: 6a4b247 Author: Yash Sharma <[email protected]> Authored: Tue Jun 2 15:32:22 2015 +0530 Committer: Raghavendra Singh <[email protected]> Committed: Tue Jun 2 15:32:22 2015 +0530 ---------------------------------------------------------------------- lens-api/src/main/resources/cube-0.1.xsd | 14 +- .../apache/lens/cli/TestLensFactCommands.java | 2 + .../TestLensFactCommandsWithMissingWeight.java | 150 +++++++++++++++++++ .../resources/cube_with_no_weight_facts.xml | 47 ++++++ .../resources/dim-local-storage-element.xml | 7 +- lens-cli/src/test/resources/dim_table.xml | 8 +- lens-cli/src/test/resources/dim_table2.xml | 6 +- .../resources/fact-local-storage-element.xml | 8 +- .../src/test/resources/fact_without_weight.xml | 50 +++++++ lens-cli/src/test/resources/sample-cube.xml | 20 +-- lens-cli/src/test/resources/test-dimension.xml | 4 +- .../apache/lens/client/LensMetadataClient.java | 17 ++- .../src/main/resources/city_subset.xml | 2 +- .../src/main/resources/customer_table.xml | 2 +- lens-examples/src/main/resources/dim_table.xml | 2 +- lens-examples/src/main/resources/dim_table2.xml | 2 +- lens-examples/src/main/resources/dim_table3.xml | 2 +- lens-examples/src/main/resources/dim_table4.xml | 2 +- .../src/main/resources/product_db_table.xml | 2 +- .../src/main/resources/product_table.xml | 2 +- .../src/main/resources/sales-aggr-fact1.xml | 6 +- .../src/main/resources/sales-aggr-fact2.xml | 6 +- lens-examples/src/main/resources/sales-cube.xml | 40 ++--- .../src/main/resources/sample-cube.xml | 18 +-- 24 files changed, 346 insertions(+), 73 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-api/src/main/resources/cube-0.1.xsd ---------------------------------------------------------------------- diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd index 1e8d71d..719d10d 100644 --- a/lens-api/src/main/resources/cube-0.1.xsd +++ b/lens-api/src/main/resources/cube-0.1.xsd @@ -626,13 +626,18 @@ </xs:documentation> </xs:annotation> </xs:attribute> - <xs:attribute name="weight" type="xs:double"> + <xs:attribute name="weight" use="required" > <xs:annotation> <xs:documentation> The weight of the dimension table. LENS will use this attribute to determine the lightest table when there are more than one eligible table for answering a query. </xs:documentation> </xs:annotation> + <xs:simpleType> + <xs:restriction base="xs:double"> + <xs:minExclusive value="0"></xs:minExclusive> + </xs:restriction> + </xs:simpleType> </xs:attribute> </xs:complexType> @@ -1157,13 +1162,18 @@ </xs:documentation> </xs:annotation> </xs:attribute> - <xs:attribute name="weight" type="xs:double"> + <xs:attribute name="weight" use="required" > <xs:annotation> <xs:documentation> The weight of the fact table. LENS will use this attribute to decide the lightest table to query when there are more than one eligible tables. </xs:documentation> </xs:annotation> + <xs:simpleType> + <xs:restriction base="xs:double"> + <xs:minExclusive value="0"></xs:minExclusive> + </xs:restriction> + </xs:simpleType> </xs:attribute> </xs:complexType> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java index 1ad234d..13bfbd2 100644 --- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java +++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java @@ -69,6 +69,8 @@ public class TestLensFactCommands extends LensCliApplicationTest { String cubeList = getCubeCommand().showCubes(); assertFalse(cubeList.contains("sample_cube"), cubeList); getCubeCommand().createCube(new File(cubeSpec.toURI()).getAbsolutePath()); + cubeList = getCubeCommand().showCubes(); + assertTrue(cubeList.contains("sample_cube"), cubeList); } private void dropSampleCube() { http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommandsWithMissingWeight.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommandsWithMissingWeight.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommandsWithMissingWeight.java new file mode 100644 index 0000000..73f3a78 --- /dev/null +++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommandsWithMissingWeight.java @@ -0,0 +1,150 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cli; + +import java.io.*; + +import java.net.URISyntaxException; +import java.net.URL; + +import org.apache.lens.cli.commands.LensCubeCommands; +import org.apache.lens.cli.commands.LensFactCommands; +import org.apache.lens.client.LensClient; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.testng.Assert; +import org.testng.annotations.Test; + + +/** + * The Class TestLensFactCommands. + */ +public class TestLensFactCommandsWithMissingWeight extends LensCliApplicationTest { + + /** The Constant LOG. */ + private static final Logger LOG = LoggerFactory.getLogger(TestLensFactCommandsWithMissingWeight.class); + + /** The Constant FACT_LOCAL. */ + public static final String FACT_LOCAL = "fact_local_without_wt"; + + /* The Constant for cube name */ + public static final String CUBE_NAME = "cube_with_no_weight_facts"; + + /* The Constant for fact name */ + public static final String FACT_NAME = "fact_without_wt"; + + /* The File name with cube details */ + public static final String CUBE_XML_FILE = "cube_with_no_weight_facts.xml"; + + /* The File name with fact details */ + public static final String FACT_XML_FILE = "fact_without_weight.xml"; + + /** The command. */ + private static LensFactCommands command = null; + private static LensCubeCommands cubeCommands = null; + + /** + * Test fact commands. + * + * @throws java.io.IOException + */ + @Test + public void testFactCommands() throws IOException, URISyntaxException { + dropSampleCubeIfExists(); + dropFactIfExists(); + + createSampleCube(); + addFactTable(); + dropSampleCube(); + } + + private void createSampleCube() throws URISyntaxException { + URL cubeSpec = TestLensCubeCommands.class.getClassLoader().getResource(CUBE_XML_FILE); + String cubeList = getCubeCommand().showCubes(); + Assert.assertFalse(cubeList.contains(CUBE_NAME)); + getCubeCommand().createCube(new File(cubeSpec.toURI()).getAbsolutePath()); + cubeList = getCubeCommand().showCubes(); + Assert.assertTrue(cubeList.contains(CUBE_NAME)); + } + + private void dropSampleCube() { + getCubeCommand().dropCube(CUBE_NAME); + TestLensStorageCommands.dropStorage(FACT_LOCAL); + } + + private static LensFactCommands getCommand() { + if (command == null) { + LensClient client = new LensClient(); + command = new LensFactCommands(); + command.setClient(client); + } + return command; + } + + private static LensCubeCommands getCubeCommand() { + if (cubeCommands == null) { + LensClient client = new LensClient(); + cubeCommands = new LensCubeCommands(); + cubeCommands.setClient(client); + } + return cubeCommands; + } + + /** + * Adds the fact_without_wt table. + * + * @throws java.io.IOException + */ + public static void addFactTable() throws IOException, URISyntaxException { + LensFactCommands command = getCommand(); + String factList = command.showFacts(null); + Assert.assertEquals(command.showFacts(CUBE_NAME), "No fact found for " + CUBE_NAME); + Assert.assertEquals("No fact found", factList, "Fact tables should not be found."); + // add local storage before adding fact table + TestLensStorageCommands.addLocalStorage(FACT_LOCAL); + URL factSpec = TestLensFactCommandsWithMissingWeight.class.getClassLoader().getResource(FACT_XML_FILE); + String response = null; + response = command.createFact(new File(factSpec.toURI()).getAbsolutePath()); + + Assert.assertEquals(response, "failed", "Fact table creation should not be successful."); + Assert.assertEquals(command.showFacts(CUBE_NAME), "No fact found for " + CUBE_NAME, + "Fact tables should not be found."); + } + + /** + * Drop fact_without_wt table if exixsts. + */ + public static void dropFactIfExists() { + LensFactCommands command = getCommand(); + String factList = command.showFacts(null); + if (factList.contains(FACT_NAME)) { + command.dropFact(FACT_NAME, false); + TestLensStorageCommands.dropStorage(FACT_LOCAL); + } + } + + private void dropSampleCubeIfExists() { + String cubeList = getCubeCommand().showCubes(); + if (cubeList.contains(CUBE_NAME)) { + getCubeCommand().dropCube(CUBE_NAME); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-cli/src/test/resources/cube_with_no_weight_facts.xml ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/resources/cube_with_no_weight_facts.xml b/lens-cli/src/test/resources/cube_with_no_weight_facts.xml new file mode 100644 index 0000000..263ca88 --- /dev/null +++ b/lens-cli/src/test/resources/cube_with_no_weight_facts.xml @@ -0,0 +1,47 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<x_base_cube name="cube_with_no_weight_facts" xmlns="uri:lens:cube:0.1" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd "> + <measures> + <measure name="measure1" type="BIGINT" /> + <measure name="measure2" type="INT" default_aggr="SUM" /> + <measure name="measure3" type="FLOAT" default_aggr="MAX" start_time='2013-12-12T00:00:00' /> + <measure name="measure4" type="DOUBLE" default_aggr="MIN" /> + </measures> + <dim_attributes> + <dim_attribute name="dim1" type="INT" /> + <dim_attribute name="dim2" type="INT" start_time='2013-12-01T00:00:00' /> + <dim_attribute name="dim3" type="INT"> + <ref_spec> + <table_references> + <table_reference table="dim_table" column="id" /> + </table_references> + </ref_spec> + </dim_attribute> + </dim_attributes> + <expressions> + <expression name="expr_msr5" type="DOUBLE"> + <expr_spec expr = "measure3 + measure4" end_time='2013-12-12T00:00:00'/> + <expr_spec expr = "measure3 + measure4 + 0.01" start_time='2013-12-12T00:00:00'/> + </expression> + </expressions> +</x_base_cube> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-cli/src/test/resources/dim-local-storage-element.xml ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/resources/dim-local-storage-element.xml b/lens-cli/src/test/resources/dim-local-storage-element.xml index 5a19cfa..7a58aa3 100644 --- a/lens-cli/src/test/resources/dim-local-storage-element.xml +++ b/lens-cli/src/test/resources/dim-local-storage-element.xml @@ -21,15 +21,14 @@ --> <x_storage_table_element xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd "> + <update_periods> + <update_period>DAILY</update_period> + </update_periods> <storage_name>dim_local</storage_name> - <table_desc external="true" field_delimiter="," table_location="/tmp/examples/local"> <part_cols> <column comment="Time column" name="dt" type="STRING" /> </part_cols> <time_part_cols>dt</time_part_cols> </table_desc> - <update_periods> - <update_period>DAILY</update_period> - </update_periods> </x_storage_table_element> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-cli/src/test/resources/dim_table.xml ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/resources/dim_table.xml b/lens-cli/src/test/resources/dim_table.xml index 932c975..96b2244 100644 --- a/lens-cli/src/test/resources/dim_table.xml +++ b/lens-cli/src/test/resources/dim_table.xml @@ -19,7 +19,7 @@ under the License. --> -<x_dimension_table dimension_name="test_dim" table_name="dim_table" weight="0.0" xmlns="uri:lens:cube:0.1" +<x_dimension_table dimension_name="test_dim" table_name="dim_table" weight="100.0" xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd "> <columns> <column comment="ID" name="id" type="INT" /> @@ -32,6 +32,9 @@ </properties> <storage_tables> <storage_table> + <update_periods> + <update_period>HOURLY</update_period> + </update_periods> <storage_name>local</storage_name> <table_desc external="true" field_delimiter="," table_location="/tmp/examples/local"> <part_cols> @@ -39,9 +42,6 @@ </part_cols> <time_part_cols>dt</time_part_cols> </table_desc> - <update_periods> - <update_period>HOURLY</update_period> - </update_periods> </storage_table> </storage_tables> </x_dimension_table> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-cli/src/test/resources/dim_table2.xml ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/resources/dim_table2.xml b/lens-cli/src/test/resources/dim_table2.xml index 1d6e138..000f061 100644 --- a/lens-cli/src/test/resources/dim_table2.xml +++ b/lens-cli/src/test/resources/dim_table2.xml @@ -19,7 +19,7 @@ under the License. --> -<x_dimension_table dimension_name="test_dim" table_name="dim_table2" weight="0.0" xmlns="uri:lens:cube:0.1" +<x_dimension_table dimension_name="test_dim" table_name="dim_table2" weight="100.0" xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd "> <columns> <column comment="ID" name="id" type="INT" /> @@ -32,7 +32,9 @@ </properties> <storage_tables> <storage_table> - <update_periods>HOURLY</update_periods> + <update_periods> + <update_period>HOURLY</update_period> + </update_periods> <storage_name>dim_local</storage_name> <table_desc external="true" field_delimiter="," table_location="/tmp/examples/dim1"> <part_cols> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-cli/src/test/resources/fact-local-storage-element.xml ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/resources/fact-local-storage-element.xml b/lens-cli/src/test/resources/fact-local-storage-element.xml index 8ec3b42..d9bb111 100644 --- a/lens-cli/src/test/resources/fact-local-storage-element.xml +++ b/lens-cli/src/test/resources/fact-local-storage-element.xml @@ -21,6 +21,10 @@ --> <x_storage_table_element xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd "> + <update_periods> + <update_period>HOURLY</update_period> + <update_period>DAILY</update_period> + </update_periods> <storage_name>fact_local</storage_name> <table_desc external="true" field_delimiter="," table_location="/tmp/examples/local"> <part_cols> @@ -28,8 +32,4 @@ </part_cols> <time_part_cols>dt</time_part_cols> </table_desc> - <update_periods> - <update_period>HOURLY</update_period> - <update_period>DAILY</update_period> - </update_periods> </x_storage_table_element> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-cli/src/test/resources/fact_without_weight.xml ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/resources/fact_without_weight.xml b/lens-cli/src/test/resources/fact_without_weight.xml new file mode 100644 index 0000000..fc0c670 --- /dev/null +++ b/lens-cli/src/test/resources/fact_without_weight.xml @@ -0,0 +1,50 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<x_fact_table cube_name="cube_with_no_weight_facts" name="fact_without_wt" xmlns="uri:lens:cube:0.1" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd " > + <columns> + <column comment="" name="dim1" type="INT" /> + <column comment="" name="measure1" type="BIGINT" /> + <column comment="" name="measure2" type="INT" /> + <column comment="" name="measure3" type="FLOAT" /> + </columns> + <properties> + <property name="fact_without_wt.prop" value="f1" /> + <property name="cube.fact.is.aggregated" value="true" /> + </properties> + <storage_tables> + <storage_table> + <update_periods> + <update_period>HOURLY</update_period> + <update_period>DAILY</update_period> + <update_period>MONTHLY</update_period> + </update_periods> + <storage_name>fact_local_without_wt</storage_name> + <table_desc external="true" field_delimiter="," table_location="/tmp/examples/fact_local_without_wt"> + <part_cols> + <column comment="Time column" name="dt" type="STRING" /> + </part_cols> + <time_part_cols>dt</time_part_cols> + </table_desc> + </storage_table> + </storage_tables> +</x_fact_table> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-cli/src/test/resources/sample-cube.xml ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/resources/sample-cube.xml b/lens-cli/src/test/resources/sample-cube.xml index d54bcfb..f0eba57 100644 --- a/lens-cli/src/test/resources/sample-cube.xml +++ b/lens-cli/src/test/resources/sample-cube.xml @@ -21,18 +21,16 @@ --> <x_base_cube name="sample_cube" xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd "> + <properties> + <property name="sample_cube.prop" value="sample" /> + <property name="cube.sample_cube.timed.dimensions.list" value="dt" /> + </properties> <measures> <measure name="measure1" type="BIGINT" /> <measure name="measure2" type="INT" default_aggr="SUM" /> <measure name="measure3" type="FLOAT" default_aggr="MAX" start_time='2013-12-12T00:00:00' /> <measure name="measure4" type="DOUBLE" default_aggr="MIN" /> </measures> - <expressions> - <expression name="expr_msr5" type="DOUBLE"> - <expr_spec expr = "measure3 + measure4" end_time='2013-12-12T00:00:00'/> - <expr_spec expr = "measure3 + measure4 + 0.01" start_time='2013-12-12T00:00:00'/> - </expression> - </expressions> <dim_attributes> <dim_attribute name="dim1" type="INT" /> <dim_attribute name="dim2" type="INT" start_time='2013-12-01T00:00:00' /> @@ -44,8 +42,10 @@ </ref_spec> </dim_attribute> </dim_attributes> - <properties> - <property name="sample_cube.prop" value="sample" /> - <property name="cube.sample_cube.timed.dimensions.list" value="dt" /> - </properties> + <expressions> + <expression name="expr_msr5" type="DOUBLE"> + <expr_spec expr = "measure3 + measure4" end_time='2013-12-12T00:00:00'/> + <expr_spec expr = "measure3 + measure4 + 0.01" start_time='2013-12-12T00:00:00'/> + </expression> + </expressions> </x_base_cube> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-cli/src/test/resources/test-dimension.xml ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/resources/test-dimension.xml b/lens-cli/src/test/resources/test-dimension.xml index d602569..a9b7c08 100644 --- a/lens-cli/src/test/resources/test-dimension.xml +++ b/lens-cli/src/test/resources/test-dimension.xml @@ -24,8 +24,8 @@ <attributes> <dim_attribute name="id" type="INT" /> <dim_attribute name="name" type="STRING" /> - <dim_attribute name="detail" type="STRING" start_time='2013-12-01-00:00' /> - <dim_attribute name="d2id" type="INT" start_time='2013-12-01-00:00'> + <dim_attribute name="detail" type="STRING" start_time='2013-12-01T00:00:00' /> + <dim_attribute name="d2id" type="INT" start_time='2013-12-01T00:00:00'> <ref_spec> <table_references> <table_reference table="test_dim2" column="id" /> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java ---------------------------------------------------------------------- diff --git a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java index a6651f6..44f4f1f 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java @@ -21,6 +21,7 @@ package org.apache.lens.client; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.Charset; @@ -33,10 +34,13 @@ import javax.ws.rs.client.Entity; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.GenericType; import javax.ws.rs.core.MediaType; +import javax.xml.XMLConstants; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBElement; import javax.xml.bind.JAXBException; import javax.xml.bind.Unmarshaller; +import javax.xml.validation.Schema; +import javax.xml.validation.SchemaFactory; import org.apache.lens.api.APIResult; import org.apache.lens.api.APIResult.Status; @@ -52,6 +56,8 @@ import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.glassfish.jersey.media.multipart.FormDataMultiPart; import org.glassfish.jersey.media.multipart.MultiPartFeature; +import org.xml.sax.SAXException; + import com.google.common.base.Joiner; import com.google.common.io.Files; @@ -67,10 +73,17 @@ public class LensMetadataClient { static { try { + ClassLoader classLoader = LensMetadataClient.class.getClassLoader(); + SchemaFactory sf = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); + URI uri = classLoader.getResource("cube-0.1.xsd").toURI(); + LOG.info("URI for cube schema: " + uri.toString()); + Schema schema = sf.newSchema(uri.toURL()); JAXBContext jaxbContext = JAXBContext.newInstance(ObjectFactory.class); JAXB_UNMARSHALLER = jaxbContext.createUnmarshaller(); - } catch (JAXBException e) { - throw new RuntimeException("Could not initialize JAXBCOntext"); + JAXB_UNMARSHALLER.setSchema(schema); + } catch (JAXBException | URISyntaxException | MalformedURLException | SAXException e) { + LOG.error("Could not initialize JAXBContext. ", e); + throw new RuntimeException("Could not initialize JAXBContext. ", e); } } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-examples/src/main/resources/city_subset.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/city_subset.xml b/lens-examples/src/main/resources/city_subset.xml index 5059e5e..18c7847 100644 --- a/lens-examples/src/main/resources/city_subset.xml +++ b/lens-examples/src/main/resources/city_subset.xml @@ -19,7 +19,7 @@ under the License. --> -<x_dimension_table dimension_name="city" table_name="city_subset" weight="0.0" xmlns="uri:lens:cube:0.1" +<x_dimension_table dimension_name="city" table_name="city_subset" weight="100.0" xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd "> <columns> <column comment="ID" name="id" type="INT"/> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-examples/src/main/resources/customer_table.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/customer_table.xml b/lens-examples/src/main/resources/customer_table.xml index 18c387f..af308c2 100644 --- a/lens-examples/src/main/resources/customer_table.xml +++ b/lens-examples/src/main/resources/customer_table.xml @@ -19,7 +19,7 @@ under the License. --> -<x_dimension_table dimension_name="customer" table_name="customer_table" weight="0.0" xmlns="uri:lens:cube:0.1" +<x_dimension_table dimension_name="customer" table_name="customer_table" weight="100.0" xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd "> <columns> <column comment="ID" name="id" type="INT"/> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-examples/src/main/resources/dim_table.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/dim_table.xml b/lens-examples/src/main/resources/dim_table.xml index 25367af..10c9ac0 100644 --- a/lens-examples/src/main/resources/dim_table.xml +++ b/lens-examples/src/main/resources/dim_table.xml @@ -19,7 +19,7 @@ under the License. --> -<x_dimension_table dimension_name="sample_dim" table_name="dim_table" weight="0.0" xmlns="uri:lens:cube:0.1" +<x_dimension_table dimension_name="sample_dim" table_name="dim_table" weight="100.0" xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd "> <columns> <column comment="ID" name="id" type="INT"/> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-examples/src/main/resources/dim_table2.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/dim_table2.xml b/lens-examples/src/main/resources/dim_table2.xml index eef1513..e72f9c5 100644 --- a/lens-examples/src/main/resources/dim_table2.xml +++ b/lens-examples/src/main/resources/dim_table2.xml @@ -19,7 +19,7 @@ under the License. --> -<x_dimension_table dimension_name="sample_dim2" table_name="dim_table2" weight="0.0" xmlns="uri:lens:cube:0.1" +<x_dimension_table dimension_name="sample_dim2" table_name="dim_table2" weight="100.0" xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd "> <columns> <column comment="ID" name="id" type="INT"/> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-examples/src/main/resources/dim_table3.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/dim_table3.xml b/lens-examples/src/main/resources/dim_table3.xml index 386c394..7955426 100644 --- a/lens-examples/src/main/resources/dim_table3.xml +++ b/lens-examples/src/main/resources/dim_table3.xml @@ -19,7 +19,7 @@ under the License. --> -<x_dimension_table dimension_name="sample_db_dim" table_name="dim_table3" weight="0.0" xmlns="uri:lens:cube:0.1" +<x_dimension_table dimension_name="sample_db_dim" table_name="dim_table3" weight="100.0" xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd "> <columns> <column comment="ID" name="id" type="INT"/> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-examples/src/main/resources/dim_table4.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/dim_table4.xml b/lens-examples/src/main/resources/dim_table4.xml index 69e3c50..9de1cd7 100644 --- a/lens-examples/src/main/resources/dim_table4.xml +++ b/lens-examples/src/main/resources/dim_table4.xml @@ -19,7 +19,7 @@ under the License. --> -<x_dimension_table dimension_name="sample_dim" table_name="dim_table4" weight="0.0" xmlns="uri:lens:cube:0.1" +<x_dimension_table dimension_name="sample_dim" table_name="dim_table4" weight="100.0" xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd "> <columns> <column comment="ID" name="id" type="INT"/> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-examples/src/main/resources/product_db_table.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/product_db_table.xml b/lens-examples/src/main/resources/product_db_table.xml index d961c44..867d37b 100644 --- a/lens-examples/src/main/resources/product_db_table.xml +++ b/lens-examples/src/main/resources/product_db_table.xml @@ -19,7 +19,7 @@ under the License. --> -<x_dimension_table dimension_name="product" table_name="product_db_table" weight="0.0" xmlns="uri:lens:cube:0.1" +<x_dimension_table dimension_name="product" table_name="product_db_table" weight="100.0" xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd "> <columns> <column comment="ID" name="id" type="INT"/> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-examples/src/main/resources/product_table.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/product_table.xml b/lens-examples/src/main/resources/product_table.xml index 86a1115..303f3ad 100644 --- a/lens-examples/src/main/resources/product_table.xml +++ b/lens-examples/src/main/resources/product_table.xml @@ -19,7 +19,7 @@ under the License. --> -<x_dimension_table dimension_name="product" table_name="product_table" weight="0.0" xmlns="uri:lens:cube:0.1" +<x_dimension_table dimension_name="product" table_name="product_table" weight="100.0" xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd "> <columns> <column comment="ID" name="id" type="INT"/> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-examples/src/main/resources/sales-aggr-fact1.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/sales-aggr-fact1.xml b/lens-examples/src/main/resources/sales-aggr-fact1.xml index 073052d..728c775 100644 --- a/lens-examples/src/main/resources/sales-aggr-fact1.xml +++ b/lens-examples/src/main/resources/sales-aggr-fact1.xml @@ -64,14 +64,14 @@ <storage_name>mydb</storage_name> <table_desc external="true" field_delimiter="," table_location="/tmp/db-storage.db" storage_handler_name="org.apache.lens.storage.db.DBStorageHandler"> - <table_parameters> - <property name="lens.metastore.native.db.name" value="default"/> - </table_parameters> <part_cols> <column comment="Process time partition" name="pt" type="STRING"/> <column comment="Order time partition" name="ot" type="STRING"/> <column comment="Delivery time partition" name="dt" type="STRING"/> </part_cols> + <table_parameters> + <property name="lens.metastore.native.db.name" value="default"/> + </table_parameters> <time_part_cols>pt</time_part_cols> <time_part_cols>ot</time_part_cols> <time_part_cols>dt</time_part_cols> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-examples/src/main/resources/sales-aggr-fact2.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/sales-aggr-fact2.xml b/lens-examples/src/main/resources/sales-aggr-fact2.xml index 67b7171..61edf64 100644 --- a/lens-examples/src/main/resources/sales-aggr-fact2.xml +++ b/lens-examples/src/main/resources/sales-aggr-fact2.xml @@ -58,14 +58,14 @@ <storage_name>mydb</storage_name> <table_desc external="true" field_delimiter="," table_location="/tmp/db-storage.db" storage_handler_name="org.apache.lens.storage.db.DBStorageHandler"> - <table_parameters> - <property name="lens.metastore.native.db.name" value="default"/> - </table_parameters> <part_cols> <column comment="Process time partition" name="pt" type="STRING"/> <column comment="Order time partition" name="ot" type="STRING"/> <column comment="Delivery time partition" name="dt" type="STRING"/> </part_cols> + <table_parameters> + <property name="lens.metastore.native.db.name" value="default"/> + </table_parameters> <time_part_cols>pt</time_part_cols> <time_part_cols>ot</time_part_cols> <time_part_cols>dt</time_part_cols> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-examples/src/main/resources/sales-cube.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/sales-cube.xml b/lens-examples/src/main/resources/sales-cube.xml index e44141c..4923aa3 100644 --- a/lens-examples/src/main/resources/sales-cube.xml +++ b/lens-examples/src/main/resources/sales-cube.xml @@ -21,6 +21,13 @@ --> <x_base_cube name="sales" xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd "> + <properties> + <property name="cube.sales.timed.dimensions.list" value="order_time,delivery_time" /> + <property name="cube.timedim.partition.order_time" value="ot" /> + <property name="cube.timedim.partition.delivery_time" value="dt" /> + <property name="cube.timedim.relation.order_time" value="delivery_time+[-20 days,-1 hour]" /> + <!-- means dt-20days <= ot <= dt-1hour --> + </properties> <measures> <measure name="unit_sales" type="BIGINT" default_aggr="SUM" display_string="Unit Sales" format_string="#,###"/> <measure name="store_sales" type="DOUBLE" default_aggr="SUM" display_string="Store Sales" format_string="#,###.##"/> @@ -36,16 +43,6 @@ <measure name="rating" type="FLOAT" default_aggr="AVG" display_string="Rating"/> </measures> - <expressions> - <expression name="profit" type="DOUBLE" display_string="Profit"> - <expr_spec expr="store_sales - store_cost"/> - </expression> - <expression name="promotion_sales" type="DOUBLE" display_string="Promotion sales"> - <expr_spec expr="sum(case when promotion_id = 0 then 0 else store_sales end)" start_time='2015-04-12T00:00:00'/> - <expr_spec expr="sum(case when promotion_id = -1 then 0 when promotion_id = -1 then 0 else store_sales end)" - end_time="2015-04-12T00:00:00"/> - </expression> - </expressions> <dim_attributes> <dim_attribute name="customer_id" type="INT" /> <dim_attribute name="product_id" type="INT" /> @@ -58,24 +55,34 @@ <dim_attribute name="production_city_id" type="INT" /> <dim_attribute name="delivery_city_id" type="INT" /> <dim_attribute name="customer_city_name" type="string" description="City name to which the customer belongs" - display_string="Customer City"> + display_string="Customer City"> <ref_spec> <chain_ref_column chain_name="customer_city" ref_col="name" /> </ref_spec> </dim_attribute> <dim_attribute name="production_city_name" type="STRING" description="City name in which the product was produced" - display_string="Production City"> + display_string="Production City"> <ref_spec> <chain_ref_column chain_name="production_city" ref_col="name" /> </ref_spec> </dim_attribute> <dim_attribute name="delivery_city_name" type="STRING" description="City name to which the product was delivered" - display_string="Delivery City"> + display_string="Delivery City"> <ref_spec> <chain_ref_column chain_name="delivery_city" ref_col="name" /> </ref_spec> </dim_attribute> </dim_attributes> + <expressions> + <expression name="profit" type="DOUBLE" display_string="Profit"> + <expr_spec expr="store_sales - store_cost"/> + </expression> + <expression name="promotion_sales" type="DOUBLE" display_string="Promotion sales"> + <expr_spec expr="sum(case when promotion_id = 0 then 0 else store_sales end)" start_time='2015-04-12T00:00:00'/> + <expr_spec expr="sum(case when promotion_id = -1 then 0 when promotion_id = -1 then 0 else store_sales end)" + end_time="2015-04-12T00:00:00"/> + </expression> + </expressions> <join_chains> <join_chain name="customer_details"> <paths> @@ -150,11 +157,4 @@ </paths> </join_chain> </join_chains> - <properties> - <property name="cube.sales.timed.dimensions.list" value="order_time,delivery_time" /> - <property name="cube.timedim.partition.order_time" value="ot" /> - <property name="cube.timedim.partition.delivery_time" value="dt" /> - <property name="cube.timedim.relation.order_time" value="delivery_time+[-20 days,-1 hour]" /> - <!-- means dt-20days <= ot <= dt-1hour --> - </properties> </x_base_cube> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-examples/src/main/resources/sample-cube.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/sample-cube.xml b/lens-examples/src/main/resources/sample-cube.xml index 0d97842..7b784ea 100644 --- a/lens-examples/src/main/resources/sample-cube.xml +++ b/lens-examples/src/main/resources/sample-cube.xml @@ -21,18 +21,15 @@ --> <x_base_cube name="sample_cube" xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd "> + <properties> + <property name="cube.sample_cube.timed.dimensions.list" value="dt"/> + </properties> <measures> <measure name="measure1" type="BIGINT"/> <measure name="measure2" type="INT" default_aggr="SUM"/> <measure name="measure3" type="FLOAT" default_aggr="MAX" start_time='2013-12-12T00:00:00'/> <measure name="measure4" type="DOUBLE" default_aggr="MIN"/> </measures> - <expressions> - <expression name="expr_msr5" type="DOUBLE"> - <expr_spec expr = "measure3 + measure4" start_time='2013-12-12T00:00:00' /> - <expr_spec expr = "measure3 + measure4 + 0.01" end_time='2013-12-12T00:00:00'/> - </expression> - </expressions> <dim_attributes> <dim_attribute name="dim1" type="INT"/> <dim_attribute name="dim2" type="INT" start_time='2013-12-01T00:00:00'/> @@ -44,7 +41,10 @@ </ref_spec> </dim_attribute> </dim_attributes> - <properties> - <property name="cube.sample_cube.timed.dimensions.list" value="dt"/> - </properties> + <expressions> + <expression name="expr_msr5" type="DOUBLE"> + <expr_spec expr = "measure3 + measure4" start_time='2013-12-12T00:00:00' /> + <expr_spec expr = "measure3 + measure4 + 0.01" end_time='2013-12-12T00:00:00'/> + </expression> + </expressions> </x_base_cube>
