LENS-1412 : Add capability to define virtual facts in a cube
Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/5de45e0f Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/5de45e0f Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/5de45e0f Branch: refs/heads/master Commit: 5de45e0f8212c2e03eaf9886b720e13868a1c42a Parents: 34500f1 Author: Rajitha R <rajitha....@gmail.com> Authored: Tue Jun 6 13:59:58 2017 +0530 Committer: Amareshwari Sriramadasu <amareshw...@apache.org> Committed: Tue Jun 6 13:59:58 2017 +0530 ---------------------------------------------------------------------- .../lens/api/metastore/SchemaTraverser.java | 1 + lens-api/src/main/resources/cube-0.1.xsd | 176 +++---- lens-api/src/main/resources/lens-errors.conf | 6 + .../lens/cli/commands/LensFactCommands.java | 6 +- .../lens/cli/commands/LensSchemaCommands.java | 18 +- .../apache/lens/cli/TestLensFactCommands.java | 132 +++++- .../apache/lens/cli/TestLensSchemaCommands.java | 2 +- .../schema/cubes/base/virtual-cube.xml | 30 ++ .../resources/schema/facts/virtual_fact.xml | 28 ++ .../java/org/apache/lens/client/LensClient.java | 2 +- .../apache/lens/client/LensMetadataClient.java | 18 +- .../lens/cube/error/LensCubeErrorCode.java | 3 +- .../lens/cube/metadata/AbstractCubeTable.java | 22 +- .../lens/cube/metadata/CubeFactTable.java | 52 ++- .../lens/cube/metadata/CubeMetastoreClient.java | 463 ++++++++++++++----- .../cube/metadata/CubeVirtualFactTable.java | 186 ++++++++ .../apache/lens/cube/metadata/FactTable.java | 166 +++++++ .../apache/lens/cube/metadata/JAXBUtils.java | 24 +- .../lens/cube/metadata/MetastoreConstants.java | 2 + .../lens/cube/metadata/MetastoreUtil.java | 26 +- .../apache/lens/cube/metadata/Segmentation.java | 13 +- .../org/apache/lens/cube/parse/Candidate.java | 8 +- .../lens/cube/parse/CandidateTableResolver.java | 11 +- .../lens/cube/parse/StorageCandidate.java | 55 +-- .../cube/parse/StorageCandidateHQLContext.java | 12 +- .../lens/cube/parse/StorageTableResolver.java | 5 +- .../lens/cube/metadata/CubeFactTableTest.java | 12 +- .../cube/metadata/TestCubeMetastoreClient.java | 123 ++++- .../apache/lens/cube/parse/CubeTestSetup.java | 40 +- .../lens/cube/parse/TestCubeRewriter.java | 16 + .../parse/TestCubeSegmentationRewriter.java | 15 +- .../lens/cube/parse/TestUnionQueries.java | 2 +- .../resources/schema/cubes/base/virtualcube.xml | 36 ++ .../test/resources/schema/facts/virtualfact.xml | 27 ++ lens-driver-es/pom.xml | 3 +- lens-examples/pom.xml | 3 +- .../src/test/resources/yaml/fact1.yaml | 4 +- .../src/test/resources/yaml/fact2.yaml | 4 +- .../src/test/resources/yaml/rawfact.yaml | 4 +- .../yaml/sales-aggr-continuous-fact.yaml | 4 +- .../test/resources/yaml/sales-aggr-fact1.yaml | 4 +- .../test/resources/yaml/sales-aggr-fact2.yaml | 4 +- .../src/test/resources/yaml/sales-raw-fact.yaml | 4 +- .../api/metastore/CubeMetastoreService.java | 8 +- lens-server/pom.xml | 2 +- .../metastore/CubeMetastoreServiceImpl.java | 24 +- .../server/metastore/MetastoreResource.java | 17 +- .../lens/server/common/RestAPITestUtil.java | 7 +- .../server/metastore/TestMetastoreService.java | 274 ++++++++++- src/site/apt/user/cli.apt | 3 +- tools/scripts/generate-site-public.sh | 2 +- 51 files changed, 1646 insertions(+), 463 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java b/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java index 9564443..09c848a 100644 --- a/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java +++ b/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java @@ -42,6 +42,7 @@ public class SchemaTraverser implements Runnable { types.put("cubes/derived", XDerivedCube.class); types.put("dimensions", XDimension.class); types.put("facts", XFactTable.class); + types.put("facts/virtual", XVirtualFactTable.class); types.put("dimtables", XDimensionTable.class); types.put("dimensiontables", XDimensionTable.class); types.put("dimensiontables", XDimensionTable.class); http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-api/src/main/resources/cube-0.1.xsd ---------------------------------------------------------------------- diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd index 8158e6d..4d10d24 100644 --- a/lens-api/src/main/resources/cube-0.1.xsd +++ b/lens-api/src/main/resources/cube-0.1.xsd @@ -168,76 +168,6 @@ </xs:complexContent> </xs:complexType> - <xs:element name="x_virtual_fact_table" type="x_virtual_fact_table"/> - <xs:complexType name="x_virtual_fact_table"> - <xs:annotation> - <xs:documentation> - Virtual fact extends fact. It can override the cube of its source fact. It can have different - properties associated with it than its source. - Properties that can be set for a virtual fact are : - - Filters which would be added in query rewriting - </xs:documentation> - </xs:annotation> - <xs:sequence> - <xs:element type="x_properties" name="properties" maxOccurs="1" minOccurs="0"> - <xs:annotation> - <xs:documentation> - Properties that can be set for a virtual fact are - 1. cube.fact.query.where.filter : filter string that needs to be added in WHERE clause. This string would be added as an additional - filter when the query is being constructed in the cube query writing phase. - 2. cube.fact.absolute.start.time: start time of the fact. For queries that ask for time before this, - this fact is not a candidate. Time format can be as you would specify in the time_range_in clause. - i.e. yyyy[-mm[-dd[-hh[:MM[:ss[,SSS]]]]]] - 3. cube.fact.relative.start.time: Here you can specify fact's relative validity relative to current time. - Useful if you want to specify e.g. this fact is valid for today - 90 days. Can be specified as just - a time difference e.g. "-90 days". Or can be specified in relative syntax. - e.g. now.year or now.day - 6 hour etc. - 4. cube.fact.absolute.end.time: If you're deprecating this fact, put the final date till which the data of - the fact will be valid here. Format same as absolute start time. - 5. cube.fact.relative.end.time: You can specify the end date for fact table - relative to current date e.g. fact table is valid for next 90days starting from today. - This can be specified as just a time difference e.g. "+90 days" - - </xs:documentation> - </xs:annotation> - </xs:element> - </xs:sequence> - <xs:attribute type="xs:string" name="source_fact_name" use="required"> - <xs:annotation> - <xs:documentation> - The Source fact name over which the Virtual fact is defined. - </xs:documentation> - </xs:annotation> - </xs:attribute> - <xs:attribute type="xs:string" name="cube_name" use="required"> - <xs:annotation> - <xs:documentation> - The base cube's name to which the Virtual fact is associated. - </xs:documentation> - </xs:annotation> - </xs:attribute> - <xs:attribute name="name" type="xs:string" use="required"> - <xs:annotation> - <xs:documentation> - The virtual fact table name. - </xs:documentation> - </xs:annotation> - </xs:attribute> - <xs:attribute name="weight" use="optional" > - <xs:annotation> - <xs:documentation> - The weight of the fact table. LENS will use this attribute to decide the lightest table to query when there - are more than one eligible tables. If not defined, the source fact weight would be picked up. - </xs:documentation> - </xs:annotation> - <xs:simpleType> - <xs:restriction base="xs:double"> - <xs:minInclusive value="0"></xs:minInclusive> - </xs:restriction> - </xs:simpleType> - </xs:attribute> - </xs:complexType> - <xs:element name="x_derived_cube" type="x_derived_cube"/> <xs:complexType name="x_derived_cube"> <xs:annotation> @@ -1287,9 +1217,99 @@ </xs:sequence> </xs:complexType> + <xs:element name="x_fact" type="x_fact"/> + + <xs:complexType name="x_fact" abstract="true"> + <xs:annotation> + <xs:documentation> + XFact can either be a Cube Fact for which the user would give the full specification of the + columns and storages + or can be a Virtual fact, for which the user would specify only the Source fact and filters if any. + Virtual fact derives its columns and storage specific details all from the source Fact. It can however have + different properties associated with it than the source. + </xs:documentation> + </xs:annotation> + <xs:attribute type="xs:string" name="name" use="required"/> + <xs:attribute type="xs:string" name="description"/> + <xs:attribute type="xs:string" name="cube_name" use="required"> + <xs:annotation> + <xs:documentation> + The cube's name to which the fact is associated. + </xs:documentation> + </xs:annotation> + </xs:attribute> + </xs:complexType> + + <xs:element name="x_virtual_fact_table" type="x_virtual_fact_table"/> + <xs:complexType name="x_virtual_fact_table"> + <xs:complexContent> + <xs:extension base="x_fact"> + <xs:annotation> + <xs:documentation> + Virtual fact extends fact. It can override the cube of its source fact. It can have different + properties associated with it than its source. + Properties that can be set for a virtual fact are : + - Filters which would be added in query rewriting + </xs:documentation> + </xs:annotation> + <xs:sequence> + <xs:element type="x_properties" name="properties" maxOccurs="1" minOccurs="0"> + <xs:annotation> + <xs:documentation> + Properties that can be set for a virtual fact are : + 1. cube.fact.query.where.filter : filter string that needs to be added in WHERE clause. This string + would be added as an additional + filter when the query is being constructed in the cube query writing phase. + 2. cube.fact.absolute.start.time: start time of the fact. For queries that ask for time before this, + this fact is not a candidate. Time format can be as you would specify in the time_range_in clause. + i.e. yyyy[-mm[-dd[-hh[:MM[:ss[,SSS]]]]]] + 3. cube.fact.relative.start.time: Here you can specify fact's relative validity relative to current + time. + Useful if you want to specify e.g. this fact is valid for today - 90 days. Can be specified as just + a time difference e.g. "-90 days". Or can be specified in relative syntax. + e.g. now.year or now.day - 6 hour etc. + 4. cube.fact.absolute.end.time: If you're deprecating this fact, put the final date till which the data + of + the fact will be valid here. Format same as absolute start time. + 5. cube.fact.relative.end.time: You can specify the end date for fact table + relative to current date e.g. fact table is valid for next 90days starting from today. + This can be specified as just a time difference e.g. "+90 days" + + </xs:documentation> + </xs:annotation> + </xs:element> + </xs:sequence> + + <xs:attribute type="xs:string" name="source_fact_name" use="required"> + <xs:annotation> + <xs:documentation> + The Source fact name over which the Virtual fact is defined. + </xs:documentation> + </xs:annotation> + </xs:attribute> + <xs:attribute name="weight" use="optional"> + <xs:annotation> + <xs:documentation> + The weight of the fact table. LENS will use this attribute to decide the lightest table to query when + there + are more than one eligible tables. If not defined, the source fact weight would be picked up. + </xs:documentation> + </xs:annotation> + <xs:simpleType> + <xs:restriction base="xs:double"> + <xs:minInclusive value="0"></xs:minInclusive> + </xs:restriction> + </xs:simpleType> + </xs:attribute> + </xs:extension> + </xs:complexContent> + </xs:complexType> + <xs:element name="x_fact_table" type="x_fact_table"/> <xs:complexType name="x_fact_table"> + <xs:complexContent> + <xs:extension base="x_fact"> <xs:annotation> <xs:documentation> Fact table that is associated to a base cube. The columns in the fact table will be a subset of @@ -1334,20 +1354,6 @@ </xs:element> <xs:element name="storage_tables" type="x_storage_tables" maxOccurs="1" minOccurs="0"/> </xs:sequence> - <xs:attribute name="name" type="xs:string" use="required"> - <xs:annotation> - <xs:documentation> - The fact table name. - </xs:documentation> - </xs:annotation> - </xs:attribute> - <xs:attribute name="cube_name" type="xs:string" use="required"> - <xs:annotation> - <xs:documentation> - The base cube's name to which the fact_table is associated. - </xs:documentation> - </xs:annotation> - </xs:attribute> <xs:attribute name="weight" use="required" > <xs:annotation> <xs:documentation> @@ -1361,6 +1367,8 @@ </xs:restriction> </xs:simpleType> </xs:attribute> + </xs:extension> + </xs:complexContent> </xs:complexType> <xs:element name="x_segmentation" type="x_segmentation"/> http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-api/src/main/resources/lens-errors.conf ---------------------------------------------------------------------- diff --git a/lens-api/src/main/resources/lens-errors.conf b/lens-api/src/main/resources/lens-errors.conf index 43de1e9..fafd655 100644 --- a/lens-api/src/main/resources/lens-errors.conf +++ b/lens-api/src/main/resources/lens-errors.conf @@ -384,6 +384,12 @@ lensCubeErrorsForMetastore = [ errorMsg = "Partition filter can not be null or empty" } + { + errorCode = 3106 + httpStatusCode = ${BAD_REQUEST} + errorMsg = "Fact %s not of type %s" + } + ] lensDriverErrors = [ http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java index a01d6c0..a872998 100644 --- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java +++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java @@ -22,7 +22,7 @@ import java.io.File; import java.util.List; import org.apache.lens.api.APIResult; -import org.apache.lens.api.metastore.XFactTable; +import org.apache.lens.api.metastore.XFact; import org.apache.lens.api.metastore.XPartition; import org.apache.lens.api.metastore.XStorageTableElement; import org.apache.lens.cli.commands.annotations.UserDocumentation; @@ -39,7 +39,7 @@ import lombok.NonNull; @Component @UserDocumentation(title = "Commands for Facts Management", description = "These command provide CRUD for facts, associated storages, and fact partitions") -public class LensFactCommands extends LogicalTableCrudCommand<XFactTable> { +public class LensFactCommands extends LogicalTableCrudCommand<XFact> { /** * Show facts. @@ -352,7 +352,7 @@ public class LensFactCommands extends LogicalTableCrudCommand<XFactTable> { } @Override - protected XFactTable doRead(String name) { + protected XFact doRead(String name) { return getClient().getFactTable(name); } http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java index 20f313a..aca1cf9 100644 --- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java +++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java @@ -24,14 +24,7 @@ import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; -import org.apache.lens.api.metastore.SchemaTraverser; -import org.apache.lens.api.metastore.XBaseCube; -import org.apache.lens.api.metastore.XDerivedCube; -import org.apache.lens.api.metastore.XDimension; -import org.apache.lens.api.metastore.XDimensionTable; -import org.apache.lens.api.metastore.XFactTable; -import org.apache.lens.api.metastore.XSegmentation; -import org.apache.lens.api.metastore.XStorage; +import org.apache.lens.api.metastore.*; import org.apache.lens.cli.commands.annotations.UserDocumentation; import org.springframework.beans.factory.annotation.Autowired; @@ -81,7 +74,12 @@ public class LensSchemaCommands implements CommandMarker { + "|\n" + "|-- facts\n" + " |-- fact1.xml\n" - + " |-- fact2.xml\n\n\n" + + " |-- fact2.xml\n" + + "| |\n" + + "| |-- virtual\n" + + "| | |-- virtual_fact1.xml\n" + + "| | |-- virtual_fact2.xml\n" + + "| |\n\n\n" + "If your cubes are divided between base and derived cubes,\nit makes sense to seperate into two directories, " + "since derived cubes can't be created unless base cube exists.\nIn the other case you can keep them in the cubes " + "directory itself.\nFor dimtables, you can keep your schema files in a directory named either dimtables or " @@ -114,6 +112,8 @@ public class LensSchemaCommands implements CommandMarker { UPDATE_COMMAND_MAP.put(XDimensionTable.class, "update dimtable --dimtable_name %s --path %s"); CREATE_COMMAND_MAP.put(XFactTable.class, "create fact --path %s"); UPDATE_COMMAND_MAP.put(XFactTable.class, "update fact --fact_name %s --path %s"); + CREATE_COMMAND_MAP.put(XVirtualFactTable.class, "create fact --path %s"); + UPDATE_COMMAND_MAP.put(XVirtualFactTable.class, "update fact --name %s --path %s"); CREATE_COMMAND_MAP.put(XSegmentation.class, "create segmentation --path %s"); UPDATE_COMMAND_MAP.put(XSegmentation.class, "update segmentation --name %s --path %s"); } http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java index d31e25c..927b439 100644 --- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java +++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java @@ -59,11 +59,21 @@ public class TestLensFactCommands extends LensCliApplicationTest { @Test public void testFactCommands() throws IOException, URISyntaxException { createSampleCube(); + createVirtualCube(); + + //test base fact addFact1Table(); updateFact1Table(); testFactStorageActions(); testFactPartitionActions(); + + //test virtual fact + addVirtualFactTable(); + updateVirtualFactTable(); + dropVirtualFactTable(); + dropFact1Table(); + dropVirtualCube(); dropSampleCube(); } @@ -76,10 +86,25 @@ public class TestLensFactCommands extends LensCliApplicationTest { assertTrue(cubeList.contains("sample_cube"), cubeList); } + private void createVirtualCube() throws URISyntaxException { + URL cubeSpec = TestLensCubeCommands.class.getClassLoader().getResource("schema/cubes/base/virtual-cube.xml"); + String cubeList = getCubeCommand().showCubes(); + assertFalse(cubeList.contains("virtualcube"), cubeList); + getCubeCommand().createCube(new File(cubeSpec.toURI())); + cubeList = getCubeCommand().showCubes(); + assertTrue(cubeList.contains("virtualcube"), cubeList); + } + + private void dropSampleCube() { getCubeCommand().dropCube("sample_cube"); } + + private void dropVirtualCube() { + getCubeCommand().dropCube("virtualcube"); + } + private static LensFactCommands getCommand() { if (command == null) { LensClient client = new LensClient(); @@ -115,9 +140,9 @@ public class TestLensFactCommands extends LensCliApplicationTest { */ public static void addFact1Table() throws IOException { LensFactCommands command = getCommand(); - String factList = command.showFacts(null); + String factList = command.showFacts("sample_cube"); assertEquals(command.showFacts("sample_cube"), "No fact found for sample_cube"); - assertEquals(factList, "No fact found", "Fact tables should not be found"); + assertEquals(factList, "No fact found for sample_cube", "Fact tables should not be found"); // add local storage before adding fact table TestLensStorageCommands.addLocalStorage(FACT_LOCAL); URL factSpec = TestLensFactCommands.class.getClassLoader().getResource("schema/facts/fact1.xml"); @@ -126,7 +151,7 @@ public class TestLensFactCommands extends LensCliApplicationTest { } catch (Exception e) { fail("Unable to create fact table" + e.getMessage()); } - factList = command.showFacts(null); + factList = command.showFacts("sample_cube"); assertEquals(command.showFacts("sample_cube"), factList); try { assertEquals(command.showFacts("blah"), factList); @@ -144,6 +169,40 @@ public class TestLensFactCommands extends LensCliApplicationTest { } /** + * Adds the virtual fact table. + * + * @throws IOException + */ + public static void addVirtualFactTable() throws IOException { + LensFactCommands command = getCommand(); + String factList = command.showFacts("virtualcube"); + assertEquals(command.showFacts("virtualcube"), "No fact found for virtualcube"); + assertEquals(factList, "No fact found for virtualcube", "Fact tables should not be found"); + // add local storage before adding fact table + URL factSpec = TestLensFactCommands.class.getClassLoader().getResource("schema/facts/virtual_fact.xml"); + try { + command.createFact(new File(factSpec.toURI())); + } catch (Exception e) { + fail("Unable to create virtual fact table" + e.getMessage()); + } + factList = command.showFacts("virtualcube"); + assertEquals(command.showFacts("virtualcube"), factList); + try { + assertEquals(command.showFacts("blah"), factList); + fail(); + } catch (NotFoundException e) { + log.info("blah is not a table", e); + } + try { + assertEquals(command.showFacts("virtualfact"), factList); + fail(); + } catch (NotFoundException e) { + log.info("virtualfact is a table, but not a cube table", e); + } + assertEquals("virtualfact", factList, "Virtualfact table should be found"); + } + + /** * Update fact1 table. */ public static void updateFact1Table() { @@ -196,6 +255,54 @@ public class TestLensFactCommands extends LensCliApplicationTest { } + + /** + * Update virtual fact table. + */ + public static void updateVirtualFactTable() { + try { + LensFactCommands command = getCommand(); + URL factSpec = TestLensFactCommands.class.getClassLoader().getResource("schema/facts/virtual_fact.xml"); + StringBuilder sb = new StringBuilder(); + BufferedReader bufferedReader = new BufferedReader(new FileReader(factSpec.getFile())); + String s; + while ((s = bufferedReader.readLine()) != null) { + sb.append(s).append("\n"); + } + + bufferedReader.close(); + + String xmlContent = sb.toString(); + + xmlContent = xmlContent.replace("<property name=\"virtualfact.prop\" value=\"f1\"/>\n", + "<property name=\"virtualfact.prop\" value=\"f1\"/>\n" + + "<property name=\"virtualfact.prop1\" value=\"f2\"/>\n"); + + File newFile = new File("target/local-virtualfact.xml"); + Writer writer = new OutputStreamWriter(new FileOutputStream(newFile)); + writer.write(xmlContent); + writer.close(); + + String desc = command.describeFactTable("virtualfact"); + log.debug(desc); + String propString = "virtualfact.prop: f1"; + String propString1 = "virtualfact.prop1: f2"; + + command.updateFactTable("virtualfact", new File("target/local-virtualfact.xml")); + desc = command.describeFactTable("virtualfact"); + log.debug(desc); + assertTrue(desc.contains(propString), "The sample property value is not set"); + assertTrue(desc.contains(propString1), "The sample property value is not set"); + + newFile.delete(); + + } catch (Throwable t) { + log.error("Updating of the virtualfact table failed with ", t); + fail("Updating of the virtualfact table failed with " + t.getMessage()); + } + + } + /** * Test fact storage actions. */ @@ -336,11 +443,24 @@ public class TestLensFactCommands extends LensCliApplicationTest { */ public static void dropFact1Table() { LensFactCommands command = getCommand(); - String factList = command.showFacts(null); + String factList = command.showFacts("sample_cube"); assertEquals("fact1", factList, "Fact1 table should be found"); command.dropFact("fact1", false); - factList = command.showFacts(null); - assertEquals(factList, "No fact found", "Fact tables should not be found"); + factList = command.showFacts("sample_cube"); + assertEquals(factList, "No fact found for sample_cube", "Fact tables should not be found"); TestLensStorageCommands.dropStorage(FACT_LOCAL); } + + + /** + * Drop virtualfact table. + */ + public static void dropVirtualFactTable() { + LensFactCommands command = getCommand(); + String factList = command.showFacts("virtualcube"); + assertEquals("virtualfact", factList, "Virtualfact table should be found"); + command.dropFact("virtualfact", false); + factList = command.showFacts("virtualcube"); + assertEquals(factList, "No fact found for virtualcube", "Virtual Fact tables should not be found"); + } } http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cli/src/test/java/org/apache/lens/cli/TestLensSchemaCommands.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensSchemaCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensSchemaCommands.java index ca6db2c..4a23532 100644 --- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensSchemaCommands.java +++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensSchemaCommands.java @@ -37,7 +37,7 @@ public class TestLensSchemaCommands extends LensCLITest { assertTrue(((String) execute("show databases")).contains(dbName)); execute("show storages", "local"); execute("show dimensions", "test_detail\ntest_dim"); - execute("show cubes", "sample_cube\ncube_with_no_weight_facts"); + execute("show cubes", "virtualcube\nsample_cube\ncube_with_no_weight_facts"); assertTrue(((String) execute("show dimtables")).contains("dim_table")); assertTrue(((String) execute("show facts")).contains("fact1")); execute("show segmentations", "seg1"); http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cli/src/test/resources/schema/cubes/base/virtual-cube.xml ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/resources/schema/cubes/base/virtual-cube.xml b/lens-cli/src/test/resources/schema/cubes/base/virtual-cube.xml new file mode 100644 index 0000000..8c57cee --- /dev/null +++ b/lens-cli/src/test/resources/schema/cubes/base/virtual-cube.xml @@ -0,0 +1,30 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<x_base_cube name="virtualcube" xmlns="uri:lens:cube:0.1"> + <properties> + <property name="sample_cube.prop" value="sample" /> + <property name="cube.sample_cube.timed.dimensions.list" value="dt" /> + </properties> + <measures> + <measure name="measure1" _type="BIGINT" /> + </measures> +</x_base_cube> http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cli/src/test/resources/schema/facts/virtual_fact.xml ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/resources/schema/facts/virtual_fact.xml b/lens-cli/src/test/resources/schema/facts/virtual_fact.xml new file mode 100644 index 0000000..81e2676 --- /dev/null +++ b/lens-cli/src/test/resources/schema/facts/virtual_fact.xml @@ -0,0 +1,28 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<x_virtual_fact_table source_fact_name="fact1" cube_name="virtualcube" name="virtualfact" xmlns="uri:lens:cube:0.1" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd "> + <properties> + <property name="cube.fact.query.where.filter" value=" dim1 = 10 "/> + <property name="virtualfact.prop" value="f1"/> + </properties> +</x_virtual_fact_table> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-client/src/main/java/org/apache/lens/client/LensClient.java ---------------------------------------------------------------------- diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClient.java b/lens-client/src/main/java/org/apache/lens/client/LensClient.java index 4f8da06..1de99fe 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensClient.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensClient.java @@ -514,7 +514,7 @@ public class LensClient implements AutoCloseable { return mc.updateDimension(dimName, dimSpec); } - public XFactTable getFactTable(String factName) { + public XFact getFactTable(String factName) { return mc.getFactTable(factName); } http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java ---------------------------------------------------------------------- diff --git a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java index f077c9c..8a05952 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java @@ -425,12 +425,12 @@ public class LensMetadataClient { .delete()); } - public XFactTable getFactTable(String factTableName) { + public XFact getFactTable(String factTableName) { WebTarget target = getMetastoreWebTarget(); - JAXBElement<XFactTable> table = target.path("facts").path(factTableName) + JAXBElement<XFact> table = target.path("facts").path(factTableName) .queryParam("sessionid", this.connection.getSessionHandle()) .request(MediaType.APPLICATION_XML) - .get(new GenericType<JAXBElement<XFactTable>>() { + .get(new GenericType<JAXBElement<XFact>>() { }); return table.getValue(); } @@ -445,17 +445,17 @@ public class LensMetadataClient { return seg.getValue(); } - public APIResult createFactTable(XFactTable f) { + public APIResult createFactTable(XFact f) { WebTarget target = getMetastoreWebTarget(); return translate(target.path("facts") .queryParam("sessionid", this.connection.getSessionHandle()) .request(MediaType.APPLICATION_XML) - .post(Entity.xml(new GenericEntity<JAXBElement<XFactTable>>(objFact.createXFactTable(f)){}))); + .post(Entity.xml(new GenericEntity<JAXBElement<XFact>>(objFact.createXFact(f)){}))); } public APIResult createFactTable(String factSpec) { try { - return createFactTable(this.<XFactTable>readFromXML(factSpec)); + return createFactTable(this.<XFact>readFromXML(factSpec)); } catch (JAXBException | IOException e) { return failureAPIResult(e); } @@ -478,17 +478,17 @@ public class LensMetadataClient { } } - public APIResult updateFactTable(String factName, XFactTable table) { + public APIResult updateFactTable(String factName, XFact table) { WebTarget target = getMetastoreWebTarget(); return translate(target.path("facts").path(factName) .queryParam("sessionid", this.connection.getSessionHandle()) .request(MediaType.APPLICATION_XML_TYPE) - .put(Entity.xml(new GenericEntity<JAXBElement<XFactTable>>(objFact.createXFactTable(table)){}))); + .put(Entity.xml(new GenericEntity<JAXBElement<XFact>>(objFact.createXFact(table)){}))); } public APIResult updateFactTable(String factName, String table) { try { - return updateFactTable(factName, this.<XFactTable>readFromXML(table)); + return updateFactTable(factName, this.<XFact>readFromXML(table)); } catch (JAXBException | IOException e) { return failureAPIResult(e); } http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java b/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java index 32b9db3..ed076e2 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java @@ -65,7 +65,8 @@ public enum LensCubeErrorCode { TIMELINE_ABSENT(3102, 100), EXPRESSION_NOT_PARSABLE(3103, 1500), ENTITY_NOT_FOUND(3104, 1500), - NO_PARTITION_FILTER(3105, 1500); + NO_PARTITION_FILTER(3105, 1500), + ENTITY_TYPE_NOT_AS_EXPECTED(3106, 1500); public LensErrorInfo getLensErrorInfo() { return this.errorInfo; http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java index 67aaff8..624b294 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java @@ -20,9 +20,6 @@ package org.apache.lens.cube.metadata; import java.util.*; -import org.apache.lens.server.api.error.LensException; - -import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.metadata.Table; @@ -67,7 +64,7 @@ public abstract class AbstractCubeTable implements Named { protected void addProperties() { properties.put(MetastoreConstants.TABLE_TYPE_KEY, getTableType().name()); - properties.put(MetastoreUtil.getCubeTableWeightKey(name), String.valueOf(weight)); + properties.put(MetastoreUtil.getCubeTableWeightKey(name), String.valueOf(weight())); } public String getName() { @@ -182,23 +179,6 @@ public abstract class AbstractCubeTable implements Named { return true; } - public Date getDateFromProperty(String propKey, boolean relative, boolean start) { - String prop = getProperties().get(propKey); - try { - if (StringUtils.isNotBlank(prop)) { - if (relative) { - return DateUtil.resolveRelativeDate(prop, now()); - } else { - return DateUtil.resolveAbsoluteDate(prop); - } - } - } catch (LensException e) { - log.error("unable to parse {} {} date: {}", relative ? "relative" : "absolute", start ? "start" : "end", prop); - } - return start ? DateUtil.MIN_DATE : DateUtil.MAX_DATE; - } - - @Override public String toString() { return getName(); http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java index 88bc1fc..c57a9c1 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java @@ -34,7 +34,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; @Slf4j -public class CubeFactTable extends AbstractCubeTable { +public class CubeFactTable extends AbstractCubeTable implements FactTable { @Getter // Map<StorageName, Map<update_period, storage_table_prefix>> private final Map<String, Map<UpdatePeriod, String>> storagePrefixUpdatePeriodMap; @@ -44,7 +44,7 @@ public class CubeFactTable extends AbstractCubeTable { public CubeFactTable(Table hiveTable) { super(hiveTable); this.storageUpdatePeriods = getUpdatePeriods(getName(), getProperties()); - this.cubeName = getCubeName(getName(), getProperties()); + this.cubeName = this.getProperties().get(MetastoreUtil.getFactCubeNameKey(getName())); this.storagePrefixUpdatePeriodMap = getUpdatePeriodMap(getName(), getProperties()); } @@ -77,7 +77,7 @@ public class CubeFactTable extends AbstractCubeTable { } public boolean hasColumn(String column) { - List<String> validColumns = getValidColumns(); + Set<String> validColumns = getValidColumns(); if (validColumns != null) { return validColumns.contains(column); } else { @@ -88,7 +88,7 @@ public class CubeFactTable extends AbstractCubeTable { @Override protected void addProperties() { super.addProperties(); - addCubeNames(getName(), getProperties(), cubeName); + this.getProperties().put(MetastoreUtil.getFactCubeNameKey(getName()), cubeName); addUpdatePeriodProperies(getName(), getProperties(), storageUpdatePeriods); addStorageTableProperties(getName(), getProperties(), storagePrefixUpdatePeriodMap); } @@ -115,10 +115,6 @@ public class CubeFactTable extends AbstractCubeTable { } } - private static void addCubeNames(String factName, Map<String, String> props, String cubeName) { - props.put(MetastoreUtil.getFactCubeNameKey(factName), cubeName); - } - private Map<String, Map<UpdatePeriod, String>> getUpdatePeriodMap(String factName, Map<String, String> props) { Map<String, Map<UpdatePeriod, String>> ret = new HashMap<>(); for (Map.Entry<String, Set<UpdatePeriod>> entry : storageUpdatePeriods.entrySet()) { @@ -156,10 +152,6 @@ public class CubeFactTable extends AbstractCubeTable { return storageUpdatePeriods; } - static String getCubeName(String factName, Map<String, String> props) { - return props.get(MetastoreUtil.getFactCubeNameKey(factName)); - } - public Map<String, Set<UpdatePeriod>> getUpdatePeriods() { return storageUpdatePeriods; } @@ -276,10 +268,11 @@ public class CubeFactTable extends AbstractCubeTable { * * @return */ - public List<String> getValidColumns() { + public Set<String> getValidColumns() { String validColsStr = MetastoreUtil.getNamedStringValue(getProperties(), MetastoreUtil.getValidColumnsKey(getName())); - return validColsStr == null ? null : Arrays.asList(StringUtils.split(validColsStr.toLowerCase(), ',')); + return validColsStr == null ? null : new HashSet<>(Arrays.asList(StringUtils.split(validColsStr.toLowerCase(), + ','))); } /** @@ -374,7 +367,7 @@ public class CubeFactTable extends AbstractCubeTable { */ public void alterCubeName(String cubeName) { this.cubeName = cubeName; - addCubeNames(getName(), getProperties(), cubeName); + this.getProperties().put(MetastoreUtil.getFactCubeNameKey(getName()), cubeName); } public String getDataCompletenessTag() { @@ -390,12 +383,28 @@ public class CubeFactTable extends AbstractCubeTable { getProperties().put(MetastoreConstants.FACT_AGGREGATED_PROPERTY, Boolean.toString(isAggregated)); } + @Override + public boolean isVirtualFact() { + return false; + } + + @Override + public String getSourceFactName() { + return this.getName(); + } + + public String getTablePrefix(String storage, UpdatePeriod updatePeriod) { + return storagePrefixUpdatePeriodMap.get(storage).get(updatePeriod); + } + public Date getAbsoluteStartTime() { - return getDateFromProperty(MetastoreConstants.FACT_ABSOLUTE_START_TIME, false, true); + return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.FACT_ABSOLUTE_START_TIME), + false, true); } public Date getRelativeStartTime() { - return getDateFromProperty(MetastoreConstants.FACT_RELATIVE_START_TIME, true, true); + return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.FACT_RELATIVE_START_TIME), + true, true); } public Date getStartTime() { @@ -403,18 +412,17 @@ public class CubeFactTable extends AbstractCubeTable { } public Date getAbsoluteEndTime() { - return getDateFromProperty(MetastoreConstants.FACT_ABSOLUTE_END_TIME, false, false); + return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.FACT_ABSOLUTE_END_TIME), + false, false); } public Date getRelativeEndTime() { - return getDateFromProperty(MetastoreConstants.FACT_RELATIVE_END_TIME, true, false); + return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.FACT_RELATIVE_END_TIME), + true, false); } public Date getEndTime() { return Collections.min(Lists.newArrayList(getRelativeEndTime(), getAbsoluteEndTime())); } - public String getTablePrefix(String storage, UpdatePeriod updatePeriod) { - return storagePrefixUpdatePeriodMap.get(storage).get(updatePeriod); - } } http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index b5c4c89..749e44c 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -28,17 +28,7 @@ import java.text.ParseException; import java.util.*; import java.util.concurrent.ConcurrentHashMap; -import org.apache.lens.api.metastore.XCube; -import org.apache.lens.api.metastore.XDerivedCube; -import org.apache.lens.api.metastore.XDimension; -import org.apache.lens.api.metastore.XDimensionTable; -import org.apache.lens.api.metastore.XFactTable; -import org.apache.lens.api.metastore.XSegmentation; -import org.apache.lens.api.metastore.XStorage; -import org.apache.lens.api.metastore.XStorageTableElement; -import org.apache.lens.api.metastore.XUpdatePeriod; -import org.apache.lens.api.metastore.XUpdatePeriodTableDescriptor; -import org.apache.lens.api.metastore.XUpdatePeriods; +import org.apache.lens.api.metastore.*; import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.cube.metadata.Storage.LatestInfo; import org.apache.lens.cube.metadata.Storage.LatestPartColumnInfo; @@ -67,9 +57,11 @@ import org.jvnet.jaxb2_commons.lang.Equals; import org.jvnet.jaxb2_commons.lang.HashCode; import org.jvnet.jaxb2_commons.lang.ToString; +import com.google.common.base.Optional; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; + import lombok.extern.slf4j.Slf4j; /** @@ -98,7 +90,9 @@ public class CubeMetastoreClient { private final Map<String, CubeDimensionTable> allDimTables = Maps.newConcurrentMap(); private volatile boolean allDimTablesPopulated = false; // map from fact name to fact table - private final Map<String, CubeFactTable> allFactTables = Maps.newConcurrentMap(); + private final Map<String, FactTable> allFactTables = Maps.newConcurrentMap(); + // map from fact name to all virtual fact tables, any changes to facts must reflect in all of its virtual facts + private final Map<String, List<String>> factToVirtualFactMapping = Maps.newConcurrentMap(); private volatile boolean allFactTablesPopulated = false; //map from segmentation name to segmentation private final Map<String, Segmentation> allSegmentations = Maps.newConcurrentMap(); @@ -133,8 +127,8 @@ public class CubeMetastoreClient { } /** extract storage name from fact and storage table name. String operation */ - private String extractStorageName(CubeFactTable fact, String storageTableName) throws LensException { - int ind = storageTableName.lastIndexOf(fact.getName()); + private String extractStorageName(FactTable fact, String storageTableName) throws LensException { + int ind = storageTableName.lastIndexOf(fact.getSourceFactName()); if (ind <= 0) { throw new LensException("storageTable: " + storageTableName + ", does not belong to fact: " + fact.getName()); } @@ -164,7 +158,7 @@ public class CubeMetastoreClient { String partCol = cube.getPartitionColumnOfTimeDim(timeDimension); Date max = new Date(Long.MIN_VALUE); boolean updated = false; - for (CubeFactTable fact : getAllFacts(cube)) { + for (FactTable fact : getAllFacts(cube)) { for (String storage : fact.getStorages()) { for (UpdatePeriod updatePeriod : fact.getUpdatePeriods().get(storage)) { PartitionTimeline timeline = partitionTimelineCache.get(fact.getName(), storage, updatePeriod, partCol); @@ -191,7 +185,7 @@ public class CubeMetastoreClient { throws LensException, HiveException { UpdatePeriod updatePeriod = updatePeriodStr == null ? null : UpdatePeriod.valueOf(updatePeriodStr.toUpperCase()); List<PartitionTimeline> ret = Lists.newArrayList(); - CubeFactTable fact = getCubeFact(factName); + CubeFactTable fact = getCubeFactTable(factName); List<String> storageList = Lists.newArrayList(); if (storage != null) { storageList.add(storage); @@ -315,7 +309,7 @@ public class CubeMetastoreClient { properties, storageUpdatePeriodMap); createCubeTable(factTable, storageTableDescs); // do a get to update cache - getCubeFact(factName); + getFactTable(factName); } @@ -326,8 +320,8 @@ public class CubeMetastoreClient { createCube((XCube)entity); } else if (entity instanceof XDimension) { createDimension((XDimension) entity); - } else if (entity instanceof XFactTable) { - createCubeFactTable((XFactTable) entity); + } else if (entity instanceof XFact) { + createFactTable((XFact) entity); } else if (entity instanceof XDimensionTable) { createCubeDimensionTable((XDimensionTable) entity); } else if (entity instanceof XSegmentation) { @@ -345,8 +339,8 @@ public class CubeMetastoreClient { alterCube((XCube)entity); } else if (entity instanceof XDimension) { alterDimension((XDimension) entity); - } else if (entity instanceof XFactTable) { - alterCubeFactTable((XFactTable) entity); + } else if (entity instanceof XFact) { + alterCubeFactTable((XFact) entity); } else if (entity instanceof XDimensionTable) { alterCubeDimensionTable((XDimensionTable) entity); } else if (entity instanceof XSegmentation) { @@ -356,24 +350,45 @@ public class CubeMetastoreClient { } } - public static Map<String, String> addFactColStartTimePropertyToFactProperties(XFactTable fact) { Map<String, String> props = new HashMap<String, String>(); props.putAll(JAXBUtils.mapFromXProperties(fact.getProperties())); props.putAll(JAXBUtils.columnStartAndEndTimeFromXColumns(fact.getColumns())); return props; } - public void createCubeFactTable(XFactTable fact) throws LensException { - createCubeFactTable(fact.getCubeName(), - fact.getName(), - JAXBUtils.fieldSchemaListFromColumns(fact.getColumns()), - JAXBUtils.getFactUpdatePeriodsFromStorageTables(fact.getStorageTables()), - fact.getWeight(), - addFactColStartTimePropertyToFactProperties(fact), - JAXBUtils.tableDescPrefixMapFromXStorageTables(fact.getStorageTables()), - JAXBUtils.storageTablePrefixMapOfStorage(fact.getStorageTables())); + + public void createFactTable(XFact fact) throws LensException { + + if (fact instanceof XVirtualFactTable) { + XVirtualFactTable xvf = (XVirtualFactTable) fact; + createVirtualFactTable(xvf.getCubeName(), xvf.getName(), xvf.getSourceFactName(), + xvf.getWeight(), JAXBUtils.mapFromXProperties(xvf.getProperties())); + } else { + XFactTable xf = (XFactTable) fact; + createCubeFactTable(fact.getCubeName(), + fact.getName(), + JAXBUtils.fieldSchemaListFromColumns(xf.getColumns()), + JAXBUtils.getFactUpdatePeriodsFromStorageTables(xf.getStorageTables()), + xf.getWeight(), + addFactColStartTimePropertyToFactProperties(xf), + JAXBUtils.tableDescPrefixMapFromXStorageTables(xf.getStorageTables()), + JAXBUtils.storageTablePrefixMapOfStorage(xf.getStorageTables())); + } } + public void createVirtualFactTable(String cubeName, String virtualFactName, String sourceFactName, Double weight, + Map<String, String> properties) throws LensException { + FactTable sourceFact = getFactTable(sourceFactName); + + Optional<Double> optionalWeight = Optional.fromNullable(weight); + + CubeVirtualFactTable factTable = new CubeVirtualFactTable(cubeName, virtualFactName, + optionalWeight, properties, sourceFact); + createCubeTable(factTable, null); + // do a get to update cache + getFactTable(virtualFactName); + + } /** * In-memory storage of {@link PartitionTimeline} objects for each valid @@ -442,7 +457,7 @@ public class CubeMetastoreClient { private void loadTimeLines(String fact, String storage, String timeLineKey) throws LensException, HiveException { Set<String> uniqueStorageTables = new HashSet<>(); Map<UpdatePeriod, String> updatePeriodTableName = new HashMap<>(); - for (UpdatePeriod updatePeriod : getCubeFact(fact).getUpdatePeriods().get(storage)) { + for (UpdatePeriod updatePeriod : getFactTable(fact).getUpdatePeriods().get(storage)) { String storageTableName = getStorageTableName(fact, storage, updatePeriod); updatePeriodTableName.put(updatePeriod, storageTableName); Table storageTable = getTable(storageTableName); @@ -477,7 +492,7 @@ public class CubeMetastoreClient { // Not found in table properties either, compute from all partitions of the fact-storage table. // First make sure all combinations of update period and partition column have an entry even // if no partitions exist - if (getCubeFact(fact).getUpdatePeriods() != null && getCubeFact(fact).getUpdatePeriods().get(storage) != null) { + if (getFactTable(fact).getUpdatePeriods() != null && getFactTable(fact).getUpdatePeriods().get(storage) != null) { log.info("loading from all partitions: {}", storageTableName); Table storageTable = getTable(storageTableName); for (String partCol : getTimePartColNamesOfTable(storageTable)) { @@ -848,7 +863,7 @@ public class CubeMetastoreClient { new CubeFactTable(cubeName, factName, columns, storageAggregatePeriods, weight, properties); createCubeTable(factTable, storageTableDescs); // do a get to update cache - getCubeFact(factName); + getFactTable(factName); } /** @@ -1089,7 +1104,7 @@ public class CubeMetastoreClient { public Date getStorageTableEndDate(String storageTable, String factTableName) throws LensException { List<Date> endDates = getStorageTimes(storageTable, MetastoreUtil.getStoragetableEndTimesKey()); - endDates.add(getFactTable(factTableName).getEndTime()); + endDates.add((getFactTable(factTableName)).getEndTime()); return Collections.min(endDates); } @@ -1408,11 +1423,11 @@ public class CubeMetastoreClient { } /** extract storage name and check in timeline cache for existance */ - public boolean factPartitionExists(CubeFactTable fact, FactPartition part, String storageTableName) + public boolean factPartitionExists(FactTable fact, FactPartition part, String storageTableName) throws HiveException, LensException { String storage = extractStorageName(fact, storageTableName); - return partitionTimelineCache.partitionTimeExists(fact.getName(), storage, part.getPeriod(), part.getPartCol(), - part.getPartSpec()); + return partitionTimelineCache.partitionTimeExists(fact.getSourceFactName(), storage, + part.getPeriod(), part.getPartCol(), part.getPartSpec()); } public boolean factPartitionExists(String factName, String storageName, UpdatePeriod updatePeriod, @@ -1638,12 +1653,35 @@ public class CubeMetastoreClient { boolean isFactTable(Table tbl) { String tableType = tbl.getParameters().get(MetastoreConstants.TABLE_TYPE_KEY); - return CubeTableType.FACT.name().equals(tableType); + return CubeTableType.FACT.name().equals(tableType) + && tbl.getParameters().get(getSourceFactNameKey(tbl.getTableName())) == null; } - boolean isFactTableForCube(Table tbl, String cube) { - return isFactTable(tbl) && CubeFactTable.getCubeName(tbl.getTableName(), tbl.getParameters()) + return isFactTable(tbl) && tbl.getParameters().get(MetastoreUtil.getFactCubeNameKey(tbl.getTableName())) + .equalsIgnoreCase(cube.toLowerCase()); + } + + /** + * Is the table name passed a virtual fact table? + * + * @param virtualTableName table name + * @return true if it is virtual fact, false otherwise + * @throws HiveException + */ + public boolean isVirtualFactTable(String virtualTableName) throws LensException { + Table tbl = getTable(virtualTableName); + return isVirtualFactTable(tbl); + } + + boolean isVirtualFactTable(Table tbl) { + String tableType = tbl.getParameters().get(MetastoreConstants.TABLE_TYPE_KEY); + return CubeTableType.FACT.name().equals(tableType) + && tbl.getParameters().get(getSourceFactNameKey(tbl.getTableName())) != null; + } + + boolean isVirtualFactTableForCube(Table tbl, String cube) { + return isVirtualFactTable(tbl) && tbl.getParameters().get(MetastoreUtil.getFactCubeNameKey(tbl.getTableName())) .equalsIgnoreCase(cube.toLowerCase()); } @@ -1724,58 +1762,57 @@ public class CubeMetastoreClient { return CubeTableType.DIMENSION.name().equals(tableType); } - public XFactTable getXFactTable(String tableName) throws LensException { + public XFact getXFactTable(String tableName) throws LensException { return getXFactTable(getFactTable(tableName)); } - public XFactTable getXFactTable(CubeFactTable cft) throws LensException { + public XFact getXFactTable(FactTable ft) throws LensException { - XFactTable factTable = JAXBUtils.factTableFromCubeFactTable(cft); - Map<String, Map<UpdatePeriod, String>> storageMap = cft.getStoragePrefixUpdatePeriodMap(); - for (String storageName : cft.getStorages()) { - Set<UpdatePeriod> updatePeriods = cft.getUpdatePeriods().get(storageName); - // This map tells if there are different tables for different update period. - Map<UpdatePeriod, String> updatePeriodToTableMap = storageMap.get(storageName); - Set<String> tableNames = new HashSet<>(); - for (UpdatePeriod updatePeriod : updatePeriods) { - tableNames.add(updatePeriodToTableMap.get(updatePeriod)); - } - if (tableNames.size() <= 1) { - XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( - getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(), storageName))); - tblElement.setStorageName(storageName); - for (UpdatePeriod p : updatePeriods) { - tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); + XFact fact; + if (ft.isVirtualFact()) { + CubeVirtualFactTable cvft = (CubeVirtualFactTable) ft; + XVirtualFactTable factTable = JAXBUtils.virtualFactTableFromVirtualCubeFactTable(cvft); + factTable.setSourceFactName(cvft.getSourceCubeFactTable().getName()); + fact = factTable; + } else { + CubeFactTable cft = (CubeFactTable) ft; + XFactTable factTable = JAXBUtils.factTableFromCubeFactTable(cft); + Map<String, Map<UpdatePeriod, String>> storageMap = cft.getStoragePrefixUpdatePeriodMap(); + for (String storageName : cft.getStorages()) { + Set<UpdatePeriod> updatePeriods = cft.getUpdatePeriods().get(storageName); + // This map tells if there are different tables for different update period. + Map<UpdatePeriod, String> updatePeriodToTableMap = storageMap.get(storageName); + Set<String> tableNames = new HashSet<>(); + for (UpdatePeriod updatePeriod : updatePeriods) { + tableNames.add(updatePeriodToTableMap.get(updatePeriod)); } - factTable.getStorageTables().getStorageTable().add(tblElement); - } else { - // Multiple storage tables. - XStorageTableElement tblElement = new XStorageTableElement(); - tblElement.setStorageName(storageName); - XUpdatePeriods xUpdatePeriods = new XUpdatePeriods(); - tblElement.setUpdatePeriods(xUpdatePeriods); - for (Map.Entry entry : updatePeriodToTableMap.entrySet()) { - XUpdatePeriodTableDescriptor updatePeriodTableDescriptor = new XUpdatePeriodTableDescriptor(); - updatePeriodTableDescriptor.setTableDesc(getStorageTableDescFromHiveTable( - this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(), + if (tableNames.size() <= 1) { + XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( + getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(), storageName))); + tblElement.setStorageName(storageName); + for (UpdatePeriod p : updatePeriods) { + tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); + } + factTable.getStorageTables().getStorageTable().add(tblElement); + } else { + // Multiple storage tables. + XStorageTableElement tblElement = new XStorageTableElement(); + tblElement.setStorageName(storageName); + XUpdatePeriods xUpdatePeriods = new XUpdatePeriods(); + tblElement.setUpdatePeriods(xUpdatePeriods); + for (Map.Entry entry : updatePeriodToTableMap.entrySet()) { + XUpdatePeriodTableDescriptor updatePeriodTableDescriptor = new XUpdatePeriodTableDescriptor(); + updatePeriodTableDescriptor.setTableDesc(getStorageTableDescFromHiveTable( + this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(), (String) entry.getValue())))); - updatePeriodTableDescriptor.setUpdatePeriod(XUpdatePeriod.valueOf(((UpdatePeriod)entry.getKey()).name())); - xUpdatePeriods.getUpdatePeriodTableDescriptor().add(updatePeriodTableDescriptor); + updatePeriodTableDescriptor.setUpdatePeriod(XUpdatePeriod.valueOf(((UpdatePeriod) entry.getKey()).name())); + xUpdatePeriods.getUpdatePeriodTableDescriptor().add(updatePeriodTableDescriptor); + } + factTable.getStorageTables().getStorageTable().add(tblElement); } - factTable.getStorageTables().getStorageTable().add(tblElement); } + fact = factTable; } - return factTable; - } - /** - * Get {@link CubeFactTable} object corresponding to the name - * - * @param tableName The cube fact name - * @return Returns CubeFactTable if table name passed is a fact table, null otherwise - * @throws LensException - */ - - public CubeFactTable getFactTable(String tableName) throws LensException { - return new CubeFactTable(getTableWithTypeFailFast(tableName, CubeTableType.FACT)); + return fact; } public Segmentation getSegmentationTable(String tableName) throws HiveException, LensException { @@ -1955,17 +1992,32 @@ public class CubeMetastoreClient { * @return Returns cube is table name passed is a cube * @throws LensException if there is no cube by the name */ - public CubeFactTable getCubeFact(String tableName) throws LensException { - return getCubeFact(tableName, true); + public FactTable getFactTable(String tableName) throws LensException { + return getFactTable(tableName, true); } - private CubeFactTable getCubeFact(String tableName, boolean throwException) throws LensException { + private FactTable getFactTable(String tableName, boolean throwException) throws LensException { tableName = tableName.trim().toLowerCase(); - CubeFactTable fact = allFactTables.get(tableName); + FactTable fact = allFactTables.get(tableName); if (fact == null) { synchronized (allFactTables) { if (!allFactTables.containsKey(tableName)) { Table tbl = getTableWithType(tableName, CubeTableType.FACT, throwException); - fact = tbl == null ? null : new CubeFactTable(tbl); + if (tbl != null){ + String sourceFactName = tbl.getParameters().get(getSourceFactNameKey(tbl.getTableName())); + if (sourceFactName != null) { + fact = new CubeVirtualFactTable(tbl, getCubeFactTable(sourceFactName)); + if (factToVirtualFactMapping.get(sourceFactName) != null) { + List<String> prevList = factToVirtualFactMapping.get(sourceFactName); + prevList.add(tableName); + }else{ + List<String> newList = new ArrayList<>(); + newList.add(tableName); + factToVirtualFactMapping.put(sourceFactName, newList); + } + } else { + fact = new CubeFactTable(tbl); + } + } if (enableCaching && fact != null) { allFactTables.put(tableName, fact); } @@ -1977,6 +2029,15 @@ public class CubeMetastoreClient { return fact; } + private FactTable getFactTable(Table tbl) throws LensException { + String sourceFact = tbl.getParameters().get(getSourceFactNameKey(tbl.getTableName())); + if (sourceFact != null) { + return new CubeVirtualFactTable(tbl, getCubeFactTable(sourceFact)); + } else { + return new CubeFactTable(tbl); + } + } + public Segmentation getSegmentation(String segName) throws LensException { return getSegmentation(segName, true); } @@ -2123,12 +2184,12 @@ public class CubeMetastoreClient { * @return List of Cube Fact Table objects * @throws LensException */ - public Collection<CubeFactTable> getAllFacts() throws LensException { + public Collection<FactTable> getAllFacts() throws LensException { if (!allFactTablesPopulated) { - List<CubeFactTable> facts = new ArrayList<>(); + List<FactTable> facts = new ArrayList<>(); try { for (String table : getAllHiveTableNames()) { - CubeFactTable fact = getCubeFact(table, false); + FactTable fact = getFactTable(table, false); if (fact != null) { facts.add(fact); } @@ -2144,6 +2205,38 @@ public class CubeMetastoreClient { } /** + * Get all facts in metastore (virtual facts optional) + * @param includeVirtualFacts set true for including virtual facts + * @return List of Cube Fact Table objects + * @throws LensException + */ + public Collection<FactTable> getAllFacts(boolean includeVirtualFacts) throws LensException { + if (!allFactTablesPopulated) { + List<FactTable> facts = new ArrayList<>(); + try { + for (String table : getAllHiveTableNames()) { + FactTable fact = getFactTable(table, false); + if (fact != null) { + if (fact.getProperties().get(getSourceFactNameKey(fact.getName())) != null) { //is virtual fact + if (includeVirtualFacts) { + facts.add(fact); + } + } else { + facts.add(fact); + } + } + } + } catch (HiveException e) { + throw new LensException("Could not get all fact tables", e); + } + allFactTablesPopulated = enableCaching; + return facts; + } else { + return allFactTables.values(); + } + } + + /** * Get all segmentations in metastore * * @return List of segmentation objects @@ -2169,8 +2262,6 @@ public class CubeMetastoreClient { } } - - private Collection<String> getAllHiveTableNames() throws HiveException, LensException { if (!allTablesPopulated) { List<String> allTables = getClient().getAllTables(); @@ -2192,7 +2283,7 @@ public class CubeMetastoreClient { * @return List of fact tables * @throws LensException */ - public List<CubeFactTable> getAllFacts(CubeInterface cube) throws LensException { + public List<FactTable> getAllFacts(CubeInterface cube) throws LensException { String cubeName = null; if (cube != null) { if (cube instanceof DerivedCube) { @@ -2200,8 +2291,33 @@ public class CubeMetastoreClient { } cubeName = cube.getName(); } - List<CubeFactTable> cubeFacts = new ArrayList<>(); - for (CubeFactTable fact : getAllFacts()) { + List<FactTable> cubeFacts = new ArrayList<>(); + for (FactTable fact : getAllFacts()) { + if (cubeName == null || fact.getCubeName().equalsIgnoreCase(cubeName)) { + cubeFacts.add(fact); + } + } + return cubeFacts; + } + + /** + * Get all facts of cube (optional virtual facts) + * + * @param cube Cube object + * @param includeVirtualFacts set true for virtual facts + * @return List of fact tables with optional virtual facts + * @throws LensException + */ + public List<FactTable> getAllFacts(CubeInterface cube, boolean includeVirtualFacts) throws LensException { + String cubeName = null; + if (cube != null) { + if (cube instanceof DerivedCube) { + cube = ((DerivedCube) cube).getParent(); + } + cubeName = cube.getName(); + } + List<FactTable> cubeFacts = new ArrayList<>(); + for (FactTable fact : getAllFacts(includeVirtualFacts)) { if (cubeName == null || fact.getCubeName().equalsIgnoreCase(cubeName)) { cubeFacts.add(fact); } @@ -2262,8 +2378,8 @@ public class CubeMetastoreClient { return dimTables; } - public boolean partColExists(String fact, String storage, String partCol) throws LensException { - for (String storageTable : getStorageTables(fact, storage)) { + public boolean partColExists(FactTable factTable, String storage, String partCol) throws LensException { + for (String storageTable : getStorageTables(factTable, storage)) { for (FieldSchema f : getTable(storageTable).getPartCols()) { if (f.getName().equalsIgnoreCase(partCol)) { return true; @@ -2278,20 +2394,21 @@ public class CubeMetastoreClient { * Note: If each update period in the storage has a different storage table, this method will return N Storage Tables * where N is the number of update periods in the storage (LENS-1386) * - * @param fact + * @param factTable * @param storage * @return * @throws LensException */ - public Set<String> getStorageTables(String fact, String storage) throws LensException { + public Set<String> getStorageTables(FactTable factTable, String storage) throws LensException { Set<String> uniqueStorageTables = new HashSet<>(); - for (UpdatePeriod updatePeriod : getFactTable(fact).getUpdatePeriods().get(storage)) { - uniqueStorageTables.add(getStorageTableName(fact, storage, updatePeriod)); + + for (UpdatePeriod updatePeriod : factTable.getUpdatePeriods().get(storage)) { + String factName = factTable.getSourceFactName(); + uniqueStorageTables.add(getStorageTableName(factName, storage, updatePeriod)); } return uniqueStorageTables; } - /** * * @param table table name @@ -2431,7 +2548,7 @@ public class CubeMetastoreClient { */ public void dropFact(String factName, boolean cascade) throws LensException { getTableWithTypeFailFast(factName, CubeTableType.FACT); - CubeFactTable fact = getFactTable(factName); + FactTable fact = getFactTable(factName); if (cascade) { for (String storage : fact.getStorages()) { dropStorageFromFact(factName, storage, false); @@ -2439,8 +2556,48 @@ public class CubeMetastoreClient { } dropHiveTable(factName); allFactTables.remove(factName.trim().toLowerCase()); + if (fact.isVirtualFact()) { + String sourceFactTable = fact.getProperties().get(getSourceFactNameKey(fact.getName())); + if (factToVirtualFactMapping.get(sourceFactTable) != null + && factToVirtualFactMapping.get(sourceFactTable).contains(fact.getName())) { + factToVirtualFactMapping.get(sourceFactTable).remove(fact.getName()); + } + } else { + dropAllVirtualFactTables(factName); + } } + private void dropAllVirtualFactTables(String cubeFactTableName) throws LensException { + if (enableCaching) { + cubeFactTableName = cubeFactTableName.trim().toLowerCase(); + if (factToVirtualFactMapping.get(cubeFactTableName) != null) { + List<String> virtualFactTableNames = factToVirtualFactMapping.get(cubeFactTableName); + factToVirtualFactMapping.remove(cubeFactTableName); + for (String vf : virtualFactTableNames) { + dropVirtualFact(vf.trim().toLowerCase()); + } + } + } + } + + /** + * Drop a virtual fact + * + * @param virtualFactName virtual fact name + * @throws LensException + */ + public void dropVirtualFact(String virtualFactName) throws LensException { + virtualFactName = virtualFactName.trim().toLowerCase(); + Table virtualTbl = getTableWithTypeFailFast(virtualFactName, CubeTableType.FACT); + String sourceFactTable = virtualTbl.getParameters().get(getSourceFactNameKey(virtualTbl.getTableName())); + if (factToVirtualFactMapping.get(sourceFactTable) != null + && factToVirtualFactMapping.get(sourceFactTable).contains(virtualFactName)) { + factToVirtualFactMapping.get(sourceFactTable).remove(virtualFactName); + } + dropHiveTable(virtualFactName); + allFactTables.remove(virtualFactName.trim().toLowerCase()); + + } public void dropSegmentation(String segName) throws LensException { getTableWithTypeFailFast(segName, CubeTableType.SEGMENTATION); @@ -2456,7 +2613,7 @@ public class CubeMetastoreClient { * @throws LensException */ public void dropStorageFromFact(String factName, String storage) throws LensException { - CubeFactTable cft = getFactTable(factName); + CubeFactTable cft = getCubeFactTable(factName); dropHiveTablesForStorage(factName, storage); cft.dropStorage(storage); alterCubeTable(factName, getTableWithTypeFailFast(factName, CubeTableType.FACT), cft); @@ -2464,7 +2621,7 @@ public class CubeMetastoreClient { } private void dropHiveTablesForStorage(String factName, String storage) throws LensException{ - CubeFactTable cft = getFactTable(factName); + CubeFactTable cft = getCubeFactTable(factName); Set<String> droppedTables = new HashSet<>(); for (Map.Entry updatePeriodEntry : cft.getStoragePrefixUpdatePeriodMap().get(storage).entrySet()) { UpdatePeriod updatePeriod = (UpdatePeriod) updatePeriodEntry.getKey(); @@ -2480,7 +2637,7 @@ public class CubeMetastoreClient { throws LensException { dropHiveTablesForStorage(factName, storage); if (updateFact) { - CubeFactTable cft = getFactTable(factName); + CubeFactTable cft = getCubeFactTable(factName); cft.dropStorage(storage); alterCubeTable(factName, getTableWithTypeFailFast(factName, CubeTableType.FACT), cft); updateFactCache(factName); @@ -2530,11 +2687,20 @@ public class CubeMetastoreClient { dropHiveTable(dimTblName); allDimTables.remove(dimTblName.trim().toLowerCase()); } - public void alterCubeFactTable(XFactTable fact) throws LensException, HiveException { - alterCubeFactTable(fact.getName(), JAXBUtils.cubeFactFromFactTable(fact), - JAXBUtils.tableDescPrefixMapFromXStorageTables(fact.getStorageTables()), - JAXBUtils.columnStartAndEndTimeFromXColumns(fact.getColumns())); + + public void alterCubeFactTable(XFact fact) throws LensException, HiveException { + if (fact instanceof XVirtualFactTable) { + XVirtualFactTable xvf = (XVirtualFactTable) fact; + alterCubeFactTable(xvf.getName(), JAXBUtils.cubeVirtualFactFromFactTable(xvf, + getFactTable(xvf.getSourceFactName())), null, new HashMap<>()); + } else { + XFactTable xf = (XFactTable) fact; + alterCubeFactTable(fact.getName(), JAXBUtils.cubeFactFromFactTable(xf), + JAXBUtils.tableDescPrefixMapFromXStorageTables(xf.getStorageTables()), + JAXBUtils.columnStartAndEndTimeFromXColumns(xf.getColumns())); + } } + /** * Alter a cubefact with new definition and alter underlying storage tables as well. * @@ -2544,7 +2710,7 @@ public class CubeMetastoreClient { * * @throws HiveException */ - public void alterCubeFactTable(String factTableName, CubeFactTable cubeFactTable, + public void alterCubeFactTable(String factTableName, FactTable cubeFactTable, Map<String, StorageTableDesc> storageTableDescs, Map<String, String> props) throws HiveException, LensException { @@ -2552,7 +2718,7 @@ public class CubeMetastoreClient { if (!props.isEmpty()) { cubeFactTable.getProperties().putAll(props); } - alterCubeTable(factTableName, factTbl, cubeFactTable); + alterCubeTable(factTableName, factTbl, (AbstractCubeTable) cubeFactTable); if (storageTableDescs != null) { // create/alter tables for each storage for (Map.Entry<String, StorageTableDesc> entry : storageTableDescs.entrySet()) { @@ -2560,11 +2726,24 @@ public class CubeMetastoreClient { } } updateFactCache(factTableName); + + updateAllVirtualFacts(getFactTable(factTableName)); } public void alterSegmentation(XSegmentation cubeSeg) throws LensException, HiveException { alterSegmentation(cubeSeg.getName(), segmentationFromXSegmentation(cubeSeg)); } + + /** + * Alter a virtual cube fact with new definition + * @param cubeVirtualFactTable cube virtual fact table + * @throws HiveException + */ + public void alterVirtualCubeFactTable(CubeVirtualFactTable cubeVirtualFactTable) + throws HiveException, LensException { + alterCubeFactTable(cubeVirtualFactTable.getName(), cubeVirtualFactTable, null, new HashMap<>()); + } + public void alterSegmentation(String segName, Segmentation seg) throws HiveException, LensException { getTableWithTypeFailFast(segName, CubeTableType.SEGMENTATION); @@ -2583,7 +2762,52 @@ public class CubeMetastoreClient { private void updateFactCache(String factTableName) throws LensException { if (enableCaching) { - allFactTables.put(factTableName.trim().toLowerCase(), new CubeFactTable(refreshTable(factTableName))); + Table factTbl = getTableWithTypeFailFast(factTableName, CubeTableType.FACT); + FactTable refreshedTable; + if (factTbl.getParameters().get(getSourceFactNameKey(factTableName)) != null){ + String sourceFactName = factTbl.getParameters().get(getSourceFactNameKey(factTableName)); + refreshedTable = new CubeVirtualFactTable(refreshTable(factTableName), + getCubeFactTable(sourceFactName)); + }else { + refreshedTable = new CubeFactTable(refreshTable(factTableName)); + } + allFactTables.put(factTableName.trim().toLowerCase(), refreshedTable); + } + } + + public CubeFactTable getCubeFactTable(String factName) throws LensException { + FactTable factTable = getFactTable(factName); + if (factTable instanceof CubeFactTable) { + return (CubeFactTable) factTable; + } else { + throw new LensException(new LensException(LensCubeErrorCode.ENTITY_TYPE_NOT_AS_EXPECTED.getLensErrorInfo(), + factName, "Fact")); + } + } + + public CubeVirtualFactTable getCubeVirtualFactTable(String factName) throws LensException { + FactTable factTable = getFactTable(factName); + if (factTable instanceof CubeVirtualFactTable) { + return (CubeVirtualFactTable) factTable; + } else { + throw new LensException(new LensException(LensCubeErrorCode.ENTITY_TYPE_NOT_AS_EXPECTED.getLensErrorInfo(), + factName, "VirtualFact")); + } + } + + private void updateAllVirtualFacts(FactTable cubeFactTable) throws LensException { + if (enableCaching) { + String cubeFactTableName = cubeFactTable.getName().trim().toLowerCase(); + if (factToVirtualFactMapping.get(cubeFactTableName) != null) { + synchronized (allFactTables) { + List<String> virtualFactTableNames = factToVirtualFactMapping.get(cubeFactTableName); + for (String vf : virtualFactTableNames) { + CubeVirtualFactTable cvf = getCubeVirtualFactTable(vf); + cvf.setSourceCubeFactTable(cubeFactTable); + allFactTables.put(vf.trim().toLowerCase(), cvf); + } + } + } } } @@ -2693,8 +2917,11 @@ public class CubeMetastoreClient { if (updatePeriod == null) { return storage; } - if (isFactTable(factOrDimTableName)) { - return getFactTable(factOrDimTableName).getTablePrefix(storage, updatePeriod); + if (isVirtualFactTable(factOrDimTableName)) { + CubeFactTable sourceFact = (CubeFactTable) getCubeVirtualFactTable(factOrDimTableName).getSourceCubeFactTable(); + return sourceFact.getTablePrefix(storage, updatePeriod); + } else if (isFactTable(factOrDimTableName)) { + return getCubeFactTable(factOrDimTableName).getTablePrefix(storage, updatePeriod); } else { return storage; } http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java new file mode 100644 index 0000000..1fc74b0 --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java @@ -0,0 +1,186 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.metadata; + +import java.util.*; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.metadata.Table; + +import com.google.common.base.Optional; + +import com.google.common.collect.Lists; +import lombok.Getter; +import lombok.Setter; + +public class CubeVirtualFactTable extends AbstractCubeTable implements FactTable { + + @Getter + @Setter + private FactTable sourceCubeFactTable; + private String cubeName; + private static final List<FieldSchema> COLUMNS = new ArrayList<FieldSchema>(); + @Getter + private Optional<Double> virtualFactWeight = Optional.absent(); + + static { + COLUMNS.add(new FieldSchema("dummy", "string", "dummy column")); + } + + public CubeVirtualFactTable(Table hiveTable, FactTable sourceCubeFactTable) { + super(hiveTable); + this.cubeName = this.getProperties().get(MetastoreUtil.getFactCubeNameKey(getName())); + this.sourceCubeFactTable = sourceCubeFactTable; + + String wtStr = getProperties().get(MetastoreUtil.getCubeTableWeightKey(getName())); + if (wtStr != null) { + this.virtualFactWeight = Optional.of(Double.parseDouble(wtStr)); + } + } + + public CubeVirtualFactTable(String cubeName, String virtualFactName, Optional<Double> weight, + Map<String, String> properties, FactTable sourceFact) { + super(virtualFactName, COLUMNS, properties, weight.isPresent() ? weight.get() : sourceFact.weight()); + this.cubeName = cubeName; + this.virtualFactWeight = weight; + this.sourceCubeFactTable = sourceFact; + addProperties(); + } + + /** + * Alters the weight of table + * + * @param weight Weight of the table. + */ + @Override + public void alterWeight(double weight) { + this.virtualFactWeight = Optional.of(weight); + this.addProperties(); + } + + @Override + protected void addProperties() { + getProperties().put(MetastoreConstants.TABLE_TYPE_KEY, getTableType().name()); + getProperties().put(MetastoreUtil.getSourceFactNameKey(this.getName()), this.sourceCubeFactTable.getName()); + if (virtualFactWeight.isPresent()) { + getProperties().put(MetastoreUtil.getCubeTableWeightKey(this.getName()), String.valueOf(virtualFactWeight.get())); + } + this.getProperties().put(MetastoreUtil.getFactCubeNameKey(getName()), cubeName); + } + + @Override + public CubeTableType getTableType() { + return CubeTableType.FACT; + } + + @Override + public Set<String> getValidColumns() { + return this.sourceCubeFactTable.getValidColumns(); + } + + @Override + public Set<String> getStorages() { + return this.sourceCubeFactTable.getStorages(); + } + + @Override + public Map<String, Set<UpdatePeriod>> getUpdatePeriods() { + return this.sourceCubeFactTable.getUpdatePeriods(); + } + + @Override + public String getCubeName() { + return this.cubeName; + } + + @Override + public String getDataCompletenessTag() { + return this.sourceCubeFactTable.getDataCompletenessTag(); + } + + @Override + public boolean isAggregated() { + return this.sourceCubeFactTable.isAggregated(); + } + + @Override + public List<FieldSchema> getColumns() { + return this.sourceCubeFactTable.getColumns(); + } + + @Override + public double weight() { + return virtualFactWeight.isPresent() ? virtualFactWeight.get() : sourceCubeFactTable.weight(); + } + + public Date getAbsoluteStartTime() { + String absoluteStartTime = this.getProperties().get(MetastoreConstants.FACT_ABSOLUTE_START_TIME); + Date absoluteDate = null; + if (StringUtils.isNotBlank(absoluteStartTime)) { + absoluteDate = MetastoreUtil.getDateFromProperty(absoluteStartTime, false, true); + } + return absoluteDate == null ? this.sourceCubeFactTable.getAbsoluteStartTime() : absoluteDate; + } + + public Date getRelativeStartTime() { + String relativeStartTime = this.getProperties().get(MetastoreConstants.FACT_ABSOLUTE_START_TIME); + Date relativeDate = null; + if (StringUtils.isNotBlank(relativeStartTime)) { + relativeDate = MetastoreUtil.getDateFromProperty(relativeStartTime, true, true); + } + return relativeDate == null ? this.sourceCubeFactTable.getRelativeStartTime() : relativeDate; + } + + public Date getStartTime() { + return Collections.max(Lists.newArrayList(getRelativeStartTime(), getAbsoluteStartTime())); + } + + public Date getAbsoluteEndTime() { + String absoluteEndTime = this.getProperties().get(MetastoreConstants.FACT_ABSOLUTE_END_TIME); + Date absoluteDate = null; + if (StringUtils.isNotBlank(absoluteEndTime)) { + absoluteDate = MetastoreUtil.getDateFromProperty(absoluteEndTime, false, false); + } + return absoluteDate == null ? this.sourceCubeFactTable.getAbsoluteEndTime() : absoluteDate; + } + + public Date getRelativeEndTime() { + String relativeEndTime = this.getProperties().get(MetastoreConstants.FACT_RELATIVE_END_TIME); + Date relativeDate = null; + if (StringUtils.isNotBlank(relativeEndTime)) { + relativeDate = MetastoreUtil.getDateFromProperty(relativeEndTime, true, false); + } + return relativeDate == null ? this.sourceCubeFactTable.getRelativeEndTime() : relativeDate; + } + + public Date getEndTime() { + return Collections.min(Lists.newArrayList(getRelativeEndTime(), getAbsoluteEndTime())); + } + + @Override + public boolean isVirtualFact() { + return true; + } + + @Override + public String getSourceFactName() { + return this.sourceCubeFactTable.getName(); + } +}