LENS-735 : Remove accepting TableReferences for ReferenceDimAttribute
Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/908530f5 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/908530f5 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/908530f5 Branch: refs/heads/current-release-line Commit: 908530f5883ae8844c6a16cb5564c926cc10bf19 Parents: c73d584 Author: Amareshwari Sriramadasu <amareshw...@gmail.com> Authored: Mon Jan 11 13:24:32 2016 +0530 Committer: Rajat Khandelwal <rajatgupt...@gmail.com> Committed: Mon Jan 11 13:24:33 2016 +0530 ---------------------------------------------------------------------- lens-api/src/main/resources/cube-0.1.xsd | 36 +- lens-api/src/main/resources/lens-errors.conf | 11 +- .../lens/cli/TestLensDimensionCommands.java | 28 +- .../resources/cube_with_no_weight_facts.xml | 8 +- lens-cli/src/test/resources/sample-cube.xml | 14 +- lens-cli/src/test/resources/test-dimension.xml | 22 +- .../lens/cube/error/LensCubeErrorCode.java | 6 +- .../lens/cube/metadata/AbstractBaseTable.java | 53 +- .../lens/cube/metadata/AbstractCubeTable.java | 26 +- .../org/apache/lens/cube/metadata/Cube.java | 111 ++- .../apache/lens/cube/metadata/CubeColumn.java | 1 - .../lens/cube/metadata/CubeDimensionTable.java | 21 +- .../lens/cube/metadata/CubeFactTable.java | 15 +- .../lens/cube/metadata/CubeMetastoreClient.java | 11 +- .../apache/lens/cube/metadata/DerivedCube.java | 31 +- .../apache/lens/cube/metadata/Dimension.java | 20 +- .../apache/lens/cube/metadata/JoinChain.java | 16 +- .../cube/metadata/ReferencedDimAtrribute.java | 195 ----- .../cube/metadata/ReferencedDimAttribute.java | 115 +++ .../apache/lens/cube/metadata/SchemaGraph.java | 377 --------- .../lens/cube/metadata/join/JoinPath.java | 101 +++ .../cube/metadata/join/TableRelationship.java | 46 + .../apache/lens/cube/parse/AutoJoinContext.java | 760 ----------------- .../apache/lens/cube/parse/CandidateDim.java | 16 +- .../lens/cube/parse/CubeQueryContext.java | 94 +-- .../cube/parse/DenormalizationResolver.java | 74 +- .../apache/lens/cube/parse/DimHQLContext.java | 4 +- .../apache/lens/cube/parse/FieldValidator.java | 9 +- .../org/apache/lens/cube/parse/HQLParser.java | 2 +- .../org/apache/lens/cube/parse/JoinClause.java | 144 ---- .../apache/lens/cube/parse/JoinResolver.java | 262 +----- .../org/apache/lens/cube/parse/JoinTree.java | 164 ---- .../org/apache/lens/cube/parse/StorageUtil.java | 8 +- .../lens/cube/parse/TimerangeResolver.java | 10 +- .../lens/cube/parse/join/AutoJoinContext.java | 719 ++++++++++++++++ .../apache/lens/cube/parse/join/JoinClause.java | 139 +++ .../apache/lens/cube/parse/join/JoinTree.java | 164 ++++ .../apache/lens/cube/parse/join/JoinUtils.java | 49 ++ .../cube/metadata/TestCubeMetastoreClient.java | 284 +++++-- .../apache/lens/cube/parse/CubeTestSetup.java | 843 ++++++++++++++----- .../FieldsCannotBeQueriedTogetherTest.java | 11 +- .../lens/cube/parse/TestBaseCubeQueries.java | 26 +- .../lens/cube/parse/TestCubeRewriter.java | 279 +++--- .../cube/parse/TestDenormalizationResolver.java | 140 +-- .../lens/cube/parse/TestExpressionContext.java | 20 +- .../lens/cube/parse/TestExpressionResolver.java | 102 +-- .../lens/cube/parse/TestJoinResolver.java | 534 +++++------- .../lens/cube/parse/TestQueryRewrite.java | 10 + .../lens/cube/parse/TestRewriterPlan.java | 10 +- .../parse/TestTimeRangeWriterWithQuery.java | 53 +- .../src/main/resources/cube-queries.sql | 74 +- lens-examples/src/main/resources/customer.xml | 4 +- .../src/main/resources/dimension-queries.sql | 14 +- lens-examples/src/main/resources/sales-cube.xml | 12 +- .../src/main/resources/sample-cube.xml | 21 +- .../main/resources/sample-db-only-dimension.xml | 24 +- .../src/main/resources/sample-dimension.xml | 24 +- .../apache/lens/server/metastore/JAXBUtils.java | 54 +- .../server/metastore/TestMetastoreService.java | 17 +- .../apache/lens/storage/db/TestDBStorage.java | 11 +- 60 files changed, 3156 insertions(+), 3293 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-api/src/main/resources/cube-0.1.xsd ---------------------------------------------------------------------- diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd index 4092133..d195b08 100644 --- a/lens-api/src/main/resources/cube-0.1.xsd +++ b/lens-api/src/main/resources/cube-0.1.xsd @@ -393,25 +393,13 @@ <xs:complexContent> <xs:extension base="x_field"> <xs:sequence> - <xs:element name="ref_spec" maxOccurs="1" minOccurs="0"> + <xs:element type="x_chain_column" name="chain_ref_column" maxOccurs="unbounded" minOccurs="0"> <xs:annotation> <xs:documentation> - Reference specifiction needs to be specified if the attribute is a reference attribute. It - can either be table reference or a chained column - - ref_spec can be specified as a list of table references to - which the attribute is refering to. - For ex : userid refers user.id, xuser.id, yuser.id, zuser.id. - - Alternately, ref_spec could be list of chained columns each specifed with chain name and column name. + Chain column specification needs to be specified if the attribute is a reference attribute. + It can be list of chained columns each specified with chain name and column name. </xs:documentation> </xs:annotation> - <xs:complexType> - <xs:choice maxOccurs="1" minOccurs="1"> - <xs:element type="x_table_references" name="table_references" maxOccurs="1" minOccurs="1"/> - <xs:element type="x_chain_column" name="chain_ref_column" maxOccurs="unbounded" minOccurs="1"/> - </xs:choice> - </xs:complexType> </xs:element> <xs:element name="hierarchy" type="x_dim_attributes" maxOccurs="1" minOccurs="0"> <xs:annotation> @@ -471,13 +459,6 @@ </xs:documentation> </xs:annotation> </xs:attribute> - <xs:attribute type="xs:boolean" name="join_key" default="true"> - <xs:annotation> - <xs:documentation> - This flag will tell whether the attribute can be used as a join key or not - </xs:documentation> - </xs:annotation> - </xs:attribute> </xs:extension> </xs:complexContent> </xs:complexType> @@ -531,17 +512,6 @@ </xs:attribute> </xs:complexType> - <xs:complexType name="x_table_references"> - <xs:annotation> - <xs:documentation> - Set of table references. - </xs:documentation> - </xs:annotation> - <xs:sequence> - <xs:element type="x_table_reference" name="table_reference" maxOccurs="unbounded" minOccurs="1"/> - </xs:sequence> - </xs:complexType> - <xs:element name="x_join_chains" type="x_join_chains"/> <xs:complexType name="x_join_chains"> http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-api/src/main/resources/lens-errors.conf ---------------------------------------------------------------------- diff --git a/lens-api/src/main/resources/lens-errors.conf b/lens-api/src/main/resources/lens-errors.conf index c7ccea1..9087fcd 100644 --- a/lens-api/src/main/resources/lens-errors.conf +++ b/lens-api/src/main/resources/lens-errors.conf @@ -207,7 +207,7 @@ lensCubeErrorsForQuery = [ { errorCode = 3018 httpStatusCode = ${BAD_REQUEST} - errorMsg = "No join condition available" + errorMsg = "Default aggregate is not set for measure: %s" } { @@ -219,7 +219,7 @@ lensCubeErrorsForQuery = [ { errorCode = 3020 httpStatusCode = ${BAD_REQUEST} - errorMsg = "Default aggregate is not set for measure: %s" + errorMsg = "No join condition available" } { @@ -294,6 +294,13 @@ lensCubeErrorsForQuery = [ httpStatusCode = ${INTERNAL_SERVER_ERROR} errorMsg = "Could not parse expression %s" } + + { + errorCode = 3033 + httpStatusCode = ${BAD_REQUEST} + errorMsg = "Could not find queried table or chain: %s" + } + ] lensCubeErrorsForMetastore = [ http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionCommands.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionCommands.java index 42c6bae..160699b 100644 --- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionCommands.java +++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionCommands.java @@ -27,7 +27,7 @@ import java.net.URISyntaxException; import java.net.URL; import java.util.Arrays; -import org.apache.lens.api.metastore.XJoinChains; +import org.apache.lens.api.metastore.*; import org.apache.lens.cli.commands.LensDimensionCommands; import org.apache.lens.cli.table.XJoinChainTable; import org.apache.lens.client.LensClient; @@ -62,6 +62,8 @@ public class TestLensDimensionCommands extends LensCliApplicationTest { * the URI syntax exception */ public static void createDimension() throws URISyntaxException { + getCommand().createDimension(new File( + TestLensCubeCommands.class.getClassLoader().getResource("test-detail.xml").toURI())); URL dimensionSpec = TestLensDimensionCommands.class.getClassLoader().getResource("test-dimension.xml"); getCommand().createDimension(new File(dimensionSpec.toURI())); } @@ -81,16 +83,38 @@ public class TestLensDimensionCommands extends LensCliApplicationTest { createDimension(); dimensionList = getCommand().showDimensions(); Assert.assertTrue(dimensionList.contains("test_dim")); + Assert.assertTrue(dimensionList.contains("test_detail")); testFields(getCommand()); testJoinChains(getCommand()); testUpdateCommand(new File(dimensionSpec.toURI()), getCommand()); getCommand().dropDimension("test_dim"); + getCommand().dropDimension("test_detail"); dimensionList = getCommand().showDimensions(); Assert.assertFalse(dimensionList.contains("test_dim")); + Assert.assertFalse(dimensionList.contains("test_detail")); } private void testJoinChains(LensDimensionCommands command) { - assertEquals(command.showJoinChains("test_dim"), new XJoinChainTable(new XJoinChains()).toString()); + XJoinChains chains = new XJoinChains(); + XJoinChain chain1 = new XJoinChain(); + chain1.setPaths(new XJoinPaths()); + XJoinPath path = new XJoinPath(); + path.setEdges(new XJoinEdges()); + XJoinEdge edge1 = new XJoinEdge(); + XTableReference ref1 = new XTableReference(); + ref1.setTable("test_dim"); + ref1.setColumn("d2id"); + XTableReference ref2 = new XTableReference(); + ref2.setTable("test_detail"); + ref2.setColumn("id"); + edge1.setFrom(ref1); + edge1.setTo(ref2); + path.getEdges().getEdge().add(edge1); + chain1.setName("dim2chain"); + chain1.getPaths().getPath().add(path); + chain1.setDestTable("test_detail"); + chains.getJoinChain().add(chain1); + assertEquals(command.showJoinChains("test_dim"), new XJoinChainTable(chains).toString()); } private void testFields(LensDimensionCommands qCom) { http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cli/src/test/resources/cube_with_no_weight_facts.xml ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/resources/cube_with_no_weight_facts.xml b/lens-cli/src/test/resources/cube_with_no_weight_facts.xml index 263ca88..13736b2 100644 --- a/lens-cli/src/test/resources/cube_with_no_weight_facts.xml +++ b/lens-cli/src/test/resources/cube_with_no_weight_facts.xml @@ -30,13 +30,7 @@ <dim_attributes> <dim_attribute name="dim1" type="INT" /> <dim_attribute name="dim2" type="INT" start_time='2013-12-01T00:00:00' /> - <dim_attribute name="dim3" type="INT"> - <ref_spec> - <table_references> - <table_reference table="dim_table" column="id" /> - </table_references> - </ref_spec> - </dim_attribute> + <dim_attribute name="dim3" type="INT"/> </dim_attributes> <expressions> <expression name="expr_msr5" type="DOUBLE"> http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cli/src/test/resources/sample-cube.xml ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/resources/sample-cube.xml b/lens-cli/src/test/resources/sample-cube.xml index d72d279..e3b3284 100644 --- a/lens-cli/src/test/resources/sample-cube.xml +++ b/lens-cli/src/test/resources/sample-cube.xml @@ -34,19 +34,11 @@ <dim_attributes> <dim_attribute name="dim1" type="INT" /> <dim_attribute name="dim2" type="INT" start_time='2013-12-01T00:00:00' /> - <dim_attribute name="dim3" type="INT"> - <ref_spec> - <table_references> - <table_reference table="dim_table" column="id" /> - </table_references> - </ref_spec> - </dim_attribute> + <dim_attribute name="dim3" type="INT"/> <dim_attribute name="dimDetail" type="string" description="City name to which the customer belongs" display_string="Customer City"> - <ref_spec> - <chain_ref_column chain_name="testdimchain" ref_col="detail" /> - <chain_ref_column chain_name="testdetailchain" ref_col="name" /> - </ref_spec> + <chain_ref_column chain_name="testdimchain" ref_col="detail" /> + <chain_ref_column chain_name="testdetailchain" ref_col="name" /> </dim_attribute> </dim_attributes> <expressions> http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cli/src/test/resources/test-dimension.xml ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/resources/test-dimension.xml b/lens-cli/src/test/resources/test-dimension.xml index 6eb3d31..2fa47f1 100644 --- a/lens-cli/src/test/resources/test-dimension.xml +++ b/lens-cli/src/test/resources/test-dimension.xml @@ -25,13 +25,7 @@ <dim_attribute name="id" type="INT" /> <dim_attribute name="name" type="STRING" /> <dim_attribute name="detail" type="STRING" start_time='2013-12-01T00:00:00' /> - <dim_attribute name="d2id" type="INT" start_time='2013-12-01T00:00:00'> - <ref_spec> - <table_references> - <table_reference table="test_dim2" column="id" /> - </table_references> - </ref_spec> - </dim_attribute> + <dim_attribute name="d2id" type="INT" start_time='2013-12-01T00:00:00'/> <dim_attribute name="inline" type="STRING" > <values>A</values> <values>B</values> @@ -46,6 +40,20 @@ </dim_attribute> </attributes> + <join_chains> + <join_chain name="dim2chain"> + <paths> + <path> + <edges> + <edge> + <from table="test_dim" column="d2id" /> + <to table="test_detail" column="id" /> + </edge> + </edges> + </path> + </paths> + </join_chain> + </join_chains> <properties> <property name="test_dim.prop" value="test" /> <property name="dimension.test_dim.timed.dimension" value="dt" /> http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java b/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java index 68cd80b..61d08b2 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java @@ -43,7 +43,7 @@ public enum LensCubeErrorCode { CANNOT_USE_TIMERANGE_WRITER(3017, 100), NO_DEFAULT_AGGREGATE(3018, 200), EXPRESSION_NOT_IN_ANY_FACT(3019, 300), - NO_JOIN_CONDITION_AVAIABLE(3020, 400), + NO_JOIN_CONDITION_AVAILABLE(3020, 400), NO_JOIN_PATH(3021, 500), COLUMN_UNAVAILABLE_IN_TIME_RANGE(3022, 600), NO_DIM_HAS_COLUMN(3023, 700), @@ -53,9 +53,11 @@ public enum LensCubeErrorCode { NO_CANDIDATE_DIM_AVAILABLE(3027, 1100), NO_CANDIDATE_FACT_AVAILABLE(3028, 1200), NO_CANDIDATE_DIM_STORAGE_TABLES(3029, 1300), - NO_STORAGE_TABLE_AVAIABLE(3030, 1400), + NO_STORAGE_TABLE_AVAILABLE(3030, 1400), STORAGE_UNION_DISABLED(3031, 1500), COULD_NOT_PARSE_EXPRESSION(3032, 1500), + QUERIED_TABLE_NOT_FOUND(3033, 0), + // Error codes greater than 3100 are errors while doing a metastore operation. ERROR_IN_ENTITY_DEFINITION(3101, 100), TIMELINE_ABSENT(3102, 100), http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractBaseTable.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractBaseTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractBaseTable.java index 88c9ee8..5543308 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractBaseTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractBaseTable.java @@ -23,7 +23,6 @@ import java.util.*; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; import com.google.common.base.Preconditions; @@ -38,7 +37,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public abstract class AbstractBaseTable extends AbstractCubeTable { private final Set<ExprColumn> expressions; - private static final List<FieldSchema> COLUMNS = new ArrayList<FieldSchema>(); + private static final List<FieldSchema> COLUMNS = new ArrayList<>(); private final Map<String, ExprColumn> exprMap; @Getter private final Set<JoinChain> joinChains; @@ -52,9 +51,9 @@ public abstract class AbstractBaseTable extends AbstractCubeTable { properties, double weight) { super(name, COLUMNS, properties, weight); - exprMap = new HashMap<String, ExprColumn>(); + exprMap = new HashMap<>(); if (exprs == null) { - this.expressions = new HashSet<ExprColumn>(); + this.expressions = new HashSet<>(); } else { this.expressions = exprs; } @@ -66,10 +65,10 @@ public abstract class AbstractBaseTable extends AbstractCubeTable { if (joinChains != null) { this.joinChains = joinChains; } else { - this.joinChains = new HashSet<JoinChain>(); + this.joinChains = new HashSet<>(); } - chainMap = new HashMap<String, JoinChain>(); + chainMap = new HashMap<>(); for (JoinChain chain : this.joinChains) { chainMap.put(chain.getName().toLowerCase(), chain); } @@ -78,12 +77,12 @@ public abstract class AbstractBaseTable extends AbstractCubeTable { public AbstractBaseTable(Table tbl) { super(tbl); this.expressions = getExpressions(getName(), getProperties()); - exprMap = new HashMap<String, ExprColumn>(); + exprMap = new HashMap<>(); for (ExprColumn expr : expressions) { exprMap.put(expr.getName().toLowerCase(), expr); } this.joinChains = getJoinChains(this, getJoinChainListPropKey(getName()), getProperties()); - chainMap = new HashMap<String, JoinChain>(); + chainMap = new HashMap<>(); for (JoinChain chain : joinChains) { chainMap.put(chain.getName().toLowerCase(), chain); } @@ -110,7 +109,7 @@ public abstract class AbstractBaseTable extends AbstractCubeTable { } private static Set<ExprColumn> getExpressions(String name, Map<String, String> props) { - Set<ExprColumn> exprs = new HashSet<ExprColumn>(); + Set<ExprColumn> exprs = new HashSet<>(); String exprStr = MetastoreUtil.getNamedStringValue(props, MetastoreUtil.getExpressionListKey(name)); if (!StringUtils.isBlank(exprStr)) { String[] names = exprStr.split(","); @@ -152,7 +151,7 @@ public abstract class AbstractBaseTable extends AbstractCubeTable { } public ExprColumn getExpressionByName(String exprName) { - return exprMap.get(exprName == null ? exprName : exprName.toLowerCase()); + return exprMap.get(exprName == null ? null : exprName.toLowerCase()); } public CubeColumn getColumnByName(String column) { @@ -162,10 +161,9 @@ public abstract class AbstractBaseTable extends AbstractCubeTable { /** * Alters the expression if already existing or just adds if it is new expression. * - * @param expr - * @throws HiveException + * @param expr ExprColumn */ - public void alterExpression(ExprColumn expr) throws HiveException { + public void alterExpression(ExprColumn expr) { if (expr == null) { throw new NullPointerException("Cannot add null expression"); } @@ -183,9 +181,9 @@ public abstract class AbstractBaseTable extends AbstractCubeTable { } /** - * Remove the measure with name specified + * Remove the expression with name specified * - * @param exprName + * @param exprName expression name */ public void removeExpression(String exprName) { if (exprMap.containsKey(exprName.toLowerCase())) { @@ -197,7 +195,7 @@ public abstract class AbstractBaseTable extends AbstractCubeTable { } public Set<String> getExpressionNames() { - Set<String> exprNames = new HashSet<String>(); + Set<String> exprNames = new HashSet<>(); for (ExprColumn f : getExpressions()) { exprNames.add(f.getName().toLowerCase()); } @@ -225,10 +223,9 @@ public abstract class AbstractBaseTable extends AbstractCubeTable { /** * Alters the joinchain if already existing or just adds if it is new chain * - * @param joinchain - * @throws HiveException + * @param joinchain join chain */ - public void alterJoinChain(JoinChain joinchain) throws HiveException { + public void alterJoinChain(JoinChain joinchain) { if (joinchain == null) { throw new NullPointerException("Cannot add null joinchain"); } @@ -251,20 +248,20 @@ public abstract class AbstractBaseTable extends AbstractCubeTable { } /** - * Returns the property key for Cube/Dimension specific join chain list + * Get the property key for Cube/Dimension specific join chain list * - * @param tblname - * @return + * @param tblName table name + * @return the property key for Cube/Dimension specific join chain list */ - protected abstract String getJoinChainListPropKey(String tblname); + protected abstract String getJoinChainListPropKey(String tblName); /** * Get join chains from properties * - * @return + * @return set of join chains */ private static Set<JoinChain> getJoinChains(AbstractBaseTable tbl, String propName, Map<String, String> props) { - Set<JoinChain> joinChains = new HashSet<JoinChain>(); + Set<JoinChain> joinChains = new HashSet<>(); String joinChainsStr = MetastoreUtil.getNamedStringValue(props, propName); if (!StringUtils.isBlank(joinChainsStr)) { String[] cnames = joinChainsStr.split(","); @@ -277,7 +274,7 @@ public abstract class AbstractBaseTable extends AbstractCubeTable { } public Set<String> getJoinChainNames() { - Set<String> chainNames = new HashSet<String>(); + Set<String> chainNames = new HashSet<>(); for (JoinChain f : getJoinChains()) { chainNames.add(f.getName().toLowerCase()); } @@ -286,9 +283,9 @@ public abstract class AbstractBaseTable extends AbstractCubeTable { /** - * Remove the joinchain with name specified + * Remove the join chain with name specified * - * @param chainName + * @param chainName chain name */ public boolean removeJoinChain(String chainName) { if (chainMap.containsKey(chainName.toLowerCase())) { http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java index da3a7e5..01098c4 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java @@ -21,16 +21,16 @@ package org.apache.lens.cube.metadata; import java.util.*; import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; +import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @Slf4j public abstract class AbstractCubeTable implements Named { private final String name; private final List<FieldSchema> columns; - private final Map<String, String> properties = new HashMap<String, String>(); + private final Map<String, String> properties = new HashMap<>(); private double weight; protected AbstractCubeTable(String name, List<FieldSchema> columns, Map<String, String> props, double weight) { @@ -82,7 +82,7 @@ public abstract class AbstractCubeTable implements Named { /** * Alters the weight of table * - * @param weight + * @param weight Weight of the table. */ public void alterWeight(double weight) { this.weight = weight; @@ -102,7 +102,7 @@ public abstract class AbstractCubeTable implements Named { /** * Remove property specified by the key * - * @param propKey + * @param propKey property key */ public void removeProperty(String propKey) { properties.remove(propKey); @@ -111,13 +111,9 @@ public abstract class AbstractCubeTable implements Named { /** * Alters the column if already existing or just adds it if it is new column * - * @param column - * @throws HiveException + * @param column The column spec as FieldSchema - name, type and a comment */ - protected void alterColumn(FieldSchema column) throws HiveException { - if (column == null) { - throw new HiveException("Column cannot be null"); - } + protected void alterColumn(@NonNull FieldSchema column) { Iterator<FieldSchema> columnItr = columns.iterator(); int alterPos = -1; int i = 0; @@ -144,13 +140,9 @@ public abstract class AbstractCubeTable implements Named { /** * Adds or alters the columns passed * - * @param columns - * @throws HiveException + * @param columns The collection of columns */ - protected void addColumns(Collection<FieldSchema> columns) throws HiveException { - if (columns == null) { - throw new HiveException("Columns cannot be null"); - } + protected void addColumns(@NonNull Collection<FieldSchema> columns) { for (FieldSchema column : columns) { alterColumn(column); } @@ -202,7 +194,7 @@ public abstract class AbstractCubeTable implements Named { public Set<String> getAllFieldNames() { List<FieldSchema> fields = getColumns(); - Set<String> columns = new HashSet<String>(fields.size()); + Set<String> columns = new HashSet<>(fields.size()); for (FieldSchema f : fields) { columns.add(f.getName().toLowerCase()); } http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java index f09da37..b376aaf 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java @@ -22,9 +22,9 @@ import java.lang.reflect.Constructor; import java.util.*; import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; +import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -55,12 +55,12 @@ public class Cube extends AbstractBaseTable implements CubeInterface { this.measures = measures; this.dimensions = dimensions; - measureMap = new HashMap<String, CubeMeasure>(); + measureMap = new HashMap<>(); for (CubeMeasure m : measures) { measureMap.put(m.getName().toLowerCase(), m); } - dimMap = new HashMap<String, CubeDimAttribute>(); + dimMap = new HashMap<>(); for (CubeDimAttribute dim : dimensions) { dimMap.put(dim.getName().toLowerCase(), dim); } @@ -73,12 +73,12 @@ public class Cube extends AbstractBaseTable implements CubeInterface { this.measures = getMeasures(getName(), getProperties()); this.dimensions = getDimensions(getName(), getProperties()); - measureMap = new HashMap<String, CubeMeasure>(); + measureMap = new HashMap<>(); for (CubeMeasure m : measures) { measureMap.put(m.getName().toLowerCase(), m); } - dimMap = new HashMap<String, CubeDimAttribute>(); + dimMap = new HashMap<>(); for (CubeDimAttribute dim : dimensions) { addAllDimsToMap(dim); } @@ -103,7 +103,7 @@ public class Cube extends AbstractBaseTable implements CubeInterface { public Set<String> getTimedDimensions() { String str = getProperties().get(MetastoreUtil.getCubeTimedDimensionListKey(getName())); - Set<String> timedDimensions = new HashSet<String>(); + Set<String> timedDimensions = new HashSet<>(); if (str != null) { timedDimensions.addAll(Arrays.asList(StringUtils.split(str, ','))); } @@ -137,7 +137,7 @@ public class Cube extends AbstractBaseTable implements CubeInterface { } public static Set<CubeMeasure> getMeasures(String name, Map<String, String> props) { - Set<CubeMeasure> measures = new HashSet<CubeMeasure>(); + Set<CubeMeasure> measures = new HashSet<>(); String measureStr = MetastoreUtil.getNamedStringValue(props, MetastoreUtil.getCubeMeasureListKey(name)); String[] names = measureStr.split(","); for (String measureName : names) { @@ -157,21 +157,23 @@ public class Cube extends AbstractBaseTable implements CubeInterface { } public static Set<CubeDimAttribute> getDimensions(String name, Map<String, String> props) { - Set<CubeDimAttribute> dimensions = new HashSet<CubeDimAttribute>(); + Set<CubeDimAttribute> dimensions = new HashSet<>(); String dimStr = MetastoreUtil.getNamedStringValue(props, MetastoreUtil.getCubeDimensionListKey(name)); - String[] names = dimStr.split(","); - for (String dimName : names) { - String className = props.get(MetastoreUtil.getDimensionClassPropertyKey(dimName)); - CubeDimAttribute dim; - try { - Class<?> clazz = Class.forName(className); - Constructor<?> constructor; - constructor = clazz.getConstructor(String.class, Map.class); - dim = (CubeDimAttribute) constructor.newInstance(new Object[]{dimName, props}); - } catch (Exception e) { - throw new IllegalArgumentException("Invalid dimension", e); + if (StringUtils.isNotBlank(dimStr)) { + String[] names = dimStr.split(","); + for (String dimName : names) { + String className = props.get(MetastoreUtil.getDimensionClassPropertyKey(dimName)); + CubeDimAttribute dim; + try { + Class<?> clazz = Class.forName(className); + Constructor<?> constructor; + constructor = clazz.getConstructor(String.class, Map.class); + dim = (CubeDimAttribute) constructor.newInstance(new Object[]{dimName, props}); + } catch (Exception e) { + throw new IllegalArgumentException("Invalid dimension", e); + } + dimensions.add(dim); } - dimensions.add(dim); } return dimensions; } @@ -226,14 +228,9 @@ public class Cube extends AbstractBaseTable implements CubeInterface { /** * Alters the measure if already existing or just adds if it is new measure. * - * @param measure - * @throws HiveException + * @param measure new measure definition */ - public void alterMeasure(CubeMeasure measure) throws HiveException { - if (measure == null) { - throw new NullPointerException("Cannot add null measure"); - } - + public void alterMeasure(@NonNull CubeMeasure measure) { // Replace measure if already existing if (measureMap.containsKey(measure.getName().toLowerCase())) { measures.remove(getMeasureByName(measure.getName())); @@ -249,9 +246,9 @@ public class Cube extends AbstractBaseTable implements CubeInterface { /** * Remove the joinchain with name specified * - * @param chainName + * @param chainName chain name */ - public boolean removeJoinChain(String chainName) { + public boolean removeJoinChain(@NonNull String chainName) { if (super.removeJoinChain(chainName)) { log.info("Removing dimension {}", getDimAttributeByName(chainName)); return true; @@ -262,14 +259,9 @@ public class Cube extends AbstractBaseTable implements CubeInterface { /** * Alters the dimension if already existing or just adds if it is new dimension * - * @param dimension - * @throws HiveException + * @param dimension the dim attribute */ - public void alterDimension(CubeDimAttribute dimension) throws HiveException { - if (dimension == null) { - throw new NullPointerException("Cannot add null dimension"); - } - + public void alterDimension(@NonNull CubeDimAttribute dimension) { // Replace dimension if already existing if (dimMap.containsKey(dimension.getName().toLowerCase())) { dimensions.remove(getDimAttributeByName(dimension.getName())); @@ -284,11 +276,11 @@ public class Cube extends AbstractBaseTable implements CubeInterface { /** - * Remove the dimension with name specified + * Remove the attribute with name specified * - * @param dimName + * @param dimName attribute name */ - public void removeDimension(String dimName) { + public void removeDimension(@NonNull String dimName) { if (dimMap.containsKey(dimName.toLowerCase())) { log.info("Removing dimension {}", getDimAttributeByName(dimName)); dimensions.remove(getDimAttributeByName(dimName)); @@ -300,9 +292,9 @@ public class Cube extends AbstractBaseTable implements CubeInterface { /** * Remove the measure with name specified * - * @param msrName + * @param msrName measure name */ - public void removeMeasure(String msrName) { + public void removeMeasure(@NonNull String msrName) { if (measureMap.containsKey(msrName.toLowerCase())) { log.info("Removing measure {}", getMeasureByName(msrName)); measures.remove(getMeasureByName(msrName)); @@ -314,17 +306,13 @@ public class Cube extends AbstractBaseTable implements CubeInterface { /** * Adds the timed dimension * - * @param timedDimension - * @throws HiveException + * @param timedDimension time dimension */ - public void addTimedDimension(String timedDimension) throws HiveException { - if (timedDimension == null || timedDimension.isEmpty()) { - throw new HiveException("Invalid timed dimension " + timedDimension); - } + public void addTimedDimension(@NonNull String timedDimension) { timedDimension = timedDimension.toLowerCase(); Set<String> timeDims = getTimedDimensions(); if (timeDims == null) { - timeDims = new LinkedHashSet<String>(); + timeDims = new LinkedHashSet<>(); } if (timeDims.contains(timedDimension)) { log.info("Timed dimension {} is already present in cube {}", timedDimension, getName()); @@ -338,13 +326,9 @@ public class Cube extends AbstractBaseTable implements CubeInterface { /** * Removes the timed dimension * - * @param timedDimension - * @throws HiveException + * @param timedDimension time dimension */ - public void removeTimedDimension(String timedDimension) throws HiveException { - if (timedDimension == null || timedDimension.isEmpty()) { - throw new HiveException("Invalid timed dimension " + timedDimension); - } + public void removeTimedDimension(@NonNull String timedDimension) { timedDimension = timedDimension.toLowerCase(); Set<String> timeDims = getTimedDimensions(); if (timeDims != null && timeDims.contains(timedDimension)) { @@ -360,7 +344,7 @@ public class Cube extends AbstractBaseTable implements CubeInterface { @Override public Set<String> getMeasureNames() { - Set<String> measureNames = new HashSet<String>(); + Set<String> measureNames = new HashSet<>(); for (CubeMeasure f : getMeasures()) { measureNames.add(f.getName().toLowerCase()); } @@ -369,7 +353,7 @@ public class Cube extends AbstractBaseTable implements CubeInterface { @Override public Set<String> getDimAttributeNames() { - Set<String> dimNames = new HashSet<String>(); + Set<String> dimNames = new HashSet<>(); for (CubeDimAttribute f : getDimAttributes()) { MetastoreUtil.addColumnNames(f, dimNames); } @@ -378,9 +362,9 @@ public class Cube extends AbstractBaseTable implements CubeInterface { @Override public boolean allFieldsQueriable() { - String canbeQueried = getProperties().get(MetastoreConstants.CUBE_ALL_FIELDS_QUERIABLE); - if (canbeQueried != null) { - return Boolean.parseBoolean(canbeQueried); + String canBeQueried = getProperties().get(MetastoreConstants.CUBE_ALL_FIELDS_QUERIABLE); + if (canBeQueried != null) { + return Boolean.parseBoolean(canBeQueried); } return true; } @@ -398,22 +382,21 @@ public class Cube extends AbstractBaseTable implements CubeInterface { * @see org.apache.lens.cube.metadata.AbstractBaseTable */ @Override - protected String getJoinChainListPropKey(String tblname) { + protected String getJoinChainListPropKey(@NonNull String tblname) { return MetastoreUtil.getCubeJoinChainListKey(getName()); } - public String getPartitionColumnOfTimeDim(String timeDimName) { + public String getPartitionColumnOfTimeDim(@NonNull String timeDimName) { String partCol = getProperties().get(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + timeDimName); return StringUtils.isNotBlank(partCol) ? partCol : timeDimName; } - public String getTimeDimOfPartitionColumn(String partCol) { + public String getTimeDimOfPartitionColumn(@NonNull String partCol) { Map<String, String> properties = getProperties(); for (Map.Entry<String, String> entry : properties.entrySet()) { if (entry.getKey().startsWith(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX) && entry.getValue().equalsIgnoreCase(partCol)) { - String timeDim = entry.getKey().replace(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX, ""); - return timeDim; + return entry.getKey().replace(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX, ""); } } return partCol; http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeColumn.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeColumn.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeColumn.java index b04532f..77024c0 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeColumn.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeColumn.java @@ -24,7 +24,6 @@ import java.util.Date; import java.util.Map; import java.util.TimeZone; - import com.google.common.base.Optional; import lombok.NonNull; http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeDimensionTable.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeDimensionTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeDimensionTable.java index cd80d64..713f476 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeDimensionTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeDimensionTable.java @@ -22,17 +22,17 @@ import java.util.*; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; import com.google.common.collect.Sets; +import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @Slf4j public final class CubeDimensionTable extends AbstractCubeTable { private String dimName; // dimension name the dimtabe belongs to - private final Map<String, UpdatePeriod> snapshotDumpPeriods = new HashMap<String, UpdatePeriod>(); + private final Map<String, UpdatePeriod> snapshotDumpPeriods = new HashMap<>(); public CubeDimensionTable(String dimName, String dimTblName, List<FieldSchema> columns, double weight, Map<String, UpdatePeriod> snapshotDumpPeriods) { @@ -61,7 +61,7 @@ public final class CubeDimensionTable extends AbstractCubeTable { private static Map<String, UpdatePeriod> getSnapshotDumpPeriods(Set<String> storages) { - Map<String, UpdatePeriod> snapshotDumpPeriods = new HashMap<String, UpdatePeriod>(); + Map<String, UpdatePeriod> snapshotDumpPeriods = new HashMap<>(); for (String storage : storages) { snapshotDumpPeriods.put(storage, null); } @@ -134,7 +134,7 @@ public final class CubeDimensionTable extends AbstractCubeTable { private static Map<String, UpdatePeriod> getDumpPeriods(String name, Map<String, String> params) { String storagesStr = params.get(MetastoreUtil.getDimensionStorageListKey(name)); if (!StringUtils.isBlank(storagesStr)) { - Map<String, UpdatePeriod> dumpPeriods = new HashMap<String, UpdatePeriod>(); + Map<String, UpdatePeriod> dumpPeriods = new HashMap<>(); for (String storage : StringUtils.split(storagesStr, ",")) { String dumpPeriod = params.get(MetastoreUtil.getDimensionDumpPeriodKey(name, storage)); if (dumpPeriod != null) { @@ -193,7 +193,7 @@ public final class CubeDimensionTable extends AbstractCubeTable { /** * Alter the dimension name that the table belongs to * - * @param newDimName + * @param newDimName new dimension name. */ public void alterUberDim(String newDimName) { this.dimName = newDimName; @@ -205,13 +205,8 @@ public final class CubeDimensionTable extends AbstractCubeTable { * * @param storage Storage name * @param period The new value - * @throws HiveException */ - public void alterSnapshotDumpPeriod(String storage, UpdatePeriod period) throws HiveException { - if (storage == null) { - throw new HiveException("Cannot add null storage for " + getName()); - } - + public void alterSnapshotDumpPeriod(@NonNull String storage, UpdatePeriod period) { if (snapshotDumpPeriods.containsKey(storage)) { log.info("Updating dump period for {} from {} to {}", storage, snapshotDumpPeriods.get(storage), period); } @@ -221,12 +216,12 @@ public final class CubeDimensionTable extends AbstractCubeTable { } @Override - public void alterColumn(FieldSchema column) throws HiveException { + public void alterColumn(FieldSchema column) { super.alterColumn(column); } @Override - public void addColumns(Collection<FieldSchema> columns) throws HiveException { + public void addColumns(Collection<FieldSchema> columns) { super.addColumns(columns); } http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java index b1fec8c..643bcfe 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java @@ -20,12 +20,12 @@ package org.apache.lens.cube.metadata; import java.util.*; +import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.cube.metadata.UpdatePeriod.UpdatePeriodComparator; import org.apache.lens.server.api.error.LensException; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; import com.google.common.collect.Lists; @@ -261,11 +261,11 @@ public class CubeFactTable extends AbstractCubeTable { * * @param storage * @param updatePeriods - * @throws HiveException */ - public void alterStorage(String storage, Set<UpdatePeriod> updatePeriods) throws HiveException { + public void alterStorage(String storage, Set<UpdatePeriod> updatePeriods) throws LensException{ if (!storageUpdatePeriods.containsKey(storage)) { - throw new HiveException("Invalid storage" + storage); + throw new LensException(LensCubeErrorCode.ERROR_IN_ENTITY_DEFINITION.getLensErrorInfo(), + "Invalid storage" + storage); } storageUpdatePeriods.put(storage, updatePeriods); addUpdatePeriodProperies(getName(), getProperties(), storageUpdatePeriods); @@ -276,9 +276,8 @@ public class CubeFactTable extends AbstractCubeTable { * * @param storage * @param updatePeriods - * @throws HiveException */ - void addStorage(String storage, Set<UpdatePeriod> updatePeriods) throws HiveException { + void addStorage(String storage, Set<UpdatePeriod> updatePeriods) { storageUpdatePeriods.put(storage, updatePeriods); addUpdatePeriodProperies(getName(), getProperties(), storageUpdatePeriods); } @@ -296,12 +295,12 @@ public class CubeFactTable extends AbstractCubeTable { } @Override - public void alterColumn(FieldSchema column) throws HiveException { + public void alterColumn(FieldSchema column) { super.alterColumn(column); } @Override - public void addColumns(Collection<FieldSchema> columns) throws HiveException { + public void addColumns(Collection<FieldSchema> columns) { super.addColumns(columns); } http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index ae0fb90..dcb932e 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -85,7 +85,6 @@ public class CubeMetastoreClient { PartitionTimelineCache partitionTimelineCache = new PartitionTimelineCache(); // dbname to client mapping private static final Map<String, CubeMetastoreClient> CLIENT_MAPPING = Maps.newConcurrentMap(); - private SchemaGraph schemaGraph; // Set of all storage table names for which latest partitions exist private final Set<String> latestLookupCache = Sets.newSetFromMap(new ConcurrentHashMap<String, Boolean>()); @@ -627,7 +626,8 @@ public class CubeMetastoreClient { * @throws HiveException */ public void createCube(String name, Set<CubeMeasure> measures, Set<CubeDimAttribute> dimensions, - Set<ExprColumn> expressions, Set<JoinChain> chains, Map<String, String> properties) throws HiveException { + Set<ExprColumn> expressions, Set<JoinChain> chains, Map<String, String> properties) + throws HiveException { Cube cube = new Cube(name, measures, dimensions, expressions, chains, properties, 0L); createCube(cube); } @@ -1858,13 +1858,6 @@ public class CubeMetastoreClient { return false; } - public synchronized SchemaGraph getSchemaGraph() throws HiveException { - if (schemaGraph == null) { - schemaGraph = new SchemaGraph(this); - } - return schemaGraph; - } - /** * * @param table table name http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cube/src/main/java/org/apache/lens/cube/metadata/DerivedCube.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/DerivedCube.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/DerivedCube.java index 681aa7b..4c73785 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/DerivedCube.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/DerivedCube.java @@ -25,22 +25,21 @@ import org.apache.lens.server.api.error.LensException; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; import com.google.common.collect.Lists; public class DerivedCube extends AbstractCubeTable implements CubeInterface { - private static final List<FieldSchema> COLUMNS = new ArrayList<FieldSchema>(); + private static final List<FieldSchema> COLUMNS = new ArrayList<>(); static { COLUMNS.add(new FieldSchema("dummy", "string", "dummy column")); } private final Cube parent; - private final Set<String> measures = new HashSet<String>(); - private final Set<String> dimensions = new HashSet<String>(); + private final Set<String> measures = new HashSet<>(); + private final Set<String> dimensions = new HashSet<>(); public DerivedCube(String name, Set<String> measures, Set<String> dimensions, Cube parent) throws LensException { this(name, measures, dimensions, new HashMap<String, String>(), 0L, parent); @@ -99,8 +98,8 @@ public class DerivedCube extends AbstractCubeTable implements CubeInterface { this.parent = parent; } - private Set<CubeMeasure> cachedMeasures = new HashSet<CubeMeasure>(); - private Set<CubeDimAttribute> cachedDims = new HashSet<CubeDimAttribute>(); + private Set<CubeMeasure> cachedMeasures = new HashSet<>(); + private Set<CubeDimAttribute> cachedDims = new HashSet<>(); public Set<CubeMeasure> getMeasures() { synchronized (measures) { @@ -152,7 +151,7 @@ public class DerivedCube extends AbstractCubeTable implements CubeInterface { } public static Set<String> getMeasures(String name, Map<String, String> props) { - Set<String> measures = new HashSet<String>(); + Set<String> measures = new HashSet<>(); String measureStr = MetastoreUtil.getNamedStringValue(props, MetastoreUtil.getCubeMeasureListKey(name)); measures.addAll(Arrays.asList(StringUtils.split(measureStr, ','))); return measures; @@ -161,7 +160,7 @@ public class DerivedCube extends AbstractCubeTable implements CubeInterface { public Set<String> getTimedDimensions() { String str = getProperties().get(MetastoreUtil.getCubeTimedDimensionListKey(getName())); if (str != null) { - Set<String> timedDimensions = new HashSet<String>(); + Set<String> timedDimensions = new HashSet<>(); timedDimensions.addAll(Arrays.asList(StringUtils.split(str, ','))); return timedDimensions; } else { @@ -170,7 +169,7 @@ public class DerivedCube extends AbstractCubeTable implements CubeInterface { } public static Set<String> getDimensions(String name, Map<String, String> props) { - Set<String> dimensions = new HashSet<String>(); + Set<String> dimensions = new HashSet<>(); String dimStr = MetastoreUtil.getNamedStringValue(props, MetastoreUtil.getCubeDimensionListKey(name)); dimensions.addAll(Arrays.asList(StringUtils.split(dimStr, ','))); return dimensions; @@ -236,10 +235,9 @@ public class DerivedCube extends AbstractCubeTable implements CubeInterface { /** * Add a new measure * - * @param measure - * @throws HiveException + * @param measure measure name */ - public void addMeasure(String measure) throws HiveException { + public void addMeasure(String measure) { measures.add(measure.toLowerCase()); updateMeasureProperties(); } @@ -247,10 +245,9 @@ public class DerivedCube extends AbstractCubeTable implements CubeInterface { /** * Add a new dimension * - * @param dimension - * @throws HiveException + * @param dimension attribute name */ - public void addDimension(String dimension) throws HiveException { + public void addDimension(String dimension) { dimensions.add(dimension.toLowerCase()); updateDimAttributeProperties(); } @@ -287,7 +284,7 @@ public class DerivedCube extends AbstractCubeTable implements CubeInterface { @Override public Set<String> getDimAttributeNames() { - Set<String> dimNames = new HashSet<String>(); + Set<String> dimNames = new HashSet<>(); for (CubeDimAttribute f : getDimAttributes()) { MetastoreUtil.addColumnNames(f, dimNames); } @@ -311,7 +308,7 @@ public class DerivedCube extends AbstractCubeTable implements CubeInterface { @Override public Set<String> getAllFieldNames() { - Set<String> fieldNames = new HashSet<String>(); + Set<String> fieldNames = new HashSet<>(); fieldNames.addAll(getMeasureNames()); fieldNames.addAll(getDimAttributeNames()); fieldNames.addAll(getTimedDimensions()); http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cube/src/main/java/org/apache/lens/cube/metadata/Dimension.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Dimension.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Dimension.java index 27cbc30..86eb6eb 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Dimension.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Dimension.java @@ -24,9 +24,9 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; -import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; +import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -48,7 +48,7 @@ public class Dimension extends AbstractBaseTable { super(name, expressions, joinChains, properties, weight); this.attributes = attributes; - attributeMap = new HashMap<String, CubeDimAttribute>(); + attributeMap = new HashMap<>(); for (CubeDimAttribute dim : attributes) { attributeMap.put(dim.getName().toLowerCase(), dim); } @@ -59,7 +59,7 @@ public class Dimension extends AbstractBaseTable { super(tbl); this.attributes = getAttributes(getName(), getProperties()); - attributeMap = new HashMap<String, CubeDimAttribute>(); + attributeMap = new HashMap<>(); for (CubeDimAttribute attr : attributes) { addAllAttributesToMap(attr); } @@ -108,7 +108,7 @@ public class Dimension extends AbstractBaseTable { } public static Set<CubeDimAttribute> getAttributes(String name, Map<String, String> props) { - Set<CubeDimAttribute> attributes = new HashSet<CubeDimAttribute>(); + Set<CubeDimAttribute> attributes = new HashSet<>(); String attrStr = MetastoreUtil.getNamedStringValue(props, MetastoreUtil.getDimAttributeListKey(name)); String[] names = attrStr.split(","); for (String attrName : names) { @@ -135,11 +135,6 @@ public class Dimension extends AbstractBaseTable { return MetastoreUtil.getDimensionJoinChainListKey(tblname); } -// public boolean isChainedColumn(String name) { -// Preconditions.checkArgument(name != null); -// return ((ReferencedDimAtrribute) attributeMap.get(name.toLowerCase())).isChainedColumn(); -// } - @Override public int hashCode() { return super.hashCode(); @@ -185,13 +180,8 @@ public class Dimension extends AbstractBaseTable { * Alters the attribute if already existing or just adds if it is new attribute * * @param attribute - * @throws HiveException */ - public void alterAttribute(CubeDimAttribute attribute) throws HiveException { - if (attribute == null) { - throw new NullPointerException("Cannot add null attribute"); - } - + public void alterAttribute(@NonNull CubeDimAttribute attribute) { // Replace dimension if already existing if (attributeMap.containsKey(attribute.getName().toLowerCase())) { attributes.remove(getAttributeByName(attribute.getName())); http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cube/src/main/java/org/apache/lens/cube/metadata/JoinChain.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JoinChain.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JoinChain.java index 6250905..cc8929f 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JoinChain.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JoinChain.java @@ -20,8 +20,8 @@ package org.apache.lens.cube.metadata; import java.util.*; -import org.apache.lens.cube.metadata.SchemaGraph.JoinPath; -import org.apache.lens.cube.metadata.SchemaGraph.TableRelationship; +import org.apache.lens.cube.metadata.join.JoinPath; +import org.apache.lens.cube.metadata.join.TableRelationship; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -313,14 +313,14 @@ public class JoinChain implements Named { } /** - * Convert join paths to schemaGraph's JoinPath + * Convert join chain paths to JoinPath objects * * @param client - * @return List<SchemaGraph.JoinPath> + * @return List<JoinPath> * @throws HiveException */ - public List<SchemaGraph.JoinPath> getRelationEdges(CubeMetastoreClient client) throws HiveException { - List<SchemaGraph.JoinPath> schemaGraphPaths = new ArrayList<SchemaGraph.JoinPath>(); + public List<JoinPath> getRelationEdges(CubeMetastoreClient client) throws HiveException { + List<JoinPath> joinPaths = new ArrayList<>(); for (Path path : paths) { JoinPath jp = new JoinPath(); // Add edges from dimension to cube @@ -328,8 +328,8 @@ public class JoinChain implements Named { jp.addEdge(path.links.get(i).toDimToDimRelationship(client)); } jp.addEdge(path.links.get(0).toCubeOrDimRelationship(client)); - schemaGraphPaths.add(jp); + joinPaths.add(jp); } - return schemaGraphPaths; + return joinPaths; } } http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAtrribute.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAtrribute.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAtrribute.java deleted file mode 100644 index c51b489..0000000 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAtrribute.java +++ /dev/null @@ -1,195 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.lens.cube.metadata; - -import java.util.*; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.hive.metastore.api.FieldSchema; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.ToString; - -@EqualsAndHashCode(callSuper = true) -@ToString(callSuper = true) -public class ReferencedDimAtrribute extends BaseDimAttribute { - private static final char CHAIN_REF_COL_SEPARATOR = ','; - - @Getter - private final List<TableReference> references = new ArrayList<>(); - // boolean whether to say the key is only a denormalized variable kept or can - // be used in join resolution as well - @Getter private Boolean isJoinKey = true; - @Getter private List<ChainRefCol> chainRefColumns = new ArrayList<>(); - - @Data - public static class ChainRefCol { - private final String chainName; - private final String refColumn; - } - - public ReferencedDimAtrribute(FieldSchema column, String displayString, TableReference reference) { - this(column, displayString, reference, null, null, null); - } - - public ReferencedDimAtrribute(FieldSchema column, String displayString, TableReference reference, Date startTime, - Date endTime, Double cost) { - this(column, displayString, reference, startTime, endTime, cost, true); - } - - public ReferencedDimAtrribute(FieldSchema column, String displayString, TableReference reference, Date startTime, - Date endTime, Double cost, boolean isJoinKey) { - this(column, displayString, reference, startTime, endTime, cost, isJoinKey, null); - } - - public ReferencedDimAtrribute(FieldSchema column, String displayString, TableReference reference, Date startTime, - Date endTime, Double cost, boolean isJoinKey, Long numOfDistinctValues) { - super(column, displayString, startTime, endTime, cost, numOfDistinctValues); - this.references.add(reference); - this.isJoinKey = isJoinKey; - } - - public ReferencedDimAtrribute(FieldSchema column, String displayString, Collection<TableReference> references) { - this(column, displayString, references, null, null, null); - } - - public ReferencedDimAtrribute(FieldSchema column, String displayString, Collection<TableReference> references, - Date startTime, Date endTime, Double cost) { - this(column, displayString, references, startTime, endTime, cost, true); - } - - public ReferencedDimAtrribute(FieldSchema column, String displayString, Collection<TableReference> references, - Date startTime, Date endTime, Double cost, boolean isJoinKey) { - this(column, displayString, references, startTime, endTime, cost, isJoinKey, null); - } - - public ReferencedDimAtrribute(FieldSchema column, String displayString, Collection<TableReference> references, - Date startTime, Date endTime, Double cost, boolean isJoinKey, Long numOfDistinctValues) { - this(column, displayString, references, startTime, endTime, cost, isJoinKey, numOfDistinctValues, null); - } - - public ReferencedDimAtrribute(FieldSchema column, String displayString, Collection<TableReference> references, - Date startTime, Date endTime, Double cost, boolean isJoinKey, Long numOfDistinctValues, List<String> values) { - super(column, displayString, startTime, endTime, cost, numOfDistinctValues, values); - this.references.addAll(references); - this.isJoinKey = isJoinKey; - } - - public ReferencedDimAtrribute(FieldSchema column, String displayString, String chainName, String refColumn, - Date startTime, Date endTime, Double cost) { - this(column, displayString, chainName, refColumn, startTime, endTime, cost, null); - } - - public ReferencedDimAtrribute(FieldSchema column, String displayString, String chainName, String refColumn, - Date startTime, Date endTime, Double cost, Long numOfDistinctValues) { - this(column, displayString, - Collections.singletonList(new ChainRefCol(chainName.toLowerCase(), refColumn.toLowerCase())), startTime, endTime, - cost, numOfDistinctValues); - } - - public ReferencedDimAtrribute(FieldSchema column, String displayString, List<ChainRefCol> chainRefCols, - Date startTime, Date endTime, Double cost, Long numOfDistinctValues) { - this(column, displayString, chainRefCols, startTime, endTime, cost, numOfDistinctValues, null); - } - - public ReferencedDimAtrribute(FieldSchema column, String displayString, List<ChainRefCol> chainRefCols, - Date startTime, Date endTime, Double cost, Long numOfDistinctValues, List<String> values) { - super(column, displayString, startTime, endTime, cost, numOfDistinctValues, values); - chainRefColumns.addAll(chainRefCols); - this.isJoinKey = false; - } - - public void addReference(TableReference reference) { - references.add(reference); - } - - public boolean removeReference(TableReference ref) { - return references.remove(ref); - } - - public boolean useAsJoinKey() { - return isJoinKey; - } - - @Override - public void addProperties(Map<String, String> props) { - super.addProperties(props); - if (!chainRefColumns.isEmpty()) { - StringBuilder chainNamesValue = new StringBuilder(); - StringBuilder refColsValue = new StringBuilder(); - Iterator<ChainRefCol> iter = chainRefColumns.iterator(); - // Add the first without appending separator - ChainRefCol chainRefCol = iter.next(); - chainNamesValue.append(chainRefCol.getChainName()); - refColsValue.append(chainRefCol.getRefColumn()); - while (iter.hasNext()) { - chainRefCol = iter.next(); - chainNamesValue.append(CHAIN_REF_COL_SEPARATOR).append(chainRefCol.getChainName()); - refColsValue.append(CHAIN_REF_COL_SEPARATOR).append(chainRefCol.getRefColumn()); - } - props.put(MetastoreUtil.getDimRefChainNameKey(getName()), chainNamesValue.toString()); - props.put(MetastoreUtil.getDimRefChainColumnKey(getName()), refColsValue.toString()); - } else { - props.put(MetastoreUtil.getDimensionSrcReferenceKey(getName()), - MetastoreUtil.getReferencesString(references)); - props.put(MetastoreUtil.getDimUseAsJoinKey(getName()), isJoinKey.toString()); - } - } - - /** - * This is used only for serializing - * - * @param name - * @param props - */ - public ReferencedDimAtrribute(String name, Map<String, String> props) { - super(name, props); - String chNamesStr = props.get(MetastoreUtil.getDimRefChainNameKey(getName())); - if (!StringUtils.isBlank(chNamesStr)) { - String refColsStr = props.get(MetastoreUtil.getDimRefChainColumnKey(getName())); - String[] chainNames = StringUtils.split(chNamesStr, ","); - String[] refCols = StringUtils.split(refColsStr, ","); - for (int i = 0; i < chainNames.length; i++) { - chainRefColumns.add(new ChainRefCol(chainNames[i], refCols[i])); - } - this.isJoinKey = false; - } else { - String refListStr = props.get(MetastoreUtil.getDimensionSrcReferenceKey(getName())); - String[] refListDims = StringUtils.split(refListStr, ","); - for (String refDimRaw : refListDims) { - references.add(new TableReference(refDimRaw)); - } - String isJoinKeyStr = props.get(MetastoreUtil.getDimUseAsJoinKey(name)); - if (isJoinKeyStr != null) { - isJoinKey = Boolean.parseBoolean(isJoinKeyStr); - } - } - } - - /** - * Tells whether the attribute is retrieved from chain - * - * @return true/false - */ - public boolean isChainedColumn() { - return !chainRefColumns.isEmpty(); - } -} http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAttribute.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAttribute.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAttribute.java new file mode 100644 index 0000000..9a1c44b --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAttribute.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.metadata; + +import java.util.*; + +import org.apache.lens.cube.error.LensCubeErrorCode; +import org.apache.lens.server.api.error.LensException; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hive.metastore.api.FieldSchema; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; + +@EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) +public class ReferencedDimAttribute extends BaseDimAttribute { + private static final char CHAIN_REF_COL_SEPARATOR = ','; + @Getter + private List<ChainRefCol> chainRefColumns = new ArrayList<>(); + + @Data + public static class ChainRefCol { + private final String chainName; + private final String refColumn; + } + + public ReferencedDimAttribute(FieldSchema column, String displayString, String chainName, String refColumn, + Date startTime, Date endTime, Double cost) throws LensException { + this(column, displayString, chainName, refColumn, startTime, endTime, cost, null); + } + + public ReferencedDimAttribute(FieldSchema column, String displayString, String chainName, String refColumn, + Date startTime, Date endTime, Double cost, Long numOfDistinctValues) throws LensException { + this(column, displayString, + Collections.singletonList(new ChainRefCol(chainName.toLowerCase(), refColumn.toLowerCase())), startTime, endTime, + cost, numOfDistinctValues); + } + + public ReferencedDimAttribute(FieldSchema column, String displayString, List<ChainRefCol> chainRefCols, + Date startTime, Date endTime, Double cost, Long numOfDistinctValues) throws LensException { + this(column, displayString, chainRefCols, startTime, endTime, cost, numOfDistinctValues, null); + } + + public ReferencedDimAttribute(FieldSchema column, String displayString, List<ChainRefCol> chainRefCols, + Date startTime, Date endTime, Double cost, Long numOfDistinctValues, List<String> values) throws LensException { + super(column, displayString, startTime, endTime, cost, numOfDistinctValues, values); + if (chainRefCols.isEmpty()) { + throw new LensException(LensCubeErrorCode.ERROR_IN_ENTITY_DEFINITION.getLensErrorInfo(), " Ref column: " + + getName() + " does not have any chain_ref_column defined"); + } + chainRefColumns.addAll(chainRefCols); + } + + @Override + public void addProperties(Map<String, String> props) { + super.addProperties(props); + StringBuilder chainNamesValue = new StringBuilder(); + StringBuilder refColsValue = new StringBuilder(); + Iterator<ChainRefCol> iterator = chainRefColumns.iterator(); + // Add the first without appending separator + ChainRefCol chainRefCol = iterator.next(); + chainNamesValue.append(chainRefCol.getChainName()); + refColsValue.append(chainRefCol.getRefColumn()); + while (iterator.hasNext()) { + chainRefCol = iterator.next(); + chainNamesValue.append(CHAIN_REF_COL_SEPARATOR).append(chainRefCol.getChainName()); + refColsValue.append(CHAIN_REF_COL_SEPARATOR).append(chainRefCol.getRefColumn()); + } + props.put(MetastoreUtil.getDimRefChainNameKey(getName()), chainNamesValue.toString()); + props.put(MetastoreUtil.getDimRefChainColumnKey(getName()), refColsValue.toString()); + } + + /** + * This is used only for serializing + * + * @param name attribute name + * @param props Properties + */ + public ReferencedDimAttribute(String name, Map<String, String> props) throws LensException { + super(name, props); + String chNamesStr = props.get(MetastoreUtil.getDimRefChainNameKey(getName())); + if (!StringUtils.isBlank(chNamesStr)) { + String refColsStr = props.get(MetastoreUtil.getDimRefChainColumnKey(getName())); + String[] chainNames = StringUtils.split(chNamesStr, ","); + String[] refCols = StringUtils.split(refColsStr, ","); + for (int i = 0; i < chainNames.length; i++) { + chainRefColumns.add(new ChainRefCol(chainNames[i], refCols[i])); + } + } else { + throw new LensException(LensCubeErrorCode.ERROR_IN_ENTITY_DEFINITION.getLensErrorInfo(), " Ref column: " + + getName() + " does not have any chain_ref_column defined"); + } + } + +} http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cube/src/main/java/org/apache/lens/cube/metadata/SchemaGraph.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/SchemaGraph.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/SchemaGraph.java deleted file mode 100644 index fa230ef..0000000 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/SchemaGraph.java +++ /dev/null @@ -1,377 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.lens.cube.metadata; - -import java.util.*; - -import org.apache.hadoop.hive.ql.metadata.HiveException; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.RequiredArgsConstructor; - -public class SchemaGraph { - /* - * An edge in the schema graph - */ - @Data - @AllArgsConstructor - @RequiredArgsConstructor - public static class TableRelationship { - final String fromColumn; - final AbstractCubeTable fromTable; - final String toColumn; - final AbstractCubeTable toTable; - boolean mapsToMany = false; - - @Override - public String toString() { - return fromTable.getName() + "." + fromColumn + "->" + toTable.getName() + "." + toColumn - + (mapsToMany ? "[n]" : ""); - } - - } - - /** - * A list of table relationships that can be combined to get a join clause - */ - public static class JoinPath { - final List<TableRelationship> edges; - // Store the map of a table against all columns of that table which are in the path - private Map<AbstractCubeTable, List<String>> columnsForTable = new HashMap<AbstractCubeTable, List<String>>(); - - public JoinPath() { - edges = new ArrayList<TableRelationship>(); - } - - public JoinPath(JoinPath other) { - edges = new ArrayList<TableRelationship>(other.edges); - } - - public void initColumnsForTable() { - if (!columnsForTable.isEmpty()) { - // already inited - return; - } - for (TableRelationship edge : edges) { - addColumnsForEdge(edge); - } - } - - public void addEdge(TableRelationship edge) { - edges.add(edge); - } - - public boolean isEmpty() { - return edges.isEmpty(); - } - - public List<TableRelationship> getEdges() { - return edges; - } - - private void addColumnsForEdge(TableRelationship edge) { - addColumn(edge.getFromTable(), edge.getFromColumn()); - addColumn(edge.getToTable(), edge.getToColumn()); - } - - private void addColumn(AbstractCubeTable table, String column) { - if (table == null || column == null) { - return; - } - List<String> columns = columnsForTable.get(table); - if (columns == null) { - columns = new ArrayList<String>(); - columnsForTable.put(table, columns); - } - columns.add(column); - } - - public List<String> getColumnsForTable(AbstractCubeTable table) { - return columnsForTable.get(table); - } - - public Set<AbstractCubeTable> getAllTables() { - return columnsForTable.keySet(); - } - - public boolean containsColumnOfTable(String column, AbstractCubeTable table) { - for (TableRelationship edge : edges) { - if ((table.equals(edge.getFromTable()) && column.equals(edge.getFromColumn())) - || table.equals(edge.getToTable()) && column.equals(edge.getToColumn())) { - return true; - } - } - return false; - } - - public String toString() { - return edges.toString(); - } - } - - /** - * Perform a search for join paths on the schema graph - */ - public static class GraphSearch { - private final AbstractCubeTable source; - private final AbstractCubeTable target; - // edges going out of the table - private final Map<AbstractCubeTable, Set<TableRelationship>> outGraph; - // egds coming into the table - private final Map<AbstractCubeTable, Set<TableRelationship>> inGraph; - // Used in tests to validate that all paths are searched - - public GraphSearch(AbstractCubeTable source, AbstractCubeTable target, SchemaGraph graph) { - this.source = source; - this.target = target; - - if (target instanceof CubeInterface) { - this.outGraph = graph.getCubeGraph((CubeInterface) target); - this.inGraph = graph.getCubeInGraph((CubeInterface) target); - } else if (target instanceof Dimension) { - this.outGraph = graph.getDimOnlyGraph(); - this.inGraph = graph.getDimOnlyInGraph(); - } else { - throw new IllegalArgumentException("Target neither cube nor dimension"); - } - } - - public List<JoinPath> findAllPathsToTarget() { - return findAllPathsToTarget(source, new JoinPath(), new HashSet<AbstractCubeTable>()); - } - - /** - * Recursive DFS to get all paths between source and target. Let path till this node = p Paths at node adjacent to - * target = [edges leading to target] Path at a random node = [path till this node + p for each p in - * path(neighbors)] - */ - List<JoinPath> findAllPathsToTarget(AbstractCubeTable source, JoinPath joinPathTillSource, - Set<AbstractCubeTable> visited) { - List<JoinPath> joinPaths = new ArrayList<JoinPath>(); - visited.add(source); - - if (inGraph.get(source) == null) { - return joinPaths; - } - for (TableRelationship edge : inGraph.get(source)) { - if (visited.contains(edge.getFromTable())) { - continue; - } - - - JoinPath p = new JoinPath(joinPathTillSource); - p.addEdge(edge); - AbstractCubeTable neighbor = edge.getFromTable(); - if (neighbor.getName().equals(target.getName())) { - // Got a direct path - joinPaths.add(p); - } else if (neighbor instanceof Dimension) { - List<JoinPath> pathsFromNeighbor = findAllPathsToTarget(neighbor, new JoinPath(p), visited); - for (JoinPath pn : pathsFromNeighbor) { - if (!pn.isEmpty()) { - joinPaths.add(pn); - } - } - } - } - - return joinPaths; - } - } - - /** - * Graph of tables in the cube metastore. Links between the tables are relationships in the cube. - */ - private final CubeMetastoreClient metastore; - // Graph for each cube - // graph with out going edges - private Map<CubeInterface, Map<AbstractCubeTable, Set<TableRelationship>>> cubeOutGraph; - // graph with incoming edges - private Map<CubeInterface, Map<AbstractCubeTable, Set<TableRelationship>>> cubeInGraph; - - // sub graph that contains only dimensions, mainly used while checking connectivity between a set of dimensions - // graph with out going edges - private Map<AbstractCubeTable, Set<TableRelationship>> dimOnlyOutGraph; - // graph with incoming edges - private Map<AbstractCubeTable, Set<TableRelationship>> dimOnlyInGraph; - - public SchemaGraph(CubeMetastoreClient metastore) throws HiveException { - this.metastore = metastore; - buildSchemaGraph(); - } - - public Map<AbstractCubeTable, Set<TableRelationship>> getCubeGraph(CubeInterface cube) { - return cubeOutGraph.get(cube); - } - - public Map<AbstractCubeTable, Set<TableRelationship>> getDimOnlyGraph() { - return dimOnlyOutGraph; - } - - public Map<AbstractCubeTable, Set<TableRelationship>> getCubeInGraph(CubeInterface cube) { - return cubeInGraph.get(cube); - } - - public Map<AbstractCubeTable, Set<TableRelationship>> getDimOnlyInGraph() { - return dimOnlyInGraph; - } - - /** - * Build the schema graph for all cubes and dimensions - * - * @return - * @throws org.apache.hadoop.hive.ql.metadata.HiveException - */ - private void buildSchemaGraph() throws HiveException { - cubeOutGraph = new HashMap<CubeInterface, Map<AbstractCubeTable, Set<TableRelationship>>>(); - cubeInGraph = new HashMap<CubeInterface, Map<AbstractCubeTable, Set<TableRelationship>>>(); - for (CubeInterface cube : metastore.getAllCubes()) { - Map<AbstractCubeTable, Set<TableRelationship>> outGraph - = new HashMap<AbstractCubeTable, Set<TableRelationship>>(); - Map<AbstractCubeTable, Set<TableRelationship>> inGraph - = new HashMap<AbstractCubeTable, Set<TableRelationship>>(); - buildGraph((AbstractCubeTable) cube, outGraph, inGraph); - - for (Dimension dim : metastore.getAllDimensions()) { - buildGraph(dim, outGraph, inGraph); - } - - cubeOutGraph.put(cube, outGraph); - cubeInGraph.put(cube, inGraph); - } - - dimOnlyOutGraph = new HashMap<AbstractCubeTable, Set<TableRelationship>>(); - dimOnlyInGraph = new HashMap<AbstractCubeTable, Set<TableRelationship>>(); - for (Dimension dim : metastore.getAllDimensions()) { - buildGraph(dim, dimOnlyOutGraph, dimOnlyInGraph); - } - } - - private List<CubeDimAttribute> getRefDimensions(AbstractCubeTable cube) throws HiveException { - List<CubeDimAttribute> refDimensions = new ArrayList<CubeDimAttribute>(); - Set<CubeDimAttribute> allAttrs = null; - if (cube instanceof CubeInterface) { - allAttrs = ((CubeInterface) cube).getDimAttributes(); - } else if (cube instanceof Dimension) { - allAttrs = ((Dimension) cube).getAttributes(); - } else { - throw new HiveException("Not a valid table type" + cube); - } - // find out all dimensions which link to other dimension tables - for (CubeDimAttribute dim : allAttrs) { - if (dim instanceof ReferencedDimAtrribute) { - if (((ReferencedDimAtrribute) dim).useAsJoinKey()) { - refDimensions.add(dim); - } - } else if (dim instanceof HierarchicalDimAttribute) { - for (CubeDimAttribute hdim : ((HierarchicalDimAttribute) dim).getHierarchy()) { - if (hdim instanceof ReferencedDimAtrribute && ((ReferencedDimAtrribute) hdim).useAsJoinKey()) { - refDimensions.add(hdim); - } - } - } - } - return refDimensions; - } - - // Build schema graph for a cube/dimension - private void buildGraph(AbstractCubeTable cubeTable, Map<AbstractCubeTable, Set<TableRelationship>> outGraph, - Map<AbstractCubeTable, Set<TableRelationship>> inGraph) - throws HiveException { - List<CubeDimAttribute> refDimensions = getRefDimensions(cubeTable); - - // build graph for each linked dimension - for (CubeDimAttribute dim : refDimensions) { - // Find out references leading from dimension columns of the cube/dimension if any - if (dim instanceof ReferencedDimAtrribute) { - ReferencedDimAtrribute refDim = (ReferencedDimAtrribute) dim; - List<TableReference> refs = refDim.getReferences(); - - for (TableReference ref : refs) { - String destColumnName = ref.getDestColumn(); - String destTableName = ref.getDestTable(); - - if (metastore.isDimension(destTableName)) { - // Cube -> Dimension or Dimension -> Dimension reference - Dimension relatedDim = metastore.getDimension(destTableName); - addLinks(refDim.getName(), cubeTable, destColumnName, relatedDim, ref.isMapsToMany(), outGraph, inGraph); - } else { - throw new HiveException("Dim -> Cube references are not supported: " + dim.getName() + "." - + refDim.getName() + "->" + destTableName + "." + destColumnName); - } - } // end loop for refs from a dim - } - } - } - - private void addLinks(String srcCol, AbstractCubeTable srcTbl, String destCol, AbstractCubeTable destTbl, - boolean mapsToMany, Map<AbstractCubeTable, Set<TableRelationship>> outGraph, - Map<AbstractCubeTable, Set<TableRelationship>> inGraph) { - - TableRelationship rel = new TableRelationship(srcCol, srcTbl, destCol, destTbl, mapsToMany); - - Set<TableRelationship> inEdges = inGraph.get(destTbl); - if (inEdges == null) { - inEdges = new LinkedHashSet<TableRelationship>(); - inGraph.put(destTbl, inEdges); - } - inEdges.add(rel); - - Set<TableRelationship> outEdges = outGraph.get(srcTbl); - if (outEdges == null) { - outEdges = new LinkedHashSet<TableRelationship>(); - outGraph.put(srcTbl, outEdges); - } - - outEdges.add(rel); - - } - - public void print() { - for (CubeInterface cube : cubeOutGraph.keySet()) { - Map<AbstractCubeTable, Set<TableRelationship>> graph = cubeOutGraph.get(cube); - System.out.println("**Cube " + cube.getName() + " Out egdes"); - System.out.println("--Out Graph-Nodes=" + graph.size()); - for (AbstractCubeTable tab : graph.keySet()) { - System.out.println(tab.getName() + "::" + graph.get(tab)); - } - } - System.out.println("**Dim only outgraph"); - System.out.println("--Out Graph-Nodes=" + dimOnlyOutGraph.size()); - for (AbstractCubeTable tab : dimOnlyOutGraph.keySet()) { - System.out.println(tab.getName() + "::" + dimOnlyOutGraph.get(tab)); - } - - for (CubeInterface cube : cubeInGraph.keySet()) { - Map<AbstractCubeTable, Set<TableRelationship>> graph = cubeInGraph.get(cube); - System.out.println("**Cube " + cube.getName() + " In egdes"); - System.out.println("--In Graph-Nodes=" + graph.size()); - for (AbstractCubeTable tab : graph.keySet()) { - System.out.println(tab.getName() + "::" + graph.get(tab)); - } - } - System.out.println("**Dim only Ingraph"); - System.out.println("--In Graph-Nodes=" + dimOnlyInGraph.size()); - for (AbstractCubeTable tab : dimOnlyInGraph.keySet()) { - System.out.println(tab.getName() + "::" + dimOnlyInGraph.get(tab)); - } - - } -}