LENS-575 : Fix NPE when unreachable dims are queried (amareshwari)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/b33e6e18 Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/b33e6e18 Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/b33e6e18 Branch: refs/heads/current-release-line Commit: b33e6e18da3578afc442b016a7a168c8f0ba88db Parents: 558ac1c Author: Amareshwari Sriramadasu <[email protected]> Authored: Fri Jun 12 16:15:46 2015 +0530 Committer: Amareshwari Sriramadasu <[email protected]> Committed: Fri Jun 12 16:15:46 2015 +0530 ---------------------------------------------------------------------- lens-api/src/main/resources/cube-0.1.xsd | 2 +- .../lens/cube/parse/CandidateTableResolver.java | 23 +++++++++--- .../apache/lens/cube/parse/CubeTestSetup.java | 39 ++++++++++++++++++++ .../lens/cube/parse/TestJoinResolver.java | 14 +++++++ 4 files changed, 71 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b33e6e18/lens-api/src/main/resources/cube-0.1.xsd ---------------------------------------------------------------------- diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd index 2ae606f..cd898cc 100644 --- a/lens-api/src/main/resources/cube-0.1.xsd +++ b/lens-api/src/main/resources/cube-0.1.xsd @@ -448,7 +448,7 @@ </xs:documentation> </xs:annotation> </xs:attribute> - <xs:attribute type="xs:boolean" name="join_key" default="false"> + <xs:attribute type="xs:boolean" name="join_key" default="true"> <xs:annotation> <xs:documentation> This flag will tell whether the attribute can be used as a join key or not http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b33e6e18/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java index 53fb11f..79d6d43 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java @@ -493,19 +493,30 @@ class CandidateTableResolver implements ContextRewriter { return; } // check for source columns for denorm columns - Map<CandidateTable, List<String>> removedCandidates = new HashMap<CandidateTable, List<String>>(); + Map<CandidateTable, Collection<String>> removedCandidates = new HashMap<CandidateTable, Collection<String>>(); for (Map.Entry<Dimension, OptionalDimCtx> optdimEntry : cubeql.getOptionalDimensionMap().entrySet()) { Dimension dim = optdimEntry.getKey(); OptionalDimCtx optdim = optdimEntry.getValue(); Iterator<CandidateTable> iter = optdim.requiredForCandidates.iterator(); while (iter.hasNext()) { CandidateTable candidate = iter.next(); - List<String> colSet = cubeql.getAutoJoinCtx().getJoinPathFromColumns().get(dim).get(candidate.getBaseTable()); - if (!checkForColumnExists(candidate, colSet)) { - LOG.info("Removing candidate" + candidate + " from requiredForCandidates of" + dim + ", as columns:" + colSet - + " do not exist"); + boolean remove = false; + if (cubeql.getAutoJoinCtx().getJoinPathFromColumns().get(dim) == null) { + LOG.info("Removing candidate" + candidate + " from requiredForCandidates of" + dim + ", as no join paths" + + " exist"); + remove = true; + removedCandidates.put(candidate, optdim.colQueried); + } else { + List<String> colSet = cubeql.getAutoJoinCtx().getJoinPathFromColumns().get(dim).get(candidate.getBaseTable()); + if (!checkForColumnExists(candidate, colSet)) { + LOG.info("Removing candidate" + candidate + " from requiredForCandidates of" + dim + ", as columns:" + + colSet + " do not exist"); + remove = true; + removedCandidates.put(candidate, colSet); + } + } + if (remove) { iter.remove(); - removedCandidates.put(candidate, colSet); } } } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b33e6e18/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java index 554e709..49fabff 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java @@ -616,6 +616,8 @@ public class CubeTestSetup { new TableReference("testdim2", "id"))); cubeDimensions.add(new ReferencedDimAtrribute(new FieldSchema("cdim2", "int", "ref dim"), "Dim2 refer", new TableReference("cycledim1", "id"), NOW, null, null)); + cubeDimensions.add(new ReferencedDimAtrribute(new FieldSchema("urdimid", "int", "ref dim"), "urdim refer", + new TableReference("unreachableDim", "id"), null, null, null, false, 10L)); // denormalized reference cubeDimensions.add(new ReferencedDimAtrribute(new FieldSchema("dim2big1", "bigint", "ref dim"), "Dim2 refer", @@ -1573,6 +1575,8 @@ public class CubeTestSetup { new TableReference("citydim", "id"))); dimAttrs.add(new ReferencedDimAtrribute(new FieldSchema("cityname", "string", "name"), "cityid", new TableReference("citydim", "name"), null, null, 0.0, false)); + dimAttrs.add(new ReferencedDimAtrribute(new FieldSchema("urdimid", "int", "ref dim"), "urdim refer", + new TableReference("unreachableDim", "id"), null, null, null, false, 10L)); // add ref dim through chain dimAttrs.add(new ReferencedDimAtrribute( @@ -1909,6 +1913,40 @@ public class CubeTestSetup { client.createCubeDimensionTable(dimName, dimTblName, dimColumns, 0L, dumpPeriods, dimProps, storageTables); } + private void createUnReachabletable(CubeMetastoreClient client) throws Exception { + String dimName = "unreachableDim"; + + Set<CubeDimAttribute> dimAttrs = new HashSet<CubeDimAttribute>(); + dimAttrs.add(new BaseDimAttribute(new FieldSchema("id", "int", "code"))); + dimAttrs.add(new BaseDimAttribute(new FieldSchema("name", "int", "code"))); + Map<String, String> dimProps = new HashMap<String, String>(); + dimProps.put(MetastoreUtil.getDimTimedDimensionKey(dimName), TestCubeMetastoreClient.getDatePartitionKey()); + Dimension urDim = new Dimension(dimName, dimAttrs, dimProps, 0L); + client.createDimension(urDim); + + String dimTblName = "unreachableDimTable"; + List<FieldSchema> dimColumns = new ArrayList<FieldSchema>(); + dimColumns.add(new FieldSchema("id", "int", "code")); + dimColumns.add(new FieldSchema("name", "string", "field1")); + + Map<String, UpdatePeriod> dumpPeriods = new HashMap<String, UpdatePeriod>(); + ArrayList<FieldSchema> partCols = new ArrayList<FieldSchema>(); + List<String> timePartCols = new ArrayList<String>(); + partCols.add(TestCubeMetastoreClient.getDatePartition()); + timePartCols.add(TestCubeMetastoreClient.getDatePartitionKey()); + StorageTableDesc s1 = new StorageTableDesc(); + s1.setInputFormat(TextInputFormat.class.getCanonicalName()); + s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); + s1.setPartCols(partCols); + s1.setTimePartCols(timePartCols); + dumpPeriods.put(c1, HOURLY); + + Map<String, StorageTableDesc> storageTables = new HashMap<String, StorageTableDesc>(); + storageTables.put(c1, s1); + + client.createCubeDimensionTable(dimName, dimTblName, dimColumns, 0L, dumpPeriods, dimProps, storageTables); + } + private void createCountryTable(CubeMetastoreClient client) throws Exception { String dimName = "countrydim"; @@ -2050,6 +2088,7 @@ public class CubeTestSetup { createCountryTable(client); createStateTable(client); createCubeFactsWithValidColumns(client); + createUnReachabletable(client); } catch (Exception exc) { log.error("Exception while creating sources.", exc); throw exc; http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b33e6e18/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java index d121ba4..ed08605 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java @@ -21,6 +21,8 @@ package org.apache.lens.cube.parse; import static org.apache.lens.cube.parse.CubeTestSetup.*; +import static org.testng.Assert.*; + import java.util.*; import org.apache.lens.cube.metadata.*; @@ -30,6 +32,7 @@ import org.apache.lens.server.api.error.LensException; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.ParseException; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -739,4 +742,15 @@ public class TestJoinResolver extends TestQueryRewrite { Assert.assertFalse(cdimTables.contains("citytable3")); Assert.assertFalse(cdimTables.contains("citytable4")); } + + @Test + public void testUnreachableDim() throws ParseException, LensException { + SemanticException e1 = getSemanticExceptionInRewrite("select urdimid from testdim2", hconf); + assertNotNull(e1); + assertEquals(e1.getCanonicalErrorMsg().getErrorCode(), ErrorMsg.NO_DIM_HAS_COLUMN.getErrorCode()); + + SemanticException e2 = getSemanticExceptionInRewrite("select urdimid from testcube where " + TWO_DAYS_RANGE, hconf); + assertNotNull(e2); + assertEquals(e2.getCanonicalErrorMsg().getErrorCode(), ErrorMsg.NO_CANDIDATE_FACT_AVAILABLE.getErrorCode()); + } }
