http://git-wip-us.apache.org/repos/asf/lens/blob/4bcb7aa9/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java index 31d2d2c..12d6e3a 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java @@ -31,7 +31,6 @@ import org.apache.lens.server.api.error.LensException; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.ql.metadata.HiveException; import com.google.common.collect.Sets; @@ -88,27 +87,23 @@ class CandidateTableResolver implements ContextRewriter { } private void populateCandidateTables(CubeQueryContext cubeql) throws LensException { - try { - if (cubeql.getCube() != null) { - List<CubeFactTable> factTables = cubeql.getMetastoreClient().getAllFacts(cubeql.getCube()); - if (factTables.isEmpty()) { - throw new LensException(LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo(), - cubeql.getCube().getName() + " does not have any facts"); - } - for (CubeFactTable fact : factTables) { - CandidateFact cfact = new CandidateFact(fact, cubeql.getCube()); - cubeql.getCandidateFacts().add(cfact); - } - log.info("Populated candidate facts: {}", cubeql.getCandidateFacts()); + if (cubeql.getCube() != null) { + List<CubeFactTable> factTables = cubeql.getMetastoreClient().getAllFacts(cubeql.getCube()); + if (factTables.isEmpty()) { + throw new LensException(LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo(), + cubeql.getCube().getName() + " does not have any facts"); + } + for (CubeFactTable fact : factTables) { + CandidateFact cfact = new CandidateFact(fact, cubeql.getCube()); + cubeql.getCandidateFacts().add(cfact); } + log.info("Populated candidate facts: {}", cubeql.getCandidateFacts()); + } - if (cubeql.getDimensions().size() != 0) { - for (Dimension dim : cubeql.getDimensions()) { - populateDimTables(dim, cubeql, false); - } + if (cubeql.getDimensions().size() != 0) { + for (Dimension dim : cubeql.getDimensions()) { + populateDimTables(dim, cubeql, false); } - } catch (HiveException e) { - throw new LensException(e); } } @@ -116,27 +111,23 @@ class CandidateTableResolver implements ContextRewriter { if (cubeql.getCandidateDimTables().get(dim) != null) { return; } - try { - Set<CandidateDim> candidates = new HashSet<>(); - cubeql.getCandidateDimTables().put(dim, candidates); - List<CubeDimensionTable> dimtables = cubeql.getMetastoreClient().getAllDimensionTables(dim); - if (dimtables.isEmpty()) { - if (!optional) { - throw new LensException(LensCubeErrorCode.NO_CANDIDATE_DIM_AVAILABLE.getLensErrorInfo(), - dim.getName().concat(" has no dimension tables")); - } else { - log.info("Not considering optional dimension {} as, No dimension tables exist", dim); - removeOptionalDimWithoutAlias(cubeql, dim); - } - } - for (CubeDimensionTable dimtable : dimtables) { - CandidateDim cdim = new CandidateDim(dimtable, dim); - candidates.add(cdim); + Set<CandidateDim> candidates = new HashSet<>(); + cubeql.getCandidateDimTables().put(dim, candidates); + List<CubeDimensionTable> dimtables = cubeql.getMetastoreClient().getAllDimensionTables(dim); + if (dimtables.isEmpty()) { + if (!optional) { + throw new LensException(LensCubeErrorCode.NO_CANDIDATE_DIM_AVAILABLE.getLensErrorInfo(), + dim.getName().concat(" has no dimension tables")); + } else { + log.info("Not considering optional dimension {} as, No dimension tables exist", dim); + removeOptionalDimWithoutAlias(cubeql, dim); } - log.info("Populated candidate dims: {} for {}", cubeql.getCandidateDimTables().get(dim), dim); - } catch (HiveException e) { - throw new LensException(e); } + for (CubeDimensionTable dimtable : dimtables) { + CandidateDim cdim = new CandidateDim(dimtable, dim); + candidates.add(cdim); + } + log.info("Populated candidate dims: {} for {}", cubeql.getCandidateDimTables().get(dim), dim); } private void removeOptionalDimWithoutAlias(CubeQueryContext cubeql, Dimension dim) {
http://git-wip-us.apache.org/repos/asf/lens/blob/4bcb7aa9/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java index cf32746..63ec8b2 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java @@ -334,7 +334,8 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { } else { return false; } - } catch (HiveException e) { + } catch (LensException e) { + //TODO: check if catch can be removed return false; } return true; http://git-wip-us.apache.org/repos/asf/lens/blob/4bcb7aa9/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java index 36ee9d4..7f4aa93 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java @@ -29,7 +29,6 @@ import org.apache.lens.cube.metadata.ReferencedDimAttribute.ChainRefCol; import org.apache.lens.cube.parse.ExpressionResolver.ExprSpecContext; import org.apache.lens.server.api.error.LensException; -import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.HiveParser; @@ -54,11 +53,7 @@ public class FieldValidator implements ContextRewriter { if (!cube.allFieldsQueriable()) { // do queried field validation List<DerivedCube> dcubes; - try { - dcubes = cubeql.getMetastoreClient().getAllDerivedQueryableCubes(cube); - } catch (HiveException e) { - throw new LensException(e); - } + dcubes = cubeql.getMetastoreClient().getAllDerivedQueryableCubes(cube); ImmutableSet<String> queriedTimeDimCols = cubeql.getQueriedTimeDimCols(); http://git-wip-us.apache.org/repos/asf/lens/blob/4bcb7aa9/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java index 7dceee1..7b865bf 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java @@ -88,7 +88,7 @@ class JoinResolver implements ContextRewriter { } } - private void processJoinChains(CubeQueryContext cubeql) throws HiveException { + private void processJoinChains(CubeQueryContext cubeql) throws HiveException, LensException { for (JoinChain chain : cubeql.getJoinchains().values()) { Set<String> dims = chain.getIntermediateDimensions(); http://git-wip-us.apache.org/repos/asf/lens/blob/4bcb7aa9/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java index f685500..db26718 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java @@ -318,38 +318,34 @@ class StorageTableResolver implements ContextRewriter { private TimeRange getFallbackRange(TimeRange range, CandidateFact cfact, CubeQueryContext cubeql) throws LensException { Cube baseCube = cubeql.getBaseCube(); - try { - ArrayList<String> tableNames = Lists.newArrayList(cfact.fact.getName(), cubeql.getCube().getName()); - if (!cubeql.getCube().getName().equals(baseCube.getName())) { - tableNames.add(baseCube.getName()); - } - String fallBackString = null; - String timedim = baseCube.getTimeDimOfPartitionColumn(range.getPartitionColumn()); - for (String tableName : tableNames) { - fallBackString = cubeql.getMetastoreClient().getTable(tableName).getParameters() - .get(MetastoreConstants.TIMEDIM_RELATION + timedim); - if (StringUtils.isNotBlank(fallBackString)) { - break; - } - } - if (StringUtils.isBlank(fallBackString)) { - return null; - } - Matcher matcher = Pattern.compile("(.*?)\\+\\[(.*?),(.*?)\\]").matcher(fallBackString.replaceAll(WSPACE, "")); - if (!matcher.matches()) { - return null; + ArrayList<String> tableNames = Lists.newArrayList(cfact.fact.getName(), cubeql.getCube().getName()); + if (!cubeql.getCube().getName().equals(baseCube.getName())) { + tableNames.add(baseCube.getName()); + } + String fallBackString = null; + String timedim = baseCube.getTimeDimOfPartitionColumn(range.getPartitionColumn()); + for (String tableName : tableNames) { + fallBackString = cubeql.getMetastoreClient().getTable(tableName).getParameters() + .get(MetastoreConstants.TIMEDIM_RELATION + timedim); + if (StringUtils.isNotBlank(fallBackString)) { + break; } - DateUtil.TimeDiff diff1 = DateUtil.TimeDiff.parseFrom(matcher.group(2).trim()); - DateUtil.TimeDiff diff2 = DateUtil.TimeDiff.parseFrom(matcher.group(3).trim()); - String relatedTimeDim = matcher.group(1).trim(); - String fallbackPartCol = baseCube.getPartitionColumnOfTimeDim(relatedTimeDim); - return TimeRange.getBuilder() - .fromDate(diff2.negativeOffsetFrom(range.getFromDate())) - .toDate(diff1.negativeOffsetFrom(range.getToDate())) - .partitionColumn(fallbackPartCol).build(); - } catch (HiveException e) { - throw new LensException(e); } + if (StringUtils.isBlank(fallBackString)) { + return null; + } + Matcher matcher = Pattern.compile("(.*?)\\+\\[(.*?),(.*?)\\]").matcher(fallBackString.replaceAll(WSPACE, "")); + if (!matcher.matches()) { + return null; + } + DateUtil.TimeDiff diff1 = DateUtil.TimeDiff.parseFrom(matcher.group(2).trim()); + DateUtil.TimeDiff diff2 = DateUtil.TimeDiff.parseFrom(matcher.group(3).trim()); + String relatedTimeDim = matcher.group(1).trim(); + String fallbackPartCol = baseCube.getPartitionColumnOfTimeDim(relatedTimeDim); + return TimeRange.getBuilder() + .fromDate(diff2.negativeOffsetFrom(range.getFromDate())) + .toDate(diff1.negativeOffsetFrom(range.getToDate())) + .partitionColumn(fallbackPartCol).build(); } private void resolveFactStoragePartitions(CubeQueryContext cubeql) throws LensException { http://git-wip-us.apache.org/repos/asf/lens/blob/4bcb7aa9/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java index 7725043..fd6c30d 100644 --- a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java +++ b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java @@ -27,9 +27,9 @@ import org.apache.lens.cube.parse.CandidateTable; import org.apache.lens.cube.parse.CubeQueryContext; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.driver.DriverQueryPlan; +import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.query.cost.QueryCost; -import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; import lombok.extern.slf4j.Slf4j; @@ -73,7 +73,7 @@ public final class RewriterPlan extends DriverQueryPlan { Table tbl; try { tbl = ctx.getMetastoreClient().getTable(table); - } catch (HiveException e) { + } catch (LensException e) { log.error("Error while getting table:" + table, e); continue; } http://git-wip-us.apache.org/repos/asf/lens/blob/4bcb7aa9/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index f84f2e0..62c7ec9 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -639,7 +639,7 @@ public class TestCubeMetastoreClient { } private void validateDim(Dimension udim, Set<CubeDimAttribute> attrs, String basedim, String referdim) - throws HiveException { + throws HiveException, LensException { assertTrue(client.tableExists(udim.getName())); Table dimTbl = client.getHiveTable(udim.getName()); assertTrue(client.isDimension(dimTbl)); @@ -1048,14 +1048,14 @@ public class TestCubeMetastoreClient { // test partition StoragePartitionDesc partSpec = new StoragePartitionDesc(cubeFact.getName(), timeParts, null, HOURLY); try { - client.addPartition(partSpec, c1); + client.addPartition(partSpec, c1, CubeTableType.FACT); fail("Add should fail since non_existing_part_col is non-existing"); } catch (LensException e) { assertEquals(e.getErrorCode(), LensCubeErrorCode.TIMELINE_ABSENT.getLensErrorInfo().getErrorCode()); } timeParts.remove("non_existing_part_col"); partSpec = new StoragePartitionDesc(cubeFact.getName(), timeParts, null, HOURLY); - client.addPartition(partSpec, c1); + client.addPartition(partSpec, c1, CubeTableType.FACT); assertTrue(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts, emptyHashMap)); assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); @@ -1072,7 +1072,7 @@ public class TestCubeMetastoreClient { StoragePartitionDesc partSpec2 = new StoragePartitionDesc(cubeFact.getName(), timeParts2, null, HOURLY); partSpec2.setInputFormat(SequenceFileInputFormat.class.getCanonicalName()); partSpec2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - client.addPartition(partSpec2, c1); + client.addPartition(partSpec2, c1, CubeTableType.FACT); assertEquals(client.getAllParts(storageTableName).size(), 1); assertTrue(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts, emptyHashMap)); assertFalse(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts2, emptyHashMap)); @@ -1334,7 +1334,7 @@ public class TestCubeMetastoreClient { //test partition Map<String, Date> timeParts = getTimePartitionByOffsets(getDatePartitionKey(), 0, testDtPart.getName(), -1); StoragePartitionDesc partSpec = new StoragePartitionDesc(cubeFact.getName(), timeParts, null, HOURLY); - client.addPartition(partSpec, c1); + client.addPartition(partSpec, c1, CubeTableType.FACT); assertTrue(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts, emptyHashMap)); assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, testDtPart.getName())); assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); @@ -1434,8 +1434,8 @@ public class TestCubeMetastoreClient { final StoragePartitionDesc partSpec3 = new StoragePartitionDesc(cubeFact.getName(), timeParts3, nonTimeSpec, HOURLY); - client.addPartitions(Arrays.asList(partSpec1, partSpec2, partSpec3), c1); - client.addPartitions(Arrays.asList(partSpec1, partSpec2, partSpec3), c2); + client.addPartitions(Arrays.asList(partSpec1, partSpec2, partSpec3), c1, CubeTableType.FACT); + client.addPartitions(Arrays.asList(partSpec1, partSpec2, partSpec3), c2, CubeTableType.FACT); PartitionTimeline timeline1Temp = client.partitionTimelineCache.get(factName, c1, HOURLY, getDatePartitionKey()); PartitionTimeline timeline2Temp = client.partitionTimelineCache.get(factName, c2, HOURLY, getDatePartitionKey()); @@ -1467,8 +1467,8 @@ public class TestCubeMetastoreClient { etPart.getName(), -2); final StoragePartitionDesc partSpec5 = new StoragePartitionDesc(cubeFact.getName(), timeParts5, null, HOURLY); - client.addPartitions(Arrays.asList(partSpec4, partSpec5), c1); - client.addPartitions(Arrays.asList(partSpec4, partSpec5), c2); + client.addPartitions(Arrays.asList(partSpec4, partSpec5), c1, CubeTableType.FACT); + client.addPartitions(Arrays.asList(partSpec4, partSpec5), c2, CubeTableType.FACT); assertEquals(client.getAllParts(c1TableName).size(), 5); assertEquals(client.getAllParts(c2TableName).size(), 5); @@ -1487,8 +1487,8 @@ public class TestCubeMetastoreClient { etPart.getName(), -2); final StoragePartitionDesc partSpec6 = new StoragePartitionDesc(cubeFact.getName(), timeParts6, null, HOURLY); - client.addPartition(partSpec6, c1); - client.addPartition(partSpec6, c2); + client.addPartition(partSpec6, c1, CubeTableType.FACT); + client.addPartition(partSpec6, c2, CubeTableType.FACT); assertEquals(client.getAllParts(c1TableName).size(), 6); assertEquals(client.getAllParts(c2TableName).size(), 6); @@ -1503,8 +1503,8 @@ public class TestCubeMetastoreClient { etPart.getName(), -5); final StoragePartitionDesc partSpec7 = new StoragePartitionDesc(cubeFact.getName(), timeParts7, null, HOURLY); - client.addPartition(partSpec7, c1); - client.addPartition(partSpec7, c2); + client.addPartition(partSpec7, c1, CubeTableType.FACT); + client.addPartition(partSpec7, c2, CubeTableType.FACT); List<Partition> c1Parts = client.getAllParts(c1TableName); List<Partition> c2Parts = client.getAllParts(c2TableName); @@ -1763,7 +1763,7 @@ public class TestCubeMetastoreClient { // test partition Map<String, Date> timeParts = getTimePartitionByOffsets(getDatePartitionKey(), 0); StoragePartitionDesc partSpec = new StoragePartitionDesc(cubeFact.getName(), timeParts, null, HOURLY); - client.addPartition(partSpec, c1); + client.addPartition(partSpec, c1, CubeTableType.FACT); assertTrue(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts, emptyHashMap)); assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); String storageTableName = getFactOrDimtableStorageTableName(cubeFact.getName(), c1); @@ -1822,7 +1822,7 @@ public class TestCubeMetastoreClient { // test partition StoragePartitionDesc sPartSpec = new StoragePartitionDesc(cubeFactWithParts.getName(), timeParts, partSpec, HOURLY); - client.addPartition(sPartSpec, c1); + client.addPartition(sPartSpec, c1, CubeTableType.FACT); assertTrue(client.factPartitionExists(cubeFactWithParts.getName(), c1, HOURLY, timeParts, partSpec)); assertTrue(client.latestPartitionExists(cubeFactWithParts.getName(), c1, getDatePartitionKey())); assertFalse(client.latestPartitionExists(cubeFactWithParts.getName(), c1, factPartColumns.get(0).getName())); @@ -1898,7 +1898,7 @@ public class TestCubeMetastoreClient { storageDescs.add(sPartSpecNow); storageDescs.add(sPartSpecTwoMonthsBack); - client.addPartitions(storageDescs, c1); + client.addPartitions(storageDescs, c1, CubeTableType.FACT); assertTrue(client.factPartitionExists(cubeFactWithParts.getName(), c1, HOURLY, timePartsNow, partSpec)); assertFalse(client.factPartitionExists(cubeFactWithParts.getName(), c1, HOURLY, timePartsBeforeTwoMonths, partSpec)); @@ -1968,7 +1968,7 @@ public class TestCubeMetastoreClient { storageDescs.add(sPartSpecTwoMonthsBack); client.getTimelines(factNameSkipPart, null, null, null); - client.addPartitions(storageDescs, c1); + client.addPartitions(storageDescs, c1, CubeTableType.FACT); assertTrue(client.factPartitionExists(cubeFactWithParts.getName(), c1, HOURLY, timePartsNow, partSpec)); assertFalse(client.factPartitionExists(cubeFactWithParts.getName(), c1, HOURLY, timePartsBeforeTwoMonths, partSpec)); @@ -2022,7 +2022,7 @@ public class TestCubeMetastoreClient { // test partition StoragePartitionDesc sPartSpec = new StoragePartitionDesc(cubeFactWithParts.getName(), timeParts, partSpec, HOURLY); - client.addPartition(sPartSpec, c1); + client.addPartition(sPartSpec, c1, CubeTableType.FACT); assertTrue(client.factPartitionExists(cubeFactWithParts.getName(), c1, HOURLY, timeParts, partSpec)); assertTrue(client.latestPartitionExists(cubeFactWithParts.getName(), c1, getDatePartitionKey())); assertTrue(client.latestPartitionExists(cubeFactWithParts.getName(), c1, testDtPart.getName())); @@ -2086,7 +2086,7 @@ public class TestCubeMetastoreClient { // test partition StoragePartitionDesc sPartSpec = new StoragePartitionDesc(cubeFactWithTwoStorages.getName(), timeParts, partSpec, HOURLY); - client.addPartition(sPartSpec, c1); + client.addPartition(sPartSpec, c1, CubeTableType.FACT); assertTrue(client.factPartitionExists(cubeFactWithTwoStorages.getName(), c1, HOURLY, timeParts, partSpec)); assertTrue(client.latestPartitionExists(cubeFactWithTwoStorages.getName(), c1, getDatePartitionKey())); String storageTableName = getFactOrDimtableStorageTableName(cubeFactWithTwoStorages.getName(), c1); @@ -2096,7 +2096,7 @@ public class TestCubeMetastoreClient { StoragePartitionDesc sPartSpec2 = new StoragePartitionDesc(cubeFactWithTwoStorages.getName(), timeParts, null, HOURLY); - client.addPartition(sPartSpec2, c2); + client.addPartition(sPartSpec2, c2, CubeTableType.FACT); assertTrue(client.factPartitionExists(cubeFactWithTwoStorages.getName(), c2, HOURLY, timeParts, emptyHashMap)); assertTrue(client.latestPartitionExists(cubeFactWithTwoStorages.getName(), c2, getDatePartitionKey())); String storageTableName2 = getFactOrDimtableStorageTableName(cubeFactWithTwoStorages.getName(), c2); @@ -2159,7 +2159,7 @@ public class TestCubeMetastoreClient { // test partition Map<String, Date> timeParts = getTimePartitionByOffsets(getDatePartitionKey(), 0); StoragePartitionDesc sPartSpec = new StoragePartitionDesc(cubeDim.getName(), timeParts, null, HOURLY); - client.addPartition(sPartSpec, c1); + client.addPartition(sPartSpec, c1, CubeTableType.DIM_TABLE); assertTrue(client.dimPartitionExists(cubeDim.getName(), c1, timeParts)); assertTrue(client.latestPartitionExists(cubeDim.getName(), c1, getDatePartitionKey())); String storageTableName = getFactOrDimtableStorageTableName(dimName, c1); @@ -2229,7 +2229,7 @@ public class TestCubeMetastoreClient { Map<String, Date> timePartsNow = getHashMap(getDatePartitionKey(), NOW); StoragePartitionDesc sPartSpec0 = new StoragePartitionDesc(cubeDim.getName(), timePartsNow, null, HOURLY); - client.addPartition(sPartSpec0, c1); + client.addPartition(sPartSpec0, c1, CubeTableType.DIM_TABLE); assertTrue(client.dimPartitionExists(cubeDim.getName(), c1, timePartsNow)); assertTrue(client.latestPartitionExists(cubeDim.getName(), c1, getDatePartitionKey())); assertEquals(client.getAllParts(storageTableName).size(), 2); @@ -2249,7 +2249,7 @@ public class TestCubeMetastoreClient { StoragePartitionDesc sPartSpec1 = new StoragePartitionDesc(cubeDim.getName(), timeParts1, null, HOURLY); sPartSpec1.setInputFormat(SequenceFileInputFormat.class.getCanonicalName()); sPartSpec1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - client.addPartition(sPartSpec1, c1); + client.addPartition(sPartSpec1, c1, CubeTableType.DIM_TABLE); // assert on all partitions assertEquals(client.getAllParts(storageTableName).size(), 3); // non-latest partitions @@ -2267,7 +2267,7 @@ public class TestCubeMetastoreClient { // add one more partition Map<String, Date> timeParts2 = getTimePartitionByOffsets(getDatePartitionKey(), 2); StoragePartitionDesc sPartSpec2 = new StoragePartitionDesc(cubeDim.getName(), timeParts2, null, HOURLY); - client.addPartition(sPartSpec2, c1); + client.addPartition(sPartSpec2, c1, CubeTableType.DIM_TABLE); // assert on all partitions assertEquals(client.getAllParts(storageTableName).size(), 4); // non-latest partitions @@ -2314,7 +2314,7 @@ public class TestCubeMetastoreClient { assertEquals(client.getAllParts(storageTableName).size(), 0); assertFalse(client.dimTableLatestPartitionExists(storageTableName)); - client.addPartition(sPartSpec1, c1); + client.addPartition(sPartSpec1, c1, CubeTableType.DIM_TABLE); assertTrue(client.dimTableLatestPartitionExists(storageTableName)); client.dropStorageFromDim(cubeDim.getName(), c1); assertFalse(client.dimTableLatestPartitionExists(storageTableName)); @@ -2350,14 +2350,14 @@ public class TestCubeMetastoreClient { timeParts.put(getDatePartitionKey(), NOW); nonTimeParts.put("region", "asia"); StoragePartitionDesc sPartSpec = new StoragePartitionDesc(dimName, timeParts, nonTimeParts, HOURLY); - client.addPartition(sPartSpec, c3); + client.addPartition(sPartSpec, c3, CubeTableType.DIM_TABLE); expectedLatestValues.put("asia", NOW); assertLatestForRegions(storageTableName, expectedLatestValues); timeParts.put(getDatePartitionKey(), getDateWithOffset(HOURLY, -1)); nonTimeParts.put("region", "africa"); sPartSpec = new StoragePartitionDesc(dimName, timeParts, nonTimeParts, HOURLY); - client.addPartition(sPartSpec, c3); + client.addPartition(sPartSpec, c3, CubeTableType.DIM_TABLE); expectedLatestValues.put("asia", NOW); expectedLatestValues.put("africa", getDateWithOffset(HOURLY, -1)); assertLatestForRegions(storageTableName, expectedLatestValues); @@ -2365,7 +2365,7 @@ public class TestCubeMetastoreClient { timeParts.put(getDatePartitionKey(), getDateWithOffset(HOURLY, 1)); nonTimeParts.put("region", "africa"); sPartSpec = new StoragePartitionDesc(dimName, timeParts, nonTimeParts, HOURLY); - client.addPartition(sPartSpec, c3); + client.addPartition(sPartSpec, c3, CubeTableType.DIM_TABLE); expectedLatestValues.put("asia", NOW); expectedLatestValues.put("africa", getDateWithOffset(HOURLY, 1)); assertLatestForRegions(storageTableName, expectedLatestValues); @@ -2373,7 +2373,7 @@ public class TestCubeMetastoreClient { timeParts.put(getDatePartitionKey(), getDateWithOffset(HOURLY, 3)); nonTimeParts.put("region", "asia"); sPartSpec = new StoragePartitionDesc(dimName, timeParts, nonTimeParts, HOURLY); - client.addPartition(sPartSpec, c3); + client.addPartition(sPartSpec, c3, CubeTableType.DIM_TABLE); expectedLatestValues.put("asia", getDateWithOffset(HOURLY, 3)); expectedLatestValues.put("africa", getDateWithOffset(HOURLY, 1)); assertLatestForRegions(storageTableName, expectedLatestValues); @@ -2467,7 +2467,7 @@ public class TestCubeMetastoreClient { Map<String, Date> timeParts3 = getTimePartitionByOffsets(getDatePartitionKey(), 0, etPart.getName(), 0); final StoragePartitionDesc partSpec3 = new StoragePartitionDesc(cubeDim.getName(), timeParts3, nonTimeSpec, HOURLY); - client.addPartitions(Arrays.asList(partSpec1, partSpec2, partSpec3), c1); + client.addPartitions(Arrays.asList(partSpec1, partSpec2, partSpec3), c1, CubeTableType.DIM_TABLE); String c1TableName = getFactOrDimtableStorageTableName(cubeDim.getName(), c1); assertEquals(client.getAllParts(c1TableName).size(), 8); @@ -2481,7 +2481,7 @@ public class TestCubeMetastoreClient { etPart.getName(), -2); final StoragePartitionDesc partSpec5 = new StoragePartitionDesc(cubeDim.getName(), timeParts5, null, HOURLY); - client.addPartitions(Arrays.asList(partSpec4, partSpec5), c1); + client.addPartitions(Arrays.asList(partSpec4, partSpec5), c1, CubeTableType.DIM_TABLE); assertEquals(client.getAllParts(c1TableName).size(), 10); assertEquals(getLatestValues(c1TableName, HOURLY, partColNames, null), toPartitionArray(HOURLY, 1, 1, 1)); @@ -2489,7 +2489,7 @@ public class TestCubeMetastoreClient { etPart.getName(), -2); final StoragePartitionDesc partSpec6 = new StoragePartitionDesc(cubeDim.getName(), timeParts6, null, HOURLY); - client.addPartition(partSpec6, c1); + client.addPartition(partSpec6, c1, CubeTableType.DIM_TABLE); assertEquals(client.getAllParts(c1TableName).size(), 11); @@ -2497,7 +2497,7 @@ public class TestCubeMetastoreClient { etPart.getName(), -5); final StoragePartitionDesc partSpec7 = new StoragePartitionDesc(cubeDim.getName(), timeParts7, null, HOURLY); - client.addPartition(partSpec7, c1); + client.addPartition(partSpec7, c1, CubeTableType.DIM_TABLE); assertEquals(client.getAllParts(c1TableName).size(), 12); assertEquals(getLatestValues(c1TableName, HOURLY, partColNames, null), toPartitionArray(HOURLY, 1, 1, 1)); http://git-wip-us.apache.org/repos/asf/lens/blob/4bcb7aa9/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java index 86db011..ea6b4a1 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java @@ -1637,7 +1637,7 @@ public class CubeTestSetup { timeParts.put("ttd", temp); timeParts.put("ttd2", temp); StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY); - client.addPartition(sPartSpec, c99); + client.addPartition(sPartSpec, c99, CubeTableType.FACT); cal.add(HOUR_OF_DAY, 1); temp = cal.getTime(); } @@ -1650,7 +1650,7 @@ public class CubeTestSetup { timeParts.put("ttd", temp); timeParts.put("ttd2", temp); StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY); - client.addPartition(sPartSpec, c99); + client.addPartition(sPartSpec, c99, CubeTableType.FACT); cal.add(HOUR_OF_DAY, 1); temp = cal.getTime(); } @@ -1748,7 +1748,7 @@ public class CubeTestSetup { timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), temp); StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY); try { - client.addPartition(sPartSpec, c1); + client.addPartition(sPartSpec, c1, CubeTableType.FACT); } catch (HiveException e) { log.error("Encountered Hive exception.", e); } catch (LensException e) { @@ -1765,7 +1765,7 @@ public class CubeTestSetup { Map<String, Date> timeParts = new HashMap<String, Date>(); timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), temp); StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY); - client.addPartition(sPartSpec, c1); + client.addPartition(sPartSpec, c1, CubeTableType.FACT); cal.add(HOUR_OF_DAY, 1); temp = cal.getTime(); } @@ -1803,7 +1803,7 @@ public class CubeTestSetup { cal.add(HOUR_OF_DAY, 1); temp = cal.getTime(); } - client.addPartitions(storagePartitionDescs, c4); + client.addPartitions(storagePartitionDescs, c4, CubeTableType.FACT); client.clearHiveTableCache(); table = client.getTable(MetastoreUtil.getStorageTableName(fact.getName(), Storage.getPrefix(c4))); assertEquals(table.getParameters().get(MetastoreUtil.getPartitionTimelineCachePresenceKey()), "true"); @@ -1818,7 +1818,7 @@ public class CubeTestSetup { timeParts.put("ttd", temp); timeParts.put("ttd2", temp); StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY); - client.addPartition(sPartSpec, c4); + client.addPartition(sPartSpec, c4, CubeTableType.FACT); cal.add(HOUR_OF_DAY, 1); temp = cal.getTime(); } @@ -1879,7 +1879,7 @@ public class CubeTestSetup { Map<String, Date> timeParts = new HashMap<String, Date>(); timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), temp); StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact2.getName(), timeParts, null, HOURLY); - client.addPartition(sPartSpec, c3); + client.addPartition(sPartSpec, c3, CubeTableType.FACT); cal.add(HOUR_OF_DAY, 1); temp = cal.getTime(); } @@ -2983,7 +2983,7 @@ public class CubeTestSetup { StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, DAILY); pTimes.get(DAILY).add(ptime); iTimes.get(DAILY).add(itime); - client.addPartition(sPartSpec, storageName); + client.addPartition(sPartSpec, storageName, CubeTableType.FACT); pcal.add(DAY_OF_MONTH, 1); ical.add(HOUR_OF_DAY, 20); } else if (p == 2) { // day2 @@ -3000,7 +3000,7 @@ public class CubeTestSetup { StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, DAILY); pTimes.get(DAILY).add(ptime); iTimes.get(DAILY).add(itime); - client.addPartition(sPartSpec, storageName); + client.addPartition(sPartSpec, storageName, CubeTableType.FACT); // pt=day2-hour[0-3] it = day1-hour[20-23] // pt=day2-hour[4-23] it = day2-hour[0-19] for (int i = 0; i < 24; i++) { @@ -3012,7 +3012,7 @@ public class CubeTestSetup { sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY); pTimes.get(HOURLY).add(ptime); iTimes.get(HOURLY).add(itime); - client.addPartition(sPartSpec, storageName); + client.addPartition(sPartSpec, storageName, CubeTableType.FACT); pcal.add(HOUR_OF_DAY, 1); ical.add(HOUR_OF_DAY, 1); } @@ -3020,7 +3020,7 @@ public class CubeTestSetup { sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, DAILY); pTimes.get(DAILY).add(ptime); iTimes.get(DAILY).add(itime); - client.addPartition(sPartSpec, storageName); + client.addPartition(sPartSpec, storageName, CubeTableType.FACT); } else if (p == 3) { // day3 // pt=day3-hour[0-3] it = day2-hour[20-23] // pt=day3-hour[4-23] it = day3-hour[0-19] @@ -3034,7 +3034,7 @@ public class CubeTestSetup { new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY); pTimes.get(HOURLY).add(ptime); iTimes.get(HOURLY).add(itime); - client.addPartition(sPartSpec, storageName); + client.addPartition(sPartSpec, storageName, CubeTableType.FACT); pcal.add(HOUR_OF_DAY, 1); ical.add(HOUR_OF_DAY, 1); } http://git-wip-us.apache.org/repos/asf/lens/blob/4bcb7aa9/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java index 00d2a1c..6dff173 100644 --- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java +++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java @@ -62,6 +62,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver { /** The Constant HS2_PORT. */ static final int HS2_PORT = 12345; + static final int HS2_UI_PORT = 12346; /** The server. */ private static HiveServer2 server; @@ -91,6 +92,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver { remoteConf.set("hive.lock.manager", "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager"); HiveConf.setVar(remoteConf, HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST, HS2_HOST); HiveConf.setIntVar(remoteConf, HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT, HS2_PORT); + HiveConf.setIntVar(remoteConf, HiveConf.ConfVars.HIVE_SERVER2_WEBUI_PORT, HS2_UI_PORT); HiveConf.setIntVar(remoteConf, HiveConf.ConfVars.HIVE_SERVER2_THRIFT_CLIENT_CONNECTION_RETRY_LIMIT, 3); HiveConf.setIntVar(remoteConf, HiveConf.ConfVars.HIVE_SERVER2_THRIFT_CLIENT_RETRY_LIMIT, 3); HiveConf.setVar(remoteConf, HiveConf.ConfVars.HIVE_SERVER2_THRIFT_CLIENT_RETRY_DELAY_SECONDS, "10s"); http://git-wip-us.apache.org/repos/asf/lens/blob/4bcb7aa9/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java index 34312c0..b1fd459 100644 --- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java +++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java @@ -1328,7 +1328,7 @@ public class ColumnarSQLRewriter implements QueryRewriter { tabNameChild.insertChild(0, dbIdentifier); } } - } catch (HiveException e) { + } catch (LensException | HiveException e) { log.warn("No corresponding table in metastore:", e); } } @@ -1377,7 +1377,7 @@ public class ColumnarSQLRewriter implements QueryRewriter { /** * Gets the underlying db name. * - * @param table the table + * @param tbl the table * @return the underlying db name * @throws HiveException the hive exception */ @@ -1388,7 +1388,7 @@ public class ColumnarSQLRewriter implements QueryRewriter { /** * Gets the underlying table name. * - * @param table the table + * @param tbl the table * @return the underlying table name * @throws HiveException the hive exception */ http://git-wip-us.apache.org/repos/asf/lens/blob/4bcb7aa9/lens-server/src/main/java/org/apache/lens/server/BaseApp.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/BaseApp.java b/lens-server/src/main/java/org/apache/lens/server/BaseApp.java index 07f6837..19de31c 100644 --- a/lens-server/src/main/java/org/apache/lens/server/BaseApp.java +++ b/lens-server/src/main/java/org/apache/lens/server/BaseApp.java @@ -25,6 +25,7 @@ import javax.ws.rs.core.Application; import org.apache.lens.api.jaxb.LensJAXBContextResolver; import org.apache.lens.api.util.MoxyJsonConfigurationContextResolver; +import org.apache.lens.server.error.GenericExceptionMapper; import org.apache.lens.server.error.LensJAXBValidationExceptionMapper; import org.glassfish.jersey.filter.LoggingFilter; @@ -46,6 +47,7 @@ public abstract class BaseApp extends Application { classes.add(LensApplicationListener.class); classes.add(MoxyJsonConfigurationContextResolver.class); classes.add(MoxyJsonFeature.class); + classes.add(GenericExceptionMapper.class); return classes; } http://git-wip-us.apache.org/repos/asf/lens/blob/4bcb7aa9/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java b/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java index 0b4720e..74bc0be 100644 --- a/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java +++ b/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java @@ -537,5 +537,18 @@ public abstract class BaseLensService extends CompositeService implements Extern throw new LensException(SESSION_CLOSED.getLensErrorInfo(), handle); } } + + public class SessionContext implements AutoCloseable { + private LensSessionHandle sessionHandle; + + public SessionContext(LensSessionHandle sessionHandle) { + this.sessionHandle = sessionHandle; + acquire(sessionHandle); + } + @Override + public void close() { + release(sessionHandle); + } + } } http://git-wip-us.apache.org/repos/asf/lens/blob/4bcb7aa9/lens-server/src/main/java/org/apache/lens/server/LensApplication.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/LensApplication.java b/lens-server/src/main/java/org/apache/lens/server/LensApplication.java index 71ff9ea..bace84d 100644 --- a/lens-server/src/main/java/org/apache/lens/server/LensApplication.java +++ b/lens-server/src/main/java/org/apache/lens/server/LensApplication.java @@ -25,6 +25,7 @@ import javax.ws.rs.ApplicationPath; import javax.ws.rs.core.Application; import org.apache.lens.server.api.LensConfConstants; +import org.apache.lens.server.error.GenericExceptionMapper; import org.apache.hadoop.conf.Configuration; @@ -74,7 +75,7 @@ public class LensApplication extends Application { log.debug("LensRequestContextInitFilter added..."); classes.add(LensRequestContextInitFilter.class); - + classes.add(GenericExceptionMapper.class); return classes; } http://git-wip-us.apache.org/repos/asf/lens/blob/4bcb7aa9/lens-server/src/main/java/org/apache/lens/server/LensServices.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServices.java b/lens-server/src/main/java/org/apache/lens/server/LensServices.java index f6f2f36..7618669 100644 --- a/lens-server/src/main/java/org/apache/lens/server/LensServices.java +++ b/lens-server/src/main/java/org/apache/lens/server/LensServices.java @@ -227,7 +227,7 @@ public class LensServices extends CompositeService implements ServiceProvider { Class<? extends BaseLensService> serviceClass = (Class<? extends BaseLensService>) cls; log.info("Adding {} service with {}", sName, serviceClass); Constructor<?> constructor = serviceClass.getConstructor(CLIService.class); - BaseLensService service = (BaseLensService) constructor.newInstance(new Object[]{cliService}); + BaseLensService service = (BaseLensService) constructor.newInstance(cliService); addService(service); lensServices.add(service); } else if (Service.class.isAssignableFrom(cls)) { http://git-wip-us.apache.org/repos/asf/lens/blob/4bcb7aa9/lens-server/src/main/java/org/apache/lens/server/error/GenericExceptionMapper.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/error/GenericExceptionMapper.java b/lens-server/src/main/java/org/apache/lens/server/error/GenericExceptionMapper.java index 658fcb2..9aa74ed 100644 --- a/lens-server/src/main/java/org/apache/lens/server/error/GenericExceptionMapper.java +++ b/lens-server/src/main/java/org/apache/lens/server/error/GenericExceptionMapper.java @@ -24,6 +24,7 @@ import javax.ws.rs.core.Response; import javax.ws.rs.ext.ExceptionMapper; import javax.ws.rs.ext.Provider; +import org.apache.lens.api.APIResult; import org.apache.lens.api.error.ErrorCollection; import org.apache.lens.api.result.LensAPIResult; import org.apache.lens.api.result.LensErrorTO; @@ -64,7 +65,9 @@ public class GenericExceptionMapper implements ExceptionMapper<Exception> { } else { status = Response.Status.INTERNAL_SERVER_ERROR; } - + if (extendedUriInfo.getMatchedResourceMethod() == null) { + return Response.status(Response.Status.METHOD_NOT_ALLOWED).build(); + } if (extendedUriInfo.getMatchedResourceMethod().getInvocable().getRawResponseType() == LensAPIResult.class) { if (le != null) { return Response.status(status).entity(le.getLensAPIResult()).build(); @@ -72,6 +75,8 @@ public class GenericExceptionMapper implements ExceptionMapper<Exception> { // if no LensException construct LensAPIResult LensAPIResult lensAPIResult = constructLensAPIResult(exception, status); return Response.status(lensAPIResult.getHttpStatusCode()).entity(lensAPIResult).build(); + } else if (extendedUriInfo.getMatchedResourceMethod().getInvocable().getRawResponseType() == APIResult.class) { + return Response.status(status).entity(APIResult.failure(exception)).build(); } else { return Response.status(status).entity(exception.getMessage()).build(); }
