Repository: lens Updated Branches: refs/heads/master b63e06c0c -> 22e202208
LENS-888 : Fix limit of 4000 on total string length of dim attribute names of derived cube Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/22e20220 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/22e20220 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/22e20220 Branch: refs/heads/master Commit: 22e202208cf6e7f216321f53a5cdc2cbdc113801 Parents: b63e06c Author: Rajat Khandelwal <pro...@apache.org> Authored: Thu Dec 10 11:10:08 2015 +0530 Committer: Amareshwari Sriramadasu <amareshw...@apache.org> Committed: Thu Dec 10 11:10:08 2015 +0530 ---------------------------------------------------------------------- .../apache/lens/cube/metadata/DerivedCube.java | 27 +++++++++++-------- .../lens/cube/metadata/MetastoreUtil.java | 16 ++++++++++- .../cube/metadata/TestCubeMetastoreClient.java | 28 +++++++++++++++++--- 3 files changed, 56 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/22e20220/lens-cube/src/main/java/org/apache/lens/cube/metadata/DerivedCube.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/DerivedCube.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/DerivedCube.java index 3c30f78..681aa7b 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/DerivedCube.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/DerivedCube.java @@ -137,16 +137,23 @@ public class DerivedCube extends AbstractCubeTable implements CubeInterface { @Override public void addProperties() { super.addProperties(); - getProperties().put(MetastoreUtil.getCubeMeasureListKey(getName()), StringUtils.join(measures, ",").toLowerCase()); - getProperties().put(MetastoreUtil.getCubeDimensionListKey(getName()), - StringUtils.join(dimensions, ",").toLowerCase()); + updateMeasureProperties(); + updateDimAttributeProperties(); getProperties().put(MetastoreUtil.getParentCubeNameKey(getName()), parent.getName().toLowerCase()); getProperties().put(MetastoreUtil.getParentCubeNameKey(getName()), parent.getName().toLowerCase()); } + public void updateDimAttributeProperties() { + MetastoreUtil.addNameStrings(getProperties(), MetastoreUtil.getCubeDimensionListKey(getName()), + MetastoreUtil.getNamedSetFromStringSet(dimensions)); + } + public void updateMeasureProperties() { + MetastoreUtil.addNameStrings(getProperties(), MetastoreUtil.getCubeMeasureListKey(getName()), + MetastoreUtil.getNamedSetFromStringSet(measures)); + } public static Set<String> getMeasures(String name, Map<String, String> props) { Set<String> measures = new HashSet<String>(); - String measureStr = props.get(MetastoreUtil.getCubeMeasureListKey(name)); + String measureStr = MetastoreUtil.getNamedStringValue(props, MetastoreUtil.getCubeMeasureListKey(name)); measures.addAll(Arrays.asList(StringUtils.split(measureStr, ','))); return measures; } @@ -164,7 +171,7 @@ public class DerivedCube extends AbstractCubeTable implements CubeInterface { public static Set<String> getDimensions(String name, Map<String, String> props) { Set<String> dimensions = new HashSet<String>(); - String dimStr = props.get(MetastoreUtil.getCubeDimensionListKey(name)); + String dimStr = MetastoreUtil.getNamedStringValue(props, MetastoreUtil.getCubeDimensionListKey(name)); dimensions.addAll(Arrays.asList(StringUtils.split(dimStr, ','))); return dimensions; } @@ -234,7 +241,7 @@ public class DerivedCube extends AbstractCubeTable implements CubeInterface { */ public void addMeasure(String measure) throws HiveException { measures.add(measure.toLowerCase()); - getProperties().put(MetastoreUtil.getCubeMeasureListKey(getName()), StringUtils.join(measures, ",").toLowerCase()); + updateMeasureProperties(); } /** @@ -245,8 +252,7 @@ public class DerivedCube extends AbstractCubeTable implements CubeInterface { */ public void addDimension(String dimension) throws HiveException { dimensions.add(dimension.toLowerCase()); - getProperties().put(MetastoreUtil.getCubeDimensionListKey(getName()), - StringUtils.join(dimensions, ",").toLowerCase()); + updateDimAttributeProperties(); } /** @@ -256,8 +262,7 @@ public class DerivedCube extends AbstractCubeTable implements CubeInterface { */ public void removeDimension(String dimName) { dimensions.remove(dimName.toLowerCase()); - getProperties().put(MetastoreUtil.getCubeDimensionListKey(getName()), - StringUtils.join(dimensions, ",").toLowerCase()); + updateDimAttributeProperties(); } /** @@ -267,7 +272,7 @@ public class DerivedCube extends AbstractCubeTable implements CubeInterface { */ public void removeMeasure(String msrName) { measures.remove(msrName.toLowerCase()); - getProperties().put(MetastoreUtil.getCubeMeasureListKey(getName()), StringUtils.join(measures, ",").toLowerCase()); + updateMeasureProperties(); } @Override http://git-wip-us.apache.org/repos/asf/lens/blob/22e20220/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java index 4b57d95..4ec049c 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java @@ -28,6 +28,8 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; +import com.google.common.collect.Sets; + public class MetastoreUtil { private MetastoreUtil() { @@ -394,11 +396,23 @@ public class MetastoreUtil { private static final int MAX_PARAM_LENGTH = 3999; + public static Set<Named> getNamedSetFromStringSet(Set<String> strings) { + Set<Named> nameds = Sets.newHashSet(); + for(final String s: strings) { + nameds.add(new Named() { + @Override + public String getName() { + return s; + } + }); + } + return nameds; + } public static <E extends Named> void addNameStrings(Map<String, String> props, String key, Collection<E> set) { addNameStrings(props, key, set, MAX_PARAM_LENGTH); } - static <E extends Named> void addNameStrings(Map<String, String> props, String key, + public static <E extends Named> void addNameStrings(Map<String, String> props, String key, Collection<E> set, int maxLength) { List<String> namedStrings = getNamedStrs(set, maxLength); props.put(key + ".size", String.valueOf(namedStrings.size())); http://git-wip-us.apache.org/repos/asf/lens/blob/22e20220/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index e415a5a..e5dbde7 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -44,7 +44,10 @@ import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; -import org.apache.hadoop.hive.ql.metadata.*; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ParseException; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde.serdeConstants; @@ -176,7 +179,11 @@ public class TestCubeMetastoreClient { cubeMeasures.add(new ColumnMeasure( new FieldSchema("msrcost2", "bigint", "measure with cost"), "Measure With cost2", null, "MAX", null, null, null, 100.0, 0.0, 999999999999999999999999999.0)); - + Set<CubeMeasure> dummyMeasure = Sets.newHashSet(); + for (int i = 0; i < 5000; i++) { + dummyMeasure.add(new ColumnMeasure(new FieldSchema("dummy_msr" + i, "bigint", "dummy measure " + i), + "", null, "SUM", null, null, null, 100.0, 0.0, 999999999999999999999999999.0)); + } cubeDimensions = new HashSet<>(); List<CubeDimAttribute> locationHierarchy = new ArrayList<>(); locationHierarchy.add(new ReferencedDimAtrribute(new FieldSchema("zipcode", "int", "zip"), "Zip refer", @@ -194,6 +201,11 @@ public class TestCubeMetastoreClient { cubeDimensions.add(new BaseDimAttribute(new FieldSchema("dim1", "string", "basedim"))); cubeDimensions.add(new ReferencedDimAtrribute(new FieldSchema("dim2", "id", "ref dim"), "Dim2 refer", new TableReference("testdim2", "id"))); + Set<CubeDimAttribute> dummyDimAttributes = Sets.newHashSet(); + for (int i = 0; i < 5000; i++) { + dummyDimAttributes.add(new BaseDimAttribute(new FieldSchema("dummy_dim" + i, "string", "dummy dim " + i), + "dummy_dim" + i, null, null, null, null, regions)); + } ExprSpec expr1 = new ExprSpec(); expr1.setExpr("avg(msr1 + msr2)"); @@ -276,9 +288,17 @@ public class TestCubeMetastoreClient { joinChains.add(cityChain); cubeDimensions.add(new ReferencedDimAtrribute(new FieldSchema("zipcityname", "string", "zip city name"), "Zip city name", "cityFromZip", "name", null, null, null)); + cubeMeasures.addAll(dummyMeasure); + cubeDimensions.addAll(dummyDimAttributes); cube = new Cube(cubeName, cubeMeasures, cubeDimensions, cubeExpressions, joinChains, emptyHashMap, 0.0); measures = Sets.newHashSet("msr1", "msr2", "msr3"); + for(CubeMeasure measure: dummyMeasure) { + measures.add(measure.getName()); + } dimensions = Sets.newHashSet("dim1", "dim2", "dim3"); + for(CubeDimAttribute dimAttribute: dummyDimAttributes) { + dimensions.add(dimAttribute.getName()); + } derivedCube = new DerivedCube(derivedCubeName, measures, dimensions, cube); CUBE_PROPERTIES.put(MetastoreUtil.getCubeTimedDimensionListKey(cubeNameWithProps), "dt,mydate"); @@ -1292,16 +1312,18 @@ public class TestCubeMetastoreClient { "complete name differs at element " + i); } } + private void assertTimeline(EndsAndHolesPartitionTimeline endsAndHolesPartitionTimeline, StoreAllPartitionTimeline storeAllPartitionTimeline, UpdatePeriod updatePeriod, int firstOffset, int latestOffset, int... holeOffsets) throws LensException { Date[] holeDates = new Date[holeOffsets.length]; - for(int i = 0; i < holeOffsets.length; i++) { + for (int i = 0; i < holeOffsets.length; i++) { holeDates[i] = getDateWithOffset(holeOffsets[i]); } assertTimeline(endsAndHolesPartitionTimeline, storeAllPartitionTimeline, updatePeriod, getDateWithOffset(firstOffset), getDateWithOffset(latestOffset), holeDates); } + private void assertTimeline(EndsAndHolesPartitionTimeline endsAndHolesPartitionTimeline, StoreAllPartitionTimeline storeAllPartitionTimeline, UpdatePeriod updatePeriod, Date first, Date latest, Date... holes) throws LensException {
