Repository: lens Updated Branches: refs/heads/master 2cf0b51fd -> 90cbd94f4
LENS-39 : Add tags to cube column Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/90cbd94f Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/90cbd94f Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/90cbd94f Branch: refs/heads/master Commit: 90cbd94f439487b7e6da9fc6e085581037099928 Parents: 2cf0b51 Author: sushilmohanty <[email protected]> Authored: Wed Apr 20 12:49:28 2016 +0530 Committer: sushilmohanty <[email protected]> Committed: Wed Apr 20 12:49:28 2016 +0530 ---------------------------------------------------------------------- lens-api/src/main/resources/cube-0.1.xsd | 10 ++++ .../apache/lens/cli/TestLensCubeCommands.java | 3 +- lens-cli/src/test/resources/sample-cube.xml | 26 +++++++++ .../lens/cube/metadata/BaseDimAttribute.java | 14 +++-- .../lens/cube/metadata/ColumnMeasure.java | 15 +++-- .../apache/lens/cube/metadata/CubeColumn.java | 36 +++++++++++- .../lens/cube/metadata/CubeDimAttribute.java | 12 +++- .../apache/lens/cube/metadata/CubeMeasure.java | 11 +++- .../apache/lens/cube/metadata/ExprColumn.java | 10 +++- .../lens/cube/metadata/MetastoreConstants.java | 1 + .../lens/cube/metadata/MetastoreUtil.java | 4 ++ .../cube/metadata/ReferencedDimAttribute.java | 8 ++- .../cube/metadata/TestCubeMetastoreClient.java | 60 ++++++++++++++++++++ .../src/main/resources/sample-cube.xml | 28 ++++++++- .../examples/ExampleSchemaToStringTest.java | 2 +- .../src/test/resources/yaml/sales-cube.yaml | 14 ++++- .../src/test/resources/yaml/sample-cube.yaml | 17 +++++- .../apache/lens/server/metastore/JAXBUtils.java | 19 ++++++- 18 files changed, 261 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/90cbd94f/lens-api/src/main/resources/cube-0.1.xsd ---------------------------------------------------------------------- diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd index f221a6d..4a6a5d7 100644 --- a/lens-api/src/main/resources/cube-0.1.xsd +++ b/lens-api/src/main/resources/cube-0.1.xsd @@ -40,6 +40,16 @@ some documentation </xs:documentation> </xs:annotation> + <xs:sequence> + <xs:element name="tags" type="x_properties" minOccurs="0" maxOccurs="1"> + <xs:annotation> + <xs:documentation> + Tags can be key value pairs associated with a field so that similar + fields can be categorized into certain group. + </xs:documentation> + </xs:annotation> + </xs:element> + </xs:sequence> <xs:attribute type="xs:string" name="name" use="required"/> <xs:attribute type="xs:string" name="display_string"> <xs:annotation> http://git-wip-us.apache.org/repos/asf/lens/blob/90cbd94f/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java index 5a353df..f53ed6b 100644 --- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java +++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java @@ -137,7 +137,8 @@ public class TestLensCubeCommands extends LensCliApplicationTest { private void testFields(LensCubeCommands command) { String fields = command.showQueryableFields("sample_cube", true); for (String field : Arrays - .asList("dim1", "dim2", "dim3", "dimdetail", "measure1", "measure2", "measure3", "measure4", "expr_msr5")) { + .asList("dim1", "dim2", "dim3", "dimdetail", "dim4", "measure1", "measure2", "measure3", "measure4", + "measure5", "measure6", "expr_msr5")) { assertTrue(fields.contains(field), fields + " do not contain " + field); } assertTrue(fields.contains("measure3 + measure4 + 0.01")); http://git-wip-us.apache.org/repos/asf/lens/blob/90cbd94f/lens-cli/src/test/resources/sample-cube.xml ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/resources/sample-cube.xml b/lens-cli/src/test/resources/sample-cube.xml index 9bcf177..2d6bc34 100644 --- a/lens-cli/src/test/resources/sample-cube.xml +++ b/lens-cli/src/test/resources/sample-cube.xml @@ -30,6 +30,18 @@ <measure name="measure2" _type="INT" default_aggr="SUM" /> <measure name="measure3" _type="FLOAT" default_aggr="MAX" start_time='2013-12-12T00:00:00' /> <measure name="measure4" _type="DOUBLE" default_aggr="MIN" /> + <measure name="measure5" _type="BIGINT"> + <tags> + <property name="category" value="primary"/> + <property name="is_ui_visible" value="true"/> + </tags> + </measure> + <measure name="measure6" _type="BIGINT"> + <tags> + <property name="category" value="secondary"/> + <property name="is_ui_visible" value="false"/> + </tags> + </measure> </measures> <dim_attributes> <dim_attribute name="dim1" _type="INT" /> @@ -40,12 +52,26 @@ <chain_ref_column chain_name="testdimchain" ref_col="detail" /> <chain_ref_column chain_name="testdetailchain" ref_col="name" /> </dim_attribute> + <dim_attribute name="dim4" _type="BIGINT"> + <tags> + <property name="category" value="primary"/> + <property name="is_ui_visible" value="true"/> + </tags> + </dim_attribute> </dim_attributes> <expressions> <expression name="expr_msr5" _type="DOUBLE"> <expr_spec expr = "measure3 + measure4" end_time='2013-12-12T00:00:00'/> <expr_spec expr = "measure3 + measure4 + 0.01" start_time='2013-12-12T00:00:00'/> </expression> + <expression name="expr_msr6" _type="DOUBLE"> + <tags> + <property name="category" value="primary"/> + <property name="is_ui_visible" value="true"/> + </tags> + <expr_spec expr = "measure3 + measure4" start_time='2013-12-12T00:00:00' /> + <expr_spec expr = "measure3 + measure4 + 0.01" end_time='2013-12-12T00:00:00'/> + </expression> </expressions> <join_chains> <join_chain name="testdimchain"> http://git-wip-us.apache.org/repos/asf/lens/blob/90cbd94f/lens-cube/src/main/java/org/apache/lens/cube/metadata/BaseDimAttribute.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/BaseDimAttribute.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/BaseDimAttribute.java index bd4ae57..28423ff 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/BaseDimAttribute.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/BaseDimAttribute.java @@ -21,10 +21,7 @@ package org.apache.lens.cube.metadata; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; -import java.util.Arrays; -import java.util.Date; -import java.util.List; -import java.util.Map; +import java.util.*; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -55,8 +52,13 @@ public class BaseDimAttribute extends CubeDimAttribute { } public BaseDimAttribute(FieldSchema column, String displayString, Date startTime, Date endTime, Double cost, - Long numOfDistinctValues, List<String> values) { - super(column.getName(), column.getComment(), displayString, startTime, endTime, cost); + Long numOfDistinctValues, List<String> values) { + this(column, displayString, startTime, endTime, cost, numOfDistinctValues, values, new HashMap<String, String>()); + } + + public BaseDimAttribute(FieldSchema column, String displayString, Date startTime, Date endTime, Double cost, + Long numOfDistinctValues, List<String> values, Map<String, String> tags) { + super(column.getName(), column.getComment(), displayString, startTime, endTime, cost, tags); this.type = column.getType(); checkNotNull(type); Optional<Long> optionalNumOfDistnctValues = Optional.fromNullable(numOfDistinctValues); http://git-wip-us.apache.org/repos/asf/lens/blob/90cbd94f/lens-cube/src/main/java/org/apache/lens/cube/metadata/ColumnMeasure.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ColumnMeasure.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ColumnMeasure.java index 5fda721..87e6ee1 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ColumnMeasure.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ColumnMeasure.java @@ -19,6 +19,7 @@ package org.apache.lens.cube.metadata; import java.util.Date; +import java.util.HashMap; import java.util.Map; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -26,11 +27,11 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; public final class ColumnMeasure extends CubeMeasure { public ColumnMeasure(FieldSchema column, String displayString, String formatString, String aggregate, String unit) { - this(column, displayString, formatString, aggregate, unit, null, null, null, null, null); + this(column, displayString, formatString, aggregate, unit, null, null, null, null, null, null); } public ColumnMeasure(FieldSchema column) { - this(column, null, null, null, null, null, null, null, null, null); + this(column, null, null, null, null, null, null, null, null, null, null); } public ColumnMeasure(FieldSchema column, String displayString, String formatString, String aggregate, String unit, @@ -39,8 +40,14 @@ public final class ColumnMeasure extends CubeMeasure { } public ColumnMeasure(FieldSchema column, String displayString, String formatString, String aggregate, String unit, - Date startTime, Date endTime, Double cost, Double min, Double max) { - super(column, displayString, formatString, aggregate, unit, startTime, endTime, cost, min, max); + Date startTime, Date endTime, Double cost, Double min, Double max) { + this(column, displayString, formatString, aggregate, unit, startTime, + endTime, cost, min, max, new HashMap<String, String>()); + } + + public ColumnMeasure(FieldSchema column, String displayString, String formatString, String aggregate, String unit, + Date startTime, Date endTime, Double cost, Double min, Double max, Map<String, String> tags) { + super(column, displayString, formatString, aggregate, unit, startTime, endTime, cost, min, max, tags); } public ColumnMeasure(String name, Map<String, String> props) { http://git-wip-us.apache.org/repos/asf/lens/blob/90cbd94f/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeColumn.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeColumn.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeColumn.java index 77024c0..62ce930 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeColumn.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeColumn.java @@ -21,11 +21,13 @@ package org.apache.lens.cube.metadata; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Date; +import java.util.HashMap; import java.util.Map; import java.util.TimeZone; import com.google.common.base.Optional; +import lombok.Getter; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -38,6 +40,8 @@ public abstract class CubeColumn implements Named { private final Double cost; private final String description; private final String displayString; + @Getter + private final Map<String, String> tags; static final ThreadLocal<DateFormat> COLUMN_TIME_FORMAT = new ThreadLocal<DateFormat>() { @@ -49,7 +53,13 @@ public abstract class CubeColumn implements Named { } }; - public CubeColumn(String name, String description, String displayString, Date startTime, Date endTime, Double cost) { + public CubeColumn(String name , String description, String displayString, + Date startTime, Date endTime, Double cost) { + this(name, description, displayString, startTime, endTime, cost, new HashMap<String, String>()); + } + + public CubeColumn(String name, String description, String displayString, + Date startTime, Date endTime, Double cost, Map<String, String> tags) { assert (name != null); this.name = name.toLowerCase(); this.startTime = startTime; @@ -57,6 +67,7 @@ public abstract class CubeColumn implements Named { this.cost = cost; this.description = description; this.displayString = displayString; + this.tags = tags; } private Date getDate(String propKey, Map<String, String> props) { @@ -89,6 +100,25 @@ public abstract class CubeColumn implements Named { return null; } + private static synchronized void addTagProperties(String name, Map<String, String> props, Map<String, String> tags) { + String colName = MetastoreUtil.getCubeColTagKey(name); + if (tags != null) { + for (Map.Entry<String, String> entry : tags.entrySet()) { + props.put(colName.concat(entry.getKey()), entry.getValue()); + } + } + } + + private static Map<String, String> getColumnTags(String propKey, Map<String, String> props) { + Map<String, String> tagProp = new HashMap<>(); + for (String key : props.keySet()) { + if (key.startsWith(propKey)) { + tagProp.put(key.replace(propKey, ""), props.get(key).replace(propKey, "")); + } + } + return tagProp; + } + public CubeColumn(String name, Map<String, String> props) { this.name = name; this.startTime = getDate(MetastoreUtil.getCubeColStartTimePropertyKey(name), props); @@ -96,6 +126,7 @@ public abstract class CubeColumn implements Named { this.cost = getDouble(MetastoreUtil.getCubeColCostPropertyKey(name), props); this.description = props.get(MetastoreUtil.getCubeColDescriptionKey(name)); this.displayString = props.get(MetastoreUtil.getCubeColDisplayKey(name)); + this.tags = getColumnTags(MetastoreUtil.getCubeColTagKey(name), props); } public String getName() { @@ -290,5 +321,8 @@ public abstract class CubeColumn implements Named { if (cost != null) { props.put(MetastoreUtil.getCubeColCostPropertyKey(getName()), cost.toString()); } + if (tags != null) { + addTagProperties(name, props, tags); + } } } http://git-wip-us.apache.org/repos/asf/lens/blob/90cbd94f/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeDimAttribute.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeDimAttribute.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeDimAttribute.java index 26c24de..f5a07f1 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeDimAttribute.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeDimAttribute.java @@ -19,17 +19,23 @@ package org.apache.lens.cube.metadata; import java.util.Date; +import java.util.HashMap; import java.util.Map; public abstract class CubeDimAttribute extends CubeColumn { public CubeDimAttribute(String name, String description) { - this(name, description, null, null, null, null); + this(name, description, null, null, null, null, null); } public CubeDimAttribute(String name, String description, String displayString, Date startTime, Date endTime, - Double cost) { - super(name, description, displayString, startTime, endTime, cost); + Double cost) { + this(name, description, displayString, startTime, endTime, cost, new HashMap<String, String>()); + } + + public CubeDimAttribute(String name, String description, String displayString, Date startTime, Date endTime, + Double cost, Map<String, String> tags) { + super(name, description, displayString, startTime, endTime, cost, tags); } public CubeDimAttribute(String name, Map<String, String> props) { http://git-wip-us.apache.org/repos/asf/lens/blob/90cbd94f/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMeasure.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMeasure.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMeasure.java index d5fc0e7..b59af5b 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMeasure.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMeasure.java @@ -20,6 +20,7 @@ package org.apache.lens.cube.metadata; import java.util.Date; +import java.util.HashMap; import java.util.Map; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -33,8 +34,14 @@ public abstract class CubeMeasure extends CubeColumn { private final Double max; protected CubeMeasure(FieldSchema column, String displayString, String formatString, String aggregate, String unit, - Date startTime, Date endTime, Double cost, Double min, Double max) { - super(column.getName(), column.getComment(), displayString, startTime, endTime, cost); + Date startTime, Date endTime, Double cost, Double min, Double max) { + this(column, displayString, formatString, aggregate, unit, + startTime, endTime, cost, min, max, new HashMap<String, String>()); + } + + protected CubeMeasure(FieldSchema column, String displayString, String formatString, String aggregate, String unit, + Date startTime, Date endTime, Double cost, Double min, Double max, Map<String, String> tags) { + super(column.getName(), column.getComment(), displayString, startTime, endTime, cost, tags); this.column = column; assert (column != null); assert (column.getName() != null); http://git-wip-us.apache.org/repos/asf/lens/blob/90cbd94f/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java index da87e31..1c9d6d2 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java @@ -47,8 +47,14 @@ public class ExprColumn extends CubeColumn { this(column, displayString, new ExprSpec(expression, null, null)); } - public ExprColumn(FieldSchema column, String displayString, ExprSpec... expressions) throws LensException { - super(column.getName(), column.getComment(), displayString, null, null, 0.0); + public ExprColumn(FieldSchema column, String displayString, + ExprSpec... expressions) throws LensException { + this(column, displayString, new HashMap<String, String>(), expressions); + } + + public ExprColumn(FieldSchema column, String displayString, Map<String, String> tags, + ExprSpec... expressions) throws LensException { + super(column.getName(), column.getComment(), displayString, null, null, 0.0, tags); if (expressions == null || expressions.length == 0) { throw new IllegalArgumentException("No expressions specified for column " + column.getName()); http://git-wip-us.apache.org/repos/asf/lens/blob/90cbd94f/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java index cd4862c..1aae267 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java @@ -83,6 +83,7 @@ public final class MetastoreConstants { public static final String DESC_SFX = ".description"; public static final String DISPLAY_SFX = ".displaystring"; public static final String NUM_DISTINCT_VALUES = ".num.distinct.values"; + public static final String TAGS_PFX = ".tags."; // measure constants public static final String MEASURE_KEY_PFX = "cube.measure."; http://git-wip-us.apache.org/repos/asf/lens/blob/90cbd94f/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java index 399d5a7..006c4cb 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java @@ -174,6 +174,10 @@ public class MetastoreUtil { return getColumnKeyPrefix(colName) + DISPLAY_SFX; } + public static String getCubeColTagKey(String colName) { + return getColumnKeyPrefix(colName) + TAGS_PFX; + } + public static String getExprColumnKey(String colName) { return getColumnKeyPrefix(colName) + EXPR_SFX; } http://git-wip-us.apache.org/repos/asf/lens/blob/90cbd94f/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAttribute.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAttribute.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAttribute.java index 9a1c44b..4c98d86 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAttribute.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAttribute.java @@ -58,12 +58,14 @@ public class ReferencedDimAttribute extends BaseDimAttribute { public ReferencedDimAttribute(FieldSchema column, String displayString, List<ChainRefCol> chainRefCols, Date startTime, Date endTime, Double cost, Long numOfDistinctValues) throws LensException { - this(column, displayString, chainRefCols, startTime, endTime, cost, numOfDistinctValues, null); + this(column, displayString, chainRefCols, startTime, endTime, cost, numOfDistinctValues, + null, new HashMap<String, String>()); } public ReferencedDimAttribute(FieldSchema column, String displayString, List<ChainRefCol> chainRefCols, - Date startTime, Date endTime, Double cost, Long numOfDistinctValues, List<String> values) throws LensException { - super(column, displayString, startTime, endTime, cost, numOfDistinctValues, values); + Date startTime, Date endTime, Double cost, Long numOfDistinctValues, List<String> values, Map<String, String> tags) + throws LensException { + super(column, displayString, startTime, endTime, cost, numOfDistinctValues, values, tags); if (chainRefCols.isEmpty()) { throw new LensException(LensCubeErrorCode.ERROR_IN_ENTITY_DEFINITION.getLensErrorInfo(), " Ref column: " + getName() + " does not have any chain_ref_column defined"); http://git-wip-us.apache.org/repos/asf/lens/blob/90cbd94f/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index 18575f6..0773005 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -784,6 +784,66 @@ public class TestCubeMetastoreClient { } @Test(priority = 1) + public void testColumnTags() throws Exception { + String cubename = "cubetags"; + Map<String, String> tag1 = new HashMap<>(); + tag1.put("category", "test"); + Map<String, String> tag2 = new HashMap<>(); + tag2.put("is_ui_visible", "true"); + Set<CubeMeasure> cubeMeasures = new HashSet<>(); + cubeMeasures.add(new ColumnMeasure( + new FieldSchema("msr1", "int", "measure1 with tag"), null, null, null, null, null, null, null, 0.0, + 9999.0, tag1)); + cubeMeasures.add(new ColumnMeasure( + new FieldSchema("msr2", "int", "measure2 with tag"), + "measure2 with tag", null, null, null, NOW, null, null, 0.0, 999999.0, tag2)); + + Set<CubeDimAttribute> cubeDimensions = new HashSet<>(); + cubeDimensions.add(new BaseDimAttribute(new FieldSchema("dim1", "id", "ref dim"), "dim with tag", + null, null, null, null, null, tag1)); + + ExprSpec expr1 = new ExprSpec(); + expr1.setExpr("avg(msr1 + msr2)"); + ExprSpec expr2 = new ExprSpec(); + expr2.setExpr("avg(msr2 + msr1)"); + + Set<ExprColumn> cubeExpressions = new HashSet<>(); + cubeExpressions.add(new ExprColumn(new FieldSchema("expr_measure", "double", "expression measure"), + "expr with tag", tag2, expr1, expr2)); + + client.createCube(cubename, + cubeMeasures, cubeDimensions, cubeExpressions, null, null); + Table cubeTbl = client.getHiveTable(cubename); + assertTrue(client.isCube(cubeTbl)); + Cube cube2 = new Cube(cubeTbl); + + // measures with tag + assertNotNull(cube2.getMeasureByName("msr1")); + assertTrue(cube2.getMeasureByName("msr1").getTags().keySet().contains("category")); + assertTrue(cube2.getMeasureByName("msr1").getTags().values().contains("test")); + + assertNotNull(cube2.getMeasureByName("msr2")); + assertTrue(cube2.getMeasureByName("msr2").getTags().keySet().contains("is_ui_visible")); + assertTrue(cube2.getMeasureByName("msr2").getTags().values().contains("true")); + + // dim with tag + assertNotNull(cube2.getDimAttributeByName("dim1")); + assertTrue(cube2.getDimAttributeByName("dim1").getTags().keySet().contains("category")); + assertTrue(cube2.getDimAttributeByName("dim1").getTags().values().contains("test")); + + // expr with tag + assertNotNull(cube2.getExpressionByName("expr_measure")); + assertTrue(cube2.getExpressionByName("expr_measure").getTags().keySet().contains("is_ui_visible")); + assertTrue(cube2.getExpressionByName("expr_measure").getTags().values().contains("true")); + + // check properties + cube2.getProperties().get("cube.col.msr2.tags.is_ui_visible").equals("cube.col.msr2.tags.true"); + cube2.getProperties().get("cube.col.dim1.tags.category").equals("cube.col.dim1.tags.test"); + + client.dropCube(cubename); + } + + @Test(priority = 1) public void testAlterCube() throws Exception { String cubeName = "alter_test_cube"; client.createCube(cubeName, cubeMeasures, cubeDimensions); http://git-wip-us.apache.org/repos/asf/lens/blob/90cbd94f/lens-examples/src/main/resources/sample-cube.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/sample-cube.xml b/lens-examples/src/main/resources/sample-cube.xml index 4046e82..6b17da5 100644 --- a/lens-examples/src/main/resources/sample-cube.xml +++ b/lens-examples/src/main/resources/sample-cube.xml @@ -26,21 +26,47 @@ </properties> <measures> <measure name="measure1" _type="BIGINT"/> - <measure name="measure2" _type="INT" default_aggr="SUM"/> + <measure name="measure2" _type="INT" default_aggr="SUM" /> <measure name="measure3" _type="FLOAT" default_aggr="MAX" start_time='2013-12-12T00:00:00'/> <measure name="measure4" _type="DOUBLE" default_aggr="MIN"/> + <measure name="measure5" _type="BIGINT"> + <tags> + <property name="category" value="primary"/> + <property name="is_ui_visible" value="true"/> + </tags> + </measure> + <measure name="measure6" _type="BIGINT"> + <tags> + <property name="category" value="secondary"/> + <property name="is_ui_visible" value="false"/> + </tags> + </measure> </measures> <dim_attributes> <dim_attribute name="dim1" _type="INT"/> <dim_attribute name="dim2" _type="INT" start_time='2013-12-01T00:00:00'/> <dim_attribute name="dim3" _type="INT"> </dim_attribute> + <dim_attribute name="dim4" _type="BIGINT"> + <tags> + <property name="category" value="primary"/> + <property name="is_ui_visible" value="true"/> + </tags> + </dim_attribute> </dim_attributes> <expressions> <expression name="expr_msr5" _type="DOUBLE"> <expr_spec expr = "measure3 + measure4" start_time='2013-12-12T00:00:00' /> <expr_spec expr = "measure3 + measure4 + 0.01" end_time='2013-12-12T00:00:00'/> </expression> + <expression name="expr_msr5" _type="DOUBLE"> + <tags> + <property name="category" value="primary"/> + <property name="is_ui_visible" value="true"/> + </tags> + <expr_spec expr = "measure3 + measure4" start_time='2013-12-12T00:00:00' /> + <expr_spec expr = "measure3 + measure4 + 0.01" end_time='2013-12-12T00:00:00'/> + </expression> </expressions> <join_chains> <join_chain name="sample_dim_chain"> http://git-wip-us.apache.org/repos/asf/lens/blob/90cbd94f/lens-examples/src/test/java/org/apache/lens/examples/ExampleSchemaToStringTest.java ---------------------------------------------------------------------- diff --git a/lens-examples/src/test/java/org/apache/lens/examples/ExampleSchemaToStringTest.java b/lens-examples/src/test/java/org/apache/lens/examples/ExampleSchemaToStringTest.java index b884368..4a3c07b 100644 --- a/lens-examples/src/test/java/org/apache/lens/examples/ExampleSchemaToStringTest.java +++ b/lens-examples/src/test/java/org/apache/lens/examples/ExampleSchemaToStringTest.java @@ -37,7 +37,7 @@ public class ExampleSchemaToStringTest extends YAMLToStringStrategyTest { public List<ToStringTestData> provideData() throws URISyntaxException, JAXBException, IOException { List<ToStringTestData> ret = Lists.newArrayList(); for (String yamlName : new File(getClass().getResource("/yaml").toURI()).list()) { - ret.add(new ToStringTestData(yamlName, unmarshallFromFile("/" + yamlName.replaceAll("yaml$", "xml")), + ret.add(new ToStringTestData(yamlName, unmarshallFromFile(yamlName.replaceAll("yaml$", "xml")), readYAML("/yaml/" + yamlName))); } return ret; http://git-wip-us.apache.org/repos/asf/lens/blob/90cbd94f/lens-examples/src/test/resources/yaml/sales-cube.yaml ---------------------------------------------------------------------- diff --git a/lens-examples/src/test/resources/yaml/sales-cube.yaml b/lens-examples/src/test/resources/yaml/sales-cube.yaml index 935e3f6..2f5d7a7 100644 --- a/lens-examples/src/test/resources/yaml/sales-cube.yaml +++ b/lens-examples/src/test/resources/yaml/sales-cube.yaml @@ -53,6 +53,12 @@ dimAttributes: chainName: customer_city refCol: name type: string + Customer Interest(customer_interest) [Customer's interest]: + chainRefColumn: + - + chainName: customer_interests_chain + refCol: name + type: string production_location: hierarchy: Production City(production_city_name) [City name in which the product was produced]: @@ -89,6 +95,12 @@ joinChains: paths: - sales.customer_id=customer.id + customer_interests_chain: + paths: + - + sales.customer_id=customer.id + customer.id=customer_interests.customer_id(many) + customer_interests.interest_id=interests.id product_details: paths: - @@ -107,4 +119,4 @@ joinChains: production_city: paths: - - sales.production_city_id=city.id + sales.production_city_id=city.id \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lens/blob/90cbd94f/lens-examples/src/test/resources/yaml/sample-cube.yaml ---------------------------------------------------------------------- diff --git a/lens-examples/src/test/resources/yaml/sample-cube.yaml b/lens-examples/src/test/resources/yaml/sample-cube.yaml index 3f2ffc0..5ccc1af 100644 --- a/lens-examples/src/test/resources/yaml/sample-cube.yaml +++ b/lens-examples/src/test/resources/yaml/sample-cube.yaml @@ -24,12 +24,15 @@ measures: defaultAggr: MAX startTime: 2013-12-12T00:00:00 measure4: type: DOUBLE, defaultAggr: MIN + measure5: tags:, category: primary, is_ui_visible: true, type: BIGINT + measure6: tags:, category: secondary, is_ui_visible: false, type: BIGINT dimAttributes: dim1: type: INT dim2: type: INT startTime: 2013-12-01T00:00:00 dim3: type: INT + dim4: tags:, category: primary, is_ui_visible: true, type: BIGINT expressions: expr_msr5: exprSpec: @@ -40,8 +43,20 @@ expressions: expr: measure3 + measure4 + 0.01 endTime: 2013-12-12T00:00:00 type: DOUBLE + expr_msr5: + tags: + category: primary + is_ui_visible: true + exprSpec: + - + expr: measure3 + measure4 + startTime: 2013-12-12T00:00:00 + - + expr: measure3 + measure4 + 0.01 + endTime: 2013-12-12T00:00:00 + type: DOUBLE joinChains: sample_dim_chain: paths: - - sample_cube.dim3=sample_dim.id + sample_cube.dim3=sample_dim.id \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lens/blob/90cbd94f/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java index 6571dae..79a628f 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java @@ -182,7 +182,8 @@ public final class JAXBUtils { endDate, null, xd.getNumDistinctValues(), - xd.getValues() + xd.getValues(), + mapFromXProperties(xd.getTags()) ); } else { hiveDim = new BaseDimAttribute(new FieldSchema(xd.getName(), xd.getType().toLowerCase(), @@ -192,7 +193,8 @@ public final class JAXBUtils { endDate, null, xd.getNumDistinctValues(), - xd.getValues() + xd.getValues(), + mapFromXProperties(xd.getTags()) ); } return hiveDim; @@ -264,9 +266,16 @@ public final class JAXBUtils { xm.setEndTime(getXMLGregorianCalendar(cm.getEndTime())); xm.setMin(cm.getMin()); xm.setMax(cm.getMax()); + xm.setTags(getXProperties(xPropertiesFromMap(cm.getTags()))); return xm; } + public static XProperties getXProperties(List<XProperty> prop) { + XProperties properties = XCF.createXProperties(); + properties.getProperty().addAll(prop); + return properties; + } + /** * Create XExprColumn from hive ExprColum */ @@ -281,6 +290,7 @@ public final class JAXBUtils { xe.setDescription(ec.getDescription()); xe.setDisplayString(ec.getDisplayString()); xe.getExprSpec().addAll(xExprSpecFromExprColumn(ec.getExpressionSpecs())); + xe.setTags(getXProperties(xPropertiesFromMap(ec.getTags()))); return xe; } @@ -314,6 +324,7 @@ public final class JAXBUtils { xd.setDisplayString(cd.getDisplayString()); xd.setStartTime(getXMLGregorianCalendar(cd.getStartTime())); xd.setEndTime(getXMLGregorianCalendar(cd.getEndTime())); + xd.setTags(getXProperties(xPropertiesFromMap(cd.getTags()))); if (cd instanceof ReferencedDimAttribute) { ReferencedDimAttribute rd = (ReferencedDimAttribute) cd; if (!rd.getChainRefColumns().isEmpty()) { @@ -423,7 +434,8 @@ public final class JAXBUtils { endDate, null, xm.getMin(), - xm.getMax() + xm.getMax(), + mapFromXProperties(xm.getTags()) ); return cm; } @@ -454,6 +466,7 @@ public final class JAXBUtils { ExprColumn ec = new ExprColumn(new FieldSchema(xe.getName(), xe.getType().toLowerCase(), xe.getDescription()), xe.getDisplayString(), + mapFromXProperties(xe.getTags()), exprSpecFromXExprColumn(xe.getExprSpec())); return ec; }
