Repository: incubator-lens Updated Branches: refs/heads/master 10917c25e -> 365f5f72b
LENS-176 : Lens server should support API to retrieve and update expression columns with multiple expressions (amareshwari) Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/365f5f72 Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/365f5f72 Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/365f5f72 Branch: refs/heads/master Commit: 365f5f72bf3b06c964241a4b2a8c16cd2b4af50e Parents: 10917c2 Author: Amareshwari Sriramadasu <[email protected]> Authored: Thu May 7 11:18:49 2015 +0530 Committer: Amareshwari Sriramadasu <[email protected]> Committed: Thu May 7 11:18:49 2015 +0530 ---------------------------------------------------------------------- lens-api/src/main/resources/cube-0.1.xsd | 34 +++ lens-cli/src/test/resources/sample-cube.xml | 9 +- .../apache/lens/cube/metadata/CubeColumn.java | 15 +- .../apache/lens/cube/metadata/ExprColumn.java | 229 ++++++++++++++++--- .../lens/cube/parse/ExpressionResolver.java | 13 +- .../cube/metadata/TestCubeMetastoreClient.java | 36 ++- .../lens/cube/metadata/TestExprColumn.java | 159 ++++++++++++- lens-examples/src/main/resources/customer.xml | 2 +- lens-examples/src/main/resources/product.xml | 6 + lens-examples/src/main/resources/sales-cube.xml | 15 +- .../src/main/resources/sample-cube.xml | 9 +- .../main/resources/sample-db-only-dimension.xml | 4 +- .../src/main/resources/sample-dimension.xml | 4 +- .../src/main/resources/sample-dimension2.xml | 2 +- .../apache/lens/server/metastore/JAXBUtils.java | 43 +++- .../lens/server/ui/MetastoreUIResource.java | 2 +- .../server/metastore/TestMetastoreService.java | 54 ++++- .../lens/server/query/TestQueryService.java | 3 +- 18 files changed, 560 insertions(+), 79 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/365f5f72/lens-api/src/main/resources/cube-0.1.xsd ---------------------------------------------------------------------- diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd index 24d0d64..10b1dfa 100644 --- a/lens-api/src/main/resources/cube-0.1.xsd +++ b/lens-api/src/main/resources/cube-0.1.xsd @@ -248,6 +248,14 @@ An expression column </xs:documentation> </xs:annotation> + <xs:sequence> + <xs:annotation> + <xs:documentation> + All the expressions associated with expression column. + </xs:documentation> + </xs:annotation> + <xs:element type="x_expr_spec" name="expr_spec" maxOccurs="unbounded" minOccurs="1"/> + </xs:sequence> <xs:attribute type="xs:string" name="name" use="required" /> <xs:attribute type="xs:string" name="type" use="required"> <xs:annotation> @@ -269,6 +277,18 @@ </xs:documentation> </xs:annotation> </xs:attribute> + </xs:complexType> + + <xs:complexType name="x_expr_spec"> + <xs:annotation> + <xs:documentation> + Specification of single expression. Specification consists of required expression field, and optional start and + end times associated with expression. Expression is valid between start time and end time, if both are + specified. If only start_time is specified, is valid from start_time to till date. If only end_time is + specified, is valid upto date specified, from any time. If no start time, end time are specified, the + expression is always valid. + </xs:documentation> + </xs:annotation> <xs:attribute type="xs:string" name="expr" use="required"> <xs:annotation> <xs:documentation> @@ -276,6 +296,20 @@ </xs:documentation> </xs:annotation> </xs:attribute> + <xs:attribute type="xs:dateTime" name="start_time"> + <xs:annotation> + <xs:documentation> + The start time from when the expression is valid. + </xs:documentation> + </xs:annotation> + </xs:attribute> + <xs:attribute type="xs:dateTime" name="end_time"> + <xs:annotation> + <xs:documentation> + The end time till when the expression is valid. + </xs:documentation> + </xs:annotation> + </xs:attribute> </xs:complexType> <xs:complexType name="x_expressions"> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/365f5f72/lens-cli/src/test/resources/sample-cube.xml ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/resources/sample-cube.xml b/lens-cli/src/test/resources/sample-cube.xml index 4908f85..d54bcfb 100644 --- a/lens-cli/src/test/resources/sample-cube.xml +++ b/lens-cli/src/test/resources/sample-cube.xml @@ -24,15 +24,18 @@ <measures> <measure name="measure1" type="BIGINT" /> <measure name="measure2" type="INT" default_aggr="SUM" /> - <measure name="measure3" type="FLOAT" default_aggr="MAX" start_time='2013-12-12-00:00' /> + <measure name="measure3" type="FLOAT" default_aggr="MAX" start_time='2013-12-12T00:00:00' /> <measure name="measure4" type="DOUBLE" default_aggr="MIN" /> </measures> <expressions> - <expression name="expr_msr5" type="DOUBLE" expr="measure3 + measure4" /> + <expression name="expr_msr5" type="DOUBLE"> + <expr_spec expr = "measure3 + measure4" end_time='2013-12-12T00:00:00'/> + <expr_spec expr = "measure3 + measure4 + 0.01" start_time='2013-12-12T00:00:00'/> + </expression> </expressions> <dim_attributes> <dim_attribute name="dim1" type="INT" /> - <dim_attribute name="dim2" type="INT" start_time='2013-12-01-00:00' /> + <dim_attribute name="dim2" type="INT" start_time='2013-12-01T00:00:00' /> <dim_attribute name="dim3" type="INT"> <ref_spec> <table_references> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/365f5f72/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeColumn.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeColumn.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeColumn.java index cccb888..a2a00d2 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeColumn.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeColumn.java @@ -27,8 +27,11 @@ import java.util.TimeZone; import org.apache.lens.cube.parse.TimeRange; import com.google.common.base.Optional; + import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +@Slf4j public abstract class CubeColumn implements Named { private final String name; @@ -59,12 +62,17 @@ public abstract class CubeColumn implements Named { } private Date getDate(String propKey, Map<String, String> props) { - String timeKey = props.get(propKey); - if (timeKey != null) { + String timeStr = props.get(propKey); + return getDate(timeStr); + } + + protected Date getDate(String timeStr) { + if (timeStr != null) { try { - return COLUMN_TIME_FORMAT.get().parse(timeKey); + return COLUMN_TIME_FORMAT.get().parse(timeStr); } catch (Exception e) { // ignore and return null + log.warn("Column time passed:{} is not parsable, its ignored", timeStr, e); } } return null; @@ -77,6 +85,7 @@ public abstract class CubeColumn implements Named { return Double.parseDouble(doubleStr); } catch (Exception e) { // ignore and return null + log.warn("Property {} value {} is not parsable, its ignored", propKey, doubleStr, e); } } return null; http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/365f5f72/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java index df4cf25..a7f711f 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java @@ -30,26 +30,47 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.ParseException; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; + public class ExprColumn extends CubeColumn { public static final char EXPRESSION_DELIMITER = '|'; + public static final char EXPRESSION_SPEC_DELIMITER = ':'; private static final String EXPRESSION_ENCODED = "true"; - private final Set<String> expressionSet = new LinkedHashSet<String>(); - private List<ASTNode> astNodeList; + private final Set<ExprSpec> expressionSet = new LinkedHashSet<ExprSpec>(); + private List<ASTNode> astNodeList = new ArrayList<ASTNode>(); private final String type; private boolean hasHashCode = false; private int hashCode; - public ExprColumn(FieldSchema column, String displayString, String... expressions) throws ParseException { + // for backward compatibility + public ExprColumn(FieldSchema column, String displayString, String expression) { + this(column, displayString, new ExprSpec(expression, null, null)); + } + + public ExprColumn(FieldSchema column, String displayString, ExprSpec... expressions) { super(column.getName(), column.getComment(), displayString, null, null, 0.0); if (expressions == null || expressions.length == 0) { - throw new IllegalArgumentException("No expression specified for column " + column.getName()); + throw new IllegalArgumentException("No expressions specified for column " + column.getName()); } - for (String e : expressions) { + for (int i = 0; i < expressions.length; i++) { + ExprSpec e = expressions[i]; + if (StringUtils.isBlank(e.getExpr())) { + throw new IllegalArgumentException( + "No expression string specified for column " + column.getName() + " at index:" + i); + } + if (e.getStartTime() != null && e.getEndTime() != null) { + if (e.getStartTime().after(e.getEndTime())) { + throw new IllegalArgumentException("Start time is after end time for column " + column.getName() + + " for expression at index:" + i + " for " + e.getExpr()); + } + } expressionSet.add(e); } - this.type = column.getType(); assert (getAst() != null); } @@ -69,9 +90,26 @@ public class ExprColumn extends CubeColumn { EXPRESSION_ENCODED.equals(props.get(MetastoreUtil.getExprEncodingPropertyKey(getName()))); for (String e : expressions) { + String[] exprSpecStrs = StringUtils.splitPreserveAllTokens(e, EXPRESSION_SPEC_DELIMITER); try { - String decodedExpr = isExpressionBase64Encoded ? new String(Base64.decodeBase64(e), "UTF-8") : e; - expressionSet.add(decodedExpr); + String decodedExpr = + isExpressionBase64Encoded ? new String(Base64.decodeBase64(exprSpecStrs[0]), "UTF-8") : exprSpecStrs[0]; + ExprSpec exprSpec = new ExprSpec(); + exprSpec.setExpr(decodedExpr); + if (exprSpecStrs.length > 1) { + // start time and end time serialized + if (StringUtils.isNotBlank(exprSpecStrs[1])) { + // start time available + exprSpec.setStartTime(getDate(exprSpecStrs[1])); + } + if (exprSpecStrs.length > 2) { + if (StringUtils.isNotBlank(exprSpecStrs[2])) { + // end time available + exprSpec.setEndTime(getDate(exprSpecStrs[2])); + } + } + } + expressionSet.add(exprSpec); } catch (UnsupportedEncodingException e1) { throw new IllegalArgumentException("Error decoding expression for expression column " + name + " encoded value=" + e); @@ -81,11 +119,69 @@ public class ExprColumn extends CubeColumn { this.type = props.get(MetastoreUtil.getExprTypePropertyKey(getName())); } + @NoArgsConstructor + @ToString(exclude = {"astNode", "hasHashCode", "hashCode"}) + public static class ExprSpec { + @Getter + @Setter + private String expr; + @Getter + @Setter + private Date startTime; + @Getter + @Setter + private Date endTime; + + private transient ASTNode astNode; + private boolean hasHashCode = false; + private transient int hashCode; + + public ExprSpec(String expr, Date startTime, Date endTime) { + this.expr = expr; + this.startTime = startTime; + this.endTime = endTime; + } + + synchronized ASTNode getASTNode() { + if (astNode == null) { + try { + if (StringUtils.isNotBlank(expr)) { + astNode = HQLParser.parseExpr(getExpr()); + } + } catch (ParseException e) { + throw new IllegalArgumentException("Expression can't be parsed: " + getExpr(), e); + } + } + return astNode; + } + + @Override + public int hashCode() { + if (!hasHashCode) { + final int prime = 31; + int result = 1; + if (getASTNode() != null) { + String exprNormalized = HQLParser.getString(getASTNode()); + result = prime * result + exprNormalized.hashCode(); + } + result = prime * result + ((getStartTime() == null) ? 0 : COLUMN_TIME_FORMAT.get().format( + getStartTime()).hashCode()); + result = prime * result + ((getEndTime() == null) ? 0 : COLUMN_TIME_FORMAT.get().format( + getEndTime()).hashCode()); + hashCode = result; + hasHashCode = true; + } + return hashCode; + } + } + /** + * Returns the first expression. + * * @return the expression */ public String getExpr() { - return expressionSet.iterator().next(); + return expressionSet.iterator().next().getExpr(); } public String getType() { @@ -96,11 +192,25 @@ public class ExprColumn extends CubeColumn { public void addProperties(Map<String, String> props) { super.addProperties(props); - String[] encodedExpressions = expressionSet.toArray(new String[expressionSet.size()]); - for (int i = 0; i < encodedExpressions.length; i++) { - String expression = encodedExpressions[i]; + String[] encodedExpressions = new String[expressionSet.size()]; + StringBuilder exprSpecBuilder = new StringBuilder(); + int i = 0; + for (ExprSpec es : expressionSet) { + String expression = es.getExpr(); try { - encodedExpressions[i] = Base64.encodeBase64String(expression.getBytes("UTF-8")); + exprSpecBuilder.append(Base64.encodeBase64String(expression.getBytes("UTF-8"))); + exprSpecBuilder.append(EXPRESSION_SPEC_DELIMITER); + if (es.getStartTime() != null) { + exprSpecBuilder.append(COLUMN_TIME_FORMAT.get().format(es.getStartTime())); + } + exprSpecBuilder.append(EXPRESSION_SPEC_DELIMITER); + if (es.getEndTime() != null) { + exprSpecBuilder.append(COLUMN_TIME_FORMAT.get().format(es.getEndTime())); + } + // encoded expression contains the Base64 encoded expression, start time and end time. + encodedExpressions[i] = exprSpecBuilder.toString(); + exprSpecBuilder.setLength(0); + i++; } catch (UnsupportedEncodingException e) { throw new IllegalArgumentException("Failed to encode expression " + expression); } @@ -119,9 +229,8 @@ public class ExprColumn extends CubeColumn { int result = super.hashCode(); result = prime * result + ((getType() == null) ? 0 : getType().toLowerCase().hashCode()); - for (ASTNode exprNode : getExpressionASTList()) { - String exprNormalized = HQLParser.getString(exprNode); - result = prime * result + exprNormalized.hashCode(); + for (ExprSpec exprSpec : expressionSet) { + result = prime * result + exprSpec.hashCode(); } hashCode = result; @@ -161,6 +270,36 @@ public class ExprColumn extends CubeColumn { return false; } } + // compare start and end times for expressions + Iterator<ExprSpec> thisIter = this.expressionSet.iterator(); + Iterator<ExprSpec> otherIter = other.expressionSet.iterator(); + while (thisIter.hasNext() && otherIter.hasNext()) { + ExprSpec thisES = thisIter.next(); + ExprSpec otherES = otherIter.next(); + if (!equalDates(thisES.getStartTime(), otherES.getStartTime())) { + return false; + } + if (!equalDates(thisES.getEndTime(), otherES.getEndTime())) { + return false; + } + } + if (thisIter.hasNext() != otherIter.hasNext()) { + return false; + } + return true; + } + + private boolean equalDates(Date d1, Date d2) { + if (d1 == null) { + if (d2 != null) { + return false; + } + } else if (d2 == null) { + return false; + } else if (!COLUMN_TIME_FORMAT.get().format(d1).equals(COLUMN_TIME_FORMAT.get().format( + d2))) { + return false; + } return true; } @@ -178,35 +317,49 @@ public class ExprColumn extends CubeColumn { * @return the ast * @throws ParseException */ - public ASTNode getAst() throws ParseException { + public ASTNode getAst() { return getExpressionASTList().get(0); } public List<ASTNode> getExpressionASTList() { - if (astNodeList == null) { - astNodeList = new ArrayList<ASTNode>(expressionSet.size()); - for (String expr : expressionSet) { - try { - astNodeList.add(HQLParser.parseExpr(expr)); - } catch (ParseException e) { - // Should not throw exception since expr should have been validated when it was added - throw new IllegalStateException("Expression can't be parsed: " + expr, e); + synchronized (expressionSet) { + if (astNodeList.isEmpty()) { + for (ExprSpec expr : expressionSet) { + astNodeList.add(expr.getASTNode()); } } } return astNodeList; } - private Set<String> getAllExpressions() { + private Set<ExprSpec> getAllExpressions() { return expressionSet; } + private final Set<String> cachedExpressionStrings = new LinkedHashSet<String>(); + /** - * Get immutable view of this column's expressions + * Get immutable view of this column's expression strings * * @return */ public Collection<String> getExpressions() { + if (cachedExpressionStrings.isEmpty()) { + synchronized (expressionSet) { + for (ExprSpec es : expressionSet) { + cachedExpressionStrings.add(es.getExpr()); + } + } + } + return Collections.unmodifiableSet(cachedExpressionStrings); + } + + /** + * Get immutable view of this column's expression full spec + * + * @return + */ + public Collection<ExprSpec> getExpressionSpecs() { return Collections.unmodifiableSet(expressionSet); } @@ -216,14 +369,16 @@ public class ExprColumn extends CubeColumn { * @param expression * @throws ParseException */ - public void addExpression(String expression) throws ParseException { - if (expression == null || expression.isEmpty()) { + public void addExpression(ExprSpec expression) throws ParseException { + if (expression == null || expression.getExpr().isEmpty()) { throw new IllegalArgumentException("Empty expression not allowed"); } // Validate if expression can be correctly parsed - HQLParser.parseExpr(expression); - expressionSet.add(expression); + HQLParser.parseExpr(expression.getExpr()); + synchronized (expressionSet) { + expressionSet.add(expression); + } astNodeList = null; hasHashCode = false; } @@ -238,7 +393,17 @@ public class ExprColumn extends CubeColumn { if (expression == null || expression.isEmpty()) { throw new IllegalArgumentException("Empty expression not allowed"); } - boolean removed = expressionSet.remove(expression); + boolean removed = false; + synchronized (expressionSet) { + Iterator<ExprSpec> it = expressionSet.iterator(); + while (it.hasNext()) { + if (it.next().getExpr().equals(expression)) { + it.remove(); + removed = true; + break; + } + } + } if (removed) { astNodeList = null; hasHashCode = false; http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/365f5f72/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java index fa23b8e..c81cf34 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java @@ -32,7 +32,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.HiveParser; -import org.apache.hadoop.hive.ql.parse.ParseException; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.antlr.runtime.CommonToken; @@ -112,11 +111,7 @@ class ExpressionResolver implements ContextRewriter { if (((AbstractBaseTable) cubeql.getQueriedTable(table)).getExpressionByName(column) == null) { return null; } - try { - return ((AbstractBaseTable) cubeql.getQueriedTable(table)).getExpressionByName(column).getAst(); - } catch (ParseException e) { - throw new SemanticException(e); - } + return ((AbstractBaseTable) cubeql.getQueriedTable(table)).getExpressionByName(column).getAst(); } private ASTNode getExprAST(final CubeQueryContext cubeql, final String column) throws SemanticException { @@ -143,11 +138,7 @@ class ExpressionResolver implements ContextRewriter { if (expr == null) { return null; } - try { - return expr.getAst(); - } catch (ParseException e) { - throw new SemanticException(e); - } + return expr.getAst(); } private ASTNode replaceAlias(final ASTNode expr, final CubeQueryContext cubeql) throws SemanticException { http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/365f5f72/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index 8c8d578..d481f93 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -21,6 +21,7 @@ package org.apache.lens.cube.metadata; import java.util.*; +import org.apache.lens.cube.metadata.ExprColumn.ExprSpec; import org.apache.lens.cube.metadata.timeline.EndsAndHolesPartitionTimeline; import org.apache.lens.cube.metadata.timeline.PartitionTimeline; import org.apache.lens.cube.metadata.timeline.StoreAllPartitionTimeline; @@ -196,14 +197,30 @@ public class TestCubeMetastoreClient { cubeDimensions.add(new ReferencedDimAtrribute(new FieldSchema("dim2", "id", "ref dim"), "Dim2 refer", new TableReference("testdim2", "id"))); + ExprSpec expr1 = new ExprSpec(); + expr1.setExpr("avg(msr1 + msr2)"); + ExprSpec expr2 = new ExprSpec(); + expr2.setExpr("avg(msr2 + msr1)"); + ExprSpec expr3 = new ExprSpec(); + expr3.setExpr("avg(msr1 + msr2 - msr1 + msr1)"); cubeExpressions.add(new ExprColumn(new FieldSchema("msr5", "double", "fifth measure"), "Avg msr5", - "avg(msr1 + msr2)", "avg(msr2 + msr1)", "avg(msr1 + msr2 - msr1 + msr1)")); + expr1, expr2, expr3)); + expr1 = new ExprSpec(); + expr1.setExpr("avg(msr1 + msr2)"); cubeExpressions.add(new ExprColumn(new FieldSchema("msr5start", "double", "expr measure with start and end times"), - "AVG of SUM", "avg(msr1 + msr2)")); + "AVG of SUM", expr1)); + expr1 = new ExprSpec(); + expr1.setExpr("dim1 != 'x' AND dim2 != 10 "); + expr2 = new ExprSpec(); + expr2.setExpr("dim1 | dim2 AND dim2 = 'XYZ'"); cubeExpressions.add(new ExprColumn(new FieldSchema("booleancut", "boolean", "a boolean expression"), "Boolean Cut", - "dim1 != 'x' AND dim2 != 10 ", "dim1 | dim2 AND dim2 = 'XYZ'")); + expr1, expr2)); + expr1 = new ExprSpec(); + expr1.setExpr("substr(dim1, 3)"); + expr2 = new ExprSpec(); + expr2.setExpr("substr(dim2, 3)"); cubeExpressions.add(new ExprColumn(new FieldSchema("substrexpr", "string", "a subt string expression"), - "SUBSTR EXPR", "substr(dim1, 3)", "substr(dim2, 3)")); + "SUBSTR EXPR", expr1, expr2)); List<CubeDimAttribute> locationHierarchyWithStartTime = new ArrayList<CubeDimAttribute>(); locationHierarchyWithStartTime.add(new ReferencedDimAtrribute(new FieldSchema("zipcode2", "int", "zip"), @@ -300,7 +317,8 @@ public class TestCubeMetastoreClient { cityAttrs.add(new ReferencedDimAtrribute(new FieldSchema("stateid", "int", "state id"), "State refer", new TableReference("statedim", "id"))); dimExpressions.add(new ExprColumn(new FieldSchema("stateAndCountry", "String", "state and country together"), - "State and Country", "concat(statedim.name, \":\", countrydim.name)", "state_and_country")); + "State and Country", new ExprSpec("concat(statedim.name, \":\", countrydim.name)", null, null), + new ExprSpec("state_and_country", null, null))); dimExpressions.add(new ExprColumn(new FieldSchema("CityAddress", "string", "city with state and city and zip"), "City Address", "concat(citydim.name, \":\", statedim.name, \":\", countrydim.name, \":\", zipcode.code)")); Map<String, String> dimProps = new HashMap<String, String>(); @@ -376,11 +394,15 @@ public class TestCubeMetastoreClient { ExprColumn stateCountryExpr = new ExprColumn(new FieldSchema("stateAndCountry", "String", "state and country together with hiphen as separator"), "State and Country", "concat(statedim.name, \"-\", countrydim.name)"); - stateCountryExpr.addExpression("concat(countrydim.name, \"-\", countrydim.name)"); + ExprSpec expr1 = new ExprSpec(); + expr1.setExpr("concat(countrydim.name, \"-\", countrydim.name)"); + stateCountryExpr.addExpression(expr1); // Assert expression validation try { - stateCountryExpr.addExpression("contact(countrydim.name"); + expr1 = new ExprSpec(); + expr1.setExpr("contact(countrydim.name"); + stateCountryExpr.addExpression(expr1); Assert.fail("Expected add expression to fail because of syntax error"); } catch (ParseException exc) { // Pass http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/365f5f72/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestExprColumn.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestExprColumn.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestExprColumn.java index 51db143..d1cd590 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestExprColumn.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestExprColumn.java @@ -19,14 +19,40 @@ package org.apache.lens.cube.metadata; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.*; + +import java.util.Calendar; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +import org.apache.lens.cube.metadata.ExprColumn.ExprSpec; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.testng.annotations.Test; public class TestExprColumn { + + private final Date now; + private final Date twoDaysBack; + private final Date nowUptoHours; + private final Date twoDaysBackUptoHours; + + public TestExprColumn() { + Calendar cal = Calendar.getInstance(); + now = cal.getTime(); + cal.add(Calendar.DAY_OF_MONTH, -2); + twoDaysBack = cal.getTime(); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + twoDaysBackUptoHours = cal.getTime(); + cal.add(Calendar.DAY_OF_MONTH, +2); + nowUptoHours = cal.getTime(); + } + @Test public void testExprColumnEquality() throws Exception { FieldSchema colSchema = new FieldSchema("someExprCol", "double", "some exprcol"); @@ -36,6 +62,10 @@ public class TestExprColumn { assertEquals(col1, col2); assertEquals(col1.hashCode(), col2.hashCode()); + col2 = new ExprColumn(colSchema, "someExprDisplayString", new ExprSpec("avg(MSR1) + avg(MSR2)", null, null)); + assertEquals(col1, col2); + assertEquals(col1.hashCode(), col2.hashCode()); + ExprColumn col3 = new ExprColumn(colSchema, "someExprDisplayString", "AVG(msr1)"); assertNotEquals(col1, col3); assertNotEquals(col1.hashCode(), col3.hashCode()); @@ -49,4 +79,129 @@ public class TestExprColumn { assertEquals(col4, col6); assertEquals(col4.hashCode(), col6.hashCode()); } + + @Test + public void testExpressionStartAndEndTimes() throws Exception { + FieldSchema colSchema = new FieldSchema("multiExprCol", "double", "multi exprcol"); + + ExprColumn col1 = new ExprColumn(colSchema, "multiExprColDisplayString", new ExprSpec("avg(MSR1) + avg(MSR2)", + twoDaysBack, null)); + Map<String, String> props = new HashMap<String, String>(); + col1.addProperties(props); + ExprColumn col1FromProps = new ExprColumn("multiExprCol", props); + assertEquals(col1, col1FromProps); + assertEquals(col1.hashCode(), col1FromProps.hashCode()); + assertEquals(col1FromProps.getExpressionSpecs().iterator().next().getStartTime(), twoDaysBackUptoHours); + + col1 = new ExprColumn(colSchema, "multiExprColDisplayString", new ExprSpec("avg(MSR1) + avg(MSR2)", + null, twoDaysBack)); + props = new HashMap<String, String>(); + col1.addProperties(props); + col1FromProps = new ExprColumn("multiExprCol", props); + assertEquals(col1, col1FromProps); + assertEquals(col1.hashCode(), col1FromProps.hashCode()); + assertEquals(col1FromProps.getExpressionSpecs().iterator().next().getEndTime(), twoDaysBackUptoHours); + + + col1 = new ExprColumn(colSchema, "multiExprColDisplayString", new ExprSpec("avg(MSR1) + avg(MSR2)", + twoDaysBack, now)); + props = new HashMap<String, String>(); + col1.addProperties(props); + col1FromProps = new ExprColumn("multiExprCol", props); + assertEquals(col1, col1FromProps); + assertEquals(col1.hashCode(), col1FromProps.hashCode()); + assertEquals(col1FromProps.getExpressionSpecs().iterator().next().getStartTime(), twoDaysBackUptoHours); + assertEquals(col1FromProps.getExpressionSpecs().iterator().next().getEndTime(), nowUptoHours); + } + + @Test + public void testMultipleExpressionStartAndEndTimes() throws Exception { + FieldSchema colSchema = new FieldSchema("multiExprCol", "double", "multi exprcol"); + + ExprColumn col1 = new ExprColumn(colSchema, "multiExprColDisplayString", new ExprSpec("avg(MSR1) + avg(MSR2)", + twoDaysBack, null), new ExprSpec("avg(MSR1) + avg(MSR2) - m1 + m1", now, null), + new ExprSpec("avg(MSR1) + avg(MSR2)", null, twoDaysBack), + new ExprSpec("avg(MSR1) + avg(MSR2) + 0.01", twoDaysBack, now)); + Map<String, String> props = new HashMap<String, String>(); + col1.addProperties(props); + ExprColumn col1FromProps = new ExprColumn("multiExprCol", props); + assertEquals(col1, col1FromProps); + assertEquals(col1.hashCode(), col1FromProps.hashCode()); + Iterator<ExprSpec> it = col1FromProps.getExpressionSpecs().iterator(); + assertEquals(it.next().getStartTime(), twoDaysBackUptoHours); + assertEquals(it.next().getStartTime(), nowUptoHours); + ExprSpec endTimeSpecified = it.next(); + assertNull(endTimeSpecified.getStartTime()); + assertEquals(endTimeSpecified.getEndTime(), twoDaysBackUptoHours); + ExprSpec last = it.next(); + assertEquals(last.getStartTime(), twoDaysBackUptoHours); + assertEquals(last.getEndTime(), nowUptoHours); + assertFalse(it.hasNext()); + } + + @Test + public void testExprColumnCreationErrors() { + FieldSchema colSchema = new FieldSchema("errorColumn", "double", "multi exprcol"); + + // no expression spec passed + try { + ExprColumn col1 = new ExprColumn(colSchema, "NoExprSpec", (ExprSpec[])null); + fail(col1 + " should not be created"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("No expressions specified for column errorColumn")); + } + + // no expression passed in exprspec + try { + ExprColumn col1 = new ExprColumn(colSchema, "NoExprInExprSpec", new ExprSpec(null, null, null)); + fail(col1 + " should not be created"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("No expression string specified for column errorColumn at index:0")); + } + + // Parse error in expr passed in exprspec + try { + ExprColumn col1 = new ExprColumn(colSchema, "NoExprInExprSpec", new ExprSpec("(a+b", null, null)); + fail(col1 + " should not be created"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("Expression can't be parsed: (a+b"), e.getMessage()); + } + + // Parse error in expr passed in exprspec + try { + ExprColumn col1 = new ExprColumn(colSchema, "NoExprInExprSpec", new ExprSpec("a + b", null, null), + new ExprSpec("(a+b", null, null)); + fail(col1 + " should not be created"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("Expression can't be parsed: (a+b")); + } + + // no expression passed in exprspec + try { + ExprColumn col1 = new ExprColumn(colSchema, "NoExprInExprSpecAt1", new ExprSpec("a + b", null, null), + new ExprSpec(null, null, null)); + fail(col1 + " should not be created"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("No expression string specified for column errorColumn at index:1")); + } + + // startTime after endTime + try { + ExprColumn col1 = new ExprColumn(colSchema, "startTimeAfterEndTime", new ExprSpec("a + b", now, twoDaysBack)); + fail(col1 + " should not be created"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("Start time is after end time for column errorColumn " + + "for expression at index:0"), e.getMessage()); + } + + // startTime after endTime + try { + ExprColumn col1 = new ExprColumn(colSchema, "startTimeAfterEndTimeAt1", new ExprSpec("a + b", null, null), + new ExprSpec("a + b", now, twoDaysBack)); + fail(col1 + " should not be created"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("Start time is after end time for column errorColumn " + + "for expression at index:1"), e.getMessage()); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/365f5f72/lens-examples/src/main/resources/customer.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/customer.xml b/lens-examples/src/main/resources/customer.xml index e78dab2..64024f1 100644 --- a/lens-examples/src/main/resources/customer.xml +++ b/lens-examples/src/main/resources/customer.xml @@ -33,7 +33,7 @@ <chain_ref_column chain_name="customer_city" ref_col="name" /> </ref_spec> </dim_attribute> - <dim_attribute name="customer_credit_status" type="STRING" start_time='2015-03-01-00:00'/> + <dim_attribute name="customer_credit_status" type="STRING" start_time='2015-03-01T00:00:00'/> </attributes> <join_chains> <join_chain name="customer_city"> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/365f5f72/lens-examples/src/main/resources/product.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/product.xml b/lens-examples/src/main/resources/product.xml index 8093561..bb91997 100644 --- a/lens-examples/src/main/resources/product.xml +++ b/lens-examples/src/main/resources/product.xml @@ -30,6 +30,12 @@ <dim_attribute name="category" type="STRING" /> <dim_attribute name="manufacturer" type="STRING" /> </attributes> + <expressions> + <expression name="isHeavy" type="boolean"> + <expr_spec expr = "weight > 10" end_time="2015-04-12T00:00:00"/> + <expr_spec expr = "weight > 50" start_time="2015-04-12T00:00:00"/> + </expression> + </expressions> <properties> <property name="dimension.product.timed.dimension" value="dt"/> </properties> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/365f5f72/lens-examples/src/main/resources/sales-cube.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/sales-cube.xml b/lens-examples/src/main/resources/sales-cube.xml index 951ace9..f5e2f93 100644 --- a/lens-examples/src/main/resources/sales-cube.xml +++ b/lens-examples/src/main/resources/sales-cube.xml @@ -25,7 +25,7 @@ <measure name="unit_sales" type="BIGINT" default_aggr="SUM" display_string="Unit Sales" format_string="#,###"/> <measure name="store_sales" type="DOUBLE" default_aggr="SUM" display_string="Store Sales" format_string="#,###.##"/> <measure name="store_cost" type="DOUBLE" default_aggr="SUM" display_string="Store Cost" format_string="#,###.00" - start_time='2015-03-01-00:00' /> + start_time='2015-03-01T00:00:00' /> <measure name="line_item_quantity" type="INT" default_aggr="SUM" display_string="Line item quantity"/> <measure name="line_item_product_price" type="FLOAT" default_aggr="SUM" display_string="Line item product price"/> <measure name="line_item_discount_amount" type="FLOAT" default_aggr="SUM" display_string="Line item discount"/> @@ -35,9 +35,14 @@ display_string="Maximum Line item discount"/> </measures> <expressions> - <expression name="profit" type="DOUBLE" expr="store_sales - store_cost" display_string="Profit"/> - <expression name="promotion_sales" type="DOUBLE" expr="sum(case when promotion_id = 0 then 0 else store_sales end)" - display_string="Promotion sales"/> + <expression name="profit" type="DOUBLE" display_string="Profit"> + <expr_spec expr="store_sales - store_cost"/> + </expression> + <expression name="promotion_sales" type="DOUBLE" display_string="Promotion sales"> + <expr_spec expr="sum(case when promotion_id = 0 then 0 else store_sales end)" start_time='2015-04-12T00:00:00'/> + <expr_spec expr="sum(case when promotion_id = -1 then 0 when promotion_id = -1 then 0 else store_sales end)" + end_time="2015-04-12T00:00:00"/> + </expression> </expressions> <dim_attributes> <dim_attribute name="customer_id" type="INT" /> @@ -47,7 +52,7 @@ <dim_attribute name="order_line_number" type="INT" /> <dim_attribute name="order_time" type="TIMESTAMP" /> <dim_attribute name="delivery_time" type="TIMESTAMP" /> - <dim_attribute name="customer_city_id" type="INT" start_time='2015-03-01-00:00' /> + <dim_attribute name="customer_city_id" type="INT" start_time='2015-03-01T00:00:00' /> <dim_attribute name="production_city_id" type="INT" /> <dim_attribute name="delivery_city_id" type="INT" /> <dim_attribute name="customer_city_name" type="string" description="City name to which the customer belongs" http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/365f5f72/lens-examples/src/main/resources/sample-cube.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/sample-cube.xml b/lens-examples/src/main/resources/sample-cube.xml index f24a8cc..0d97842 100644 --- a/lens-examples/src/main/resources/sample-cube.xml +++ b/lens-examples/src/main/resources/sample-cube.xml @@ -24,15 +24,18 @@ <measures> <measure name="measure1" type="BIGINT"/> <measure name="measure2" type="INT" default_aggr="SUM"/> - <measure name="measure3" type="FLOAT" default_aggr="MAX" start_time='2013-12-12-00:00'/> + <measure name="measure3" type="FLOAT" default_aggr="MAX" start_time='2013-12-12T00:00:00'/> <measure name="measure4" type="DOUBLE" default_aggr="MIN"/> </measures> <expressions> - <expression name="expr_msr5" type="DOUBLE" expr="measure3 + measure4"/> + <expression name="expr_msr5" type="DOUBLE"> + <expr_spec expr = "measure3 + measure4" start_time='2013-12-12T00:00:00' /> + <expr_spec expr = "measure3 + measure4 + 0.01" end_time='2013-12-12T00:00:00'/> + </expression> </expressions> <dim_attributes> <dim_attribute name="dim1" type="INT"/> - <dim_attribute name="dim2" type="INT" start_time='2013-12-01-00:00'/> + <dim_attribute name="dim2" type="INT" start_time='2013-12-01T00:00:00'/> <dim_attribute name="dim3" type="INT" join_key="true"> <ref_spec> <table_references> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/365f5f72/lens-examples/src/main/resources/sample-db-only-dimension.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/sample-db-only-dimension.xml b/lens-examples/src/main/resources/sample-db-only-dimension.xml index 63a8bf8..4c6bec6 100644 --- a/lens-examples/src/main/resources/sample-db-only-dimension.xml +++ b/lens-examples/src/main/resources/sample-db-only-dimension.xml @@ -24,8 +24,8 @@ <attributes> <dim_attribute name="id" type="INT"/> <dim_attribute name="name" type="STRING"/> - <dim_attribute name="detail" type="STRING" start_time='2013-12-01-00:00'/> - <dim_attribute name="d2id" type="INT" start_time='2013-12-01-00:00' join_key="true"> + <dim_attribute name="detail" type="STRING" start_time='2013-12-01T00:00:00'/> + <dim_attribute name="d2id" type="INT" start_time='2013-12-01T00:00:00' join_key="true"> <ref_spec> <table_references> <table_reference table="sample_dim2" column="id"/> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/365f5f72/lens-examples/src/main/resources/sample-dimension.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/sample-dimension.xml b/lens-examples/src/main/resources/sample-dimension.xml index af4ab21..9b97da7 100644 --- a/lens-examples/src/main/resources/sample-dimension.xml +++ b/lens-examples/src/main/resources/sample-dimension.xml @@ -24,8 +24,8 @@ <attributes> <dim_attribute name="id" type="INT"/> <dim_attribute name="name" type="STRING"/> - <dim_attribute name="detail" type="STRING" start_time='2013-12-01-00:00'/> - <dim_attribute name="d2id" type="INT" start_time='2013-12-01-00:00' join_key="true"> + <dim_attribute name="detail" type="STRING" start_time='2013-12-01T00:00:00'/> + <dim_attribute name="d2id" type="INT" start_time='2013-12-01T00:00:00' join_key="true"> <ref_spec> <table_references> <table_reference table="sample_dim2" column="id"/> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/365f5f72/lens-examples/src/main/resources/sample-dimension2.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/sample-dimension2.xml b/lens-examples/src/main/resources/sample-dimension2.xml index d1ac80d..9740507 100644 --- a/lens-examples/src/main/resources/sample-dimension2.xml +++ b/lens-examples/src/main/resources/sample-dimension2.xml @@ -24,7 +24,7 @@ <attributes> <dim_attribute name="id" type="INT"/> <dim_attribute name="name" type="STRING"/> - <dim_attribute name="detail2" type="STRING" start_time='2013-12-01-00:00'/> + <dim_attribute name="detail2" type="STRING" start_time='2013-12-01T00:00:00'/> </attributes> <properties> <property name="dimension.sample_dim2.timed.dimension" value="dt"/> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/365f5f72/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java index 3cf7011..a628c90 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java @@ -28,6 +28,7 @@ import javax.xml.datatype.XMLGregorianCalendar; import org.apache.lens.api.metastore.*; import org.apache.lens.cube.metadata.*; +import org.apache.lens.cube.metadata.ExprColumn.ExprSpec; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -204,6 +205,14 @@ public final class JAXBUtils { return hiveDim; } + /** + * Get XMLGregorianCalendar from Date. + * + * Useful for converting from java code to XML spec. + * + * @param d Date value + * @return XML value + */ public static XMLGregorianCalendar getXMLGregorianCalendar(Date d) { if (d == null) { return null; @@ -219,6 +228,14 @@ public final class JAXBUtils { } } + /** + * Get Date from XMLGregorianCalendar + * + * Useful for converting from XML spec to java code. + * + * @param cal XML value + * @return Date value + */ public static Date getDateFromXML(XMLGregorianCalendar cal) { if (cal == null) { return null; @@ -262,10 +279,30 @@ public final class JAXBUtils { xe.setType(ec.getType()); xe.setDescription(ec.getDescription()); xe.setDisplayString(ec.getDisplayString()); - xe.setExpr(ec.getExpr()); + xe.getExprSpec().addAll(xExprSpecFromExprColumn(ec.getExpressionSpecs())); return xe; } + private static Collection<XExprSpec> xExprSpecFromExprColumn(Collection<ExprSpec> esSet) { + List<XExprSpec> xes = new ArrayList<XExprSpec>(); + for (ExprSpec es : esSet) { + XExprSpec e = new XExprSpec(); + e.setExpr(es.getExpr()); + e.setStartTime(getXMLGregorianCalendar(es.getStartTime())); + e.setEndTime(getXMLGregorianCalendar(es.getEndTime())); + xes.add(e); + } + return xes; + } + + private static ExprSpec[] exprSpecFromXExprColumn(Collection<XExprSpec> xesList) { + List<ExprSpec> esArray = new ArrayList<ExprSpec>(xesList.size()); + for (XExprSpec xes : xesList) { + esArray.add(new ExprSpec(xes.getExpr(), getDateFromXML(xes.getStartTime()), getDateFromXML(xes.getEndTime()))); + } + return esArray.toArray(new ExprSpec[0]); + } + /** * Create XDimAttribute from CubeDimAttribute */ @@ -274,6 +311,8 @@ public final class JAXBUtils { xd.setName(cd.getName()); xd.setDescription(cd.getDescription()); xd.setDisplayString(cd.getDisplayString()); + xd.setStartTime(getXMLGregorianCalendar(cd.getStartTime())); + xd.setEndTime(getXMLGregorianCalendar(cd.getEndTime())); if (cd instanceof ReferencedDimAtrribute) { ReferencedDimAtrribute rd = (ReferencedDimAtrribute) cd; List<TableReference> dimRefs = rd.getReferences(); @@ -405,7 +444,7 @@ public final class JAXBUtils { ExprColumn ec = new ExprColumn(new FieldSchema(xe.getName(), xe.getType().toLowerCase(), xe.getDescription()), xe.getDisplayString(), - xe.getExpr()); + exprSpecFromXExprColumn(xe.getExprSpec())); return ec; } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/365f5f72/lens-server/src/main/java/org/apache/lens/server/ui/MetastoreUIResource.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/ui/MetastoreUIResource.java b/lens-server/src/main/java/org/apache/lens/server/ui/MetastoreUIResource.java index afb404b..b7c60bb 100644 --- a/lens-server/src/main/java/org/apache/lens/server/ui/MetastoreUIResource.java +++ b/lens-server/src/main/java/org/apache/lens/server/ui/MetastoreUIResource.java @@ -194,7 +194,7 @@ public class MetastoreUIResource { for (XExprColumn expr : table.getExpressions().getExpression()) { try { attribList.put(new JSONObject().put("name", expr.getName()).put("type", "expression") - .put("expression", expr.getExpr())); + .put("expression", expr.getExprSpec())); } catch (JSONException j) { LOG.error(j); } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/365f5f72/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java index a35a8ff..885a69a 100644 --- a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java +++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java @@ -42,6 +42,7 @@ import org.apache.lens.api.LensSessionHandle; import org.apache.lens.api.StringList; import org.apache.lens.api.metastore.*; import org.apache.lens.cube.metadata.*; +import org.apache.lens.cube.metadata.ExprColumn.ExprSpec; import org.apache.lens.server.LensJerseyTest; import org.apache.lens.server.LensServices; import org.apache.lens.server.LensTestUtil; @@ -383,9 +384,34 @@ public class TestMetastoreService extends LensJerseyTest { xe1.setType("DOUBLE"); xe1.setDescription("first expression"); xe1.setDisplayString("Expression1"); - xe1.setExpr("msr1/1000"); + XExprSpec es = new XExprSpec(); + es.setExpr("msr1/1000"); + xe1.getExprSpec().add(es); + + XExprColumn xe2 = new XExprColumn(); + xe2.setName("expr2"); + xe2.setType("float"); + xe2.setDescription("multi expression"); + xe2.setDisplayString("Expression2"); + XExprSpec es1 = new XExprSpec(); + es1.setExpr("msr1/1000"); + xe2.getExprSpec().add(es1); + XExprSpec es2 = new XExprSpec(); + es2.setExpr("(msr1/1000) + 0.01"); + es2.setStartTime(startDate); + xe2.getExprSpec().add(es2); + XExprSpec es3 = new XExprSpec(); + es3.setExpr("(msr1/1000) + 0.03"); + es3.setEndTime(endDate); + xe2.getExprSpec().add(es3); + XExprSpec es4 = new XExprSpec(); + es4.setExpr("(msr1/1000) - 0.01"); + es4.setStartTime(startDate); + es4.setEndTime(endDate); + xe2.getExprSpec().add(es4); cube.getExpressions().getExpression().add(xe1); + cube.getExpressions().getExpression().add(xe2); XProperty xp1 = cubeObjectFactory.createXProperty(); xp1.setName("foo"); @@ -650,6 +676,27 @@ public class TestMetastoreService extends LensJerseyTest { assertNotNull(hcube.getExpressionByName("expr1")); assertEquals(hcube.getExpressionByName("expr1").getDescription(), "first expression"); assertEquals(hcube.getExpressionByName("expr1").getDisplayString(), "Expression1"); + assertNotNull(hcube.getExpressionByName("expr2")); + assertEquals(hcube.getExpressionByName("expr2").getExpressions().size(), 4); + ExprColumn expr2 = hcube.getExpressionByName("expr2"); + Iterator<ExprSpec> esIter = expr2.getExpressionSpecs().iterator(); + ExprSpec first = esIter.next(); + assertEquals(first.getExpr(), "msr1/1000"); + assertNull(first.getStartTime()); + assertNull(first.getEndTime()); + ExprSpec second = esIter.next(); + assertEquals(second.getExpr(), "(msr1/1000) + 0.01"); + assertNotNull(second.getStartTime()); + assertNull(second.getEndTime()); + ExprSpec third = esIter.next(); + assertEquals(third.getExpr(), "(msr1/1000) + 0.03"); + assertNull(third.getStartTime()); + assertNotNull(third.getEndTime()); + ExprSpec last = esIter.next(); + assertEquals(last.getExpr(), "(msr1/1000) - 0.01"); + assertNotNull(last.getStartTime()); + assertNotNull(last.getEndTime()); + assertFalse(esIter.hasNext()); Assert.assertFalse(hcube.getJoinChains().isEmpty()); Assert.assertEquals(hcube.getJoinChains().size(), 2); Assert.assertTrue(hcube.getJoinChainNames().contains("chain1")); @@ -1030,7 +1077,9 @@ public class TestMetastoreService extends LensJerseyTest { xe1.setType("STRING"); xe1.setDescription("dimension expression"); xe1.setDisplayString("Dim Expression"); - xe1.setExpr("substr(col1, 3)"); + XExprSpec es = new XExprSpec(); + es.setExpr("substr(col1, 3)"); + xe1.getExprSpec().add(es); dimension.getExpressions().getExpression().add(xe1); XProperty xp1 = cubeObjectFactory.createXProperty(); @@ -2298,6 +2347,7 @@ public class TestMetastoreService extends LensJerseyTest { "flattestcube.testdim2col2", "flattestcube.dim4", "flattestcube.expr1", + "flattestcube.expr2", "chain1-testdim.col2", "chain1-testdim.col1", "chain1-testdim.dimexpr", http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/365f5f72/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java index 6f45e88..8ecf2a8 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java @@ -259,8 +259,7 @@ public class TestQueryService extends LensJerseyTest { assertEquals(ctx.getDriverFinishTime(), 0); assertTrue(ctx.getFinishTime() > 0); Assert.assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.FAILED); - System.out.println("%% " + metricsSvc.getTotalFailedQueries()); - Assert.assertEquals(metricsSvc.getTotalFailedQueries(), failedQueries + 1); + Assert.assertTrue(metricsSvc.getTotalFailedQueries() >= failedQueries + 1); } // test with execute async post, get all queries, get query context,
