LENS-1381: Support Fact to Fact Union
Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/ae83caae Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/ae83caae Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/ae83caae Branch: refs/heads/master Commit: ae83caae54102af1923af46f4e8ca6640368ed34 Parents: 186f03f Author: Sushil Mohanty <[email protected]> Authored: Wed Apr 12 18:30:48 2017 +0530 Committer: Rajat Khandelwal <[email protected]> Committed: Wed Apr 12 18:30:48 2017 +0530 ---------------------------------------------------------------------- .../java/org/apache/lens/api/ToXMLString.java | 8 + .../apache/lens/api/jaxb/LensJAXBContext.java | 14 +- .../lens/api/metastore/SchemaTraverser.java | 71 + lens-api/src/main/resources/cube-0.1.xsd | 4 +- lens-api/src/main/resources/lens-errors.conf | 13 + .../lens/cli/commands/LensSchemaCommands.java | 111 +- .../lens/cube/error/LensCubeErrorCode.java | 2 + .../NoCandidateFactAvailableException.java | 19 +- .../lens/cube/metadata/CubeFactTable.java | 13 +- .../lens/cube/metadata/CubeMetastoreClient.java | 248 +- .../org/apache/lens/cube/metadata/DateUtil.java | 29 +- .../lens/cube/metadata/FactPartition.java | 10 + .../apache/lens/cube/metadata/JAXBUtils.java | 1114 ++++++++ .../lens/cube/metadata/MetastoreUtil.java | 1 - .../apache/lens/cube/metadata/TimeRange.java | 22 +- .../cube/parse/AbridgedTimeRangeWriter.java | 42 +- .../lens/cube/parse/AggregateResolver.java | 21 +- .../lens/cube/parse/BetweenTimeRangeWriter.java | 2 +- .../org/apache/lens/cube/parse/Candidate.java | 139 + .../parse/CandidateCoveringSetsResolver.java | 315 +++ .../apache/lens/cube/parse/CandidateDim.java | 7 +- .../apache/lens/cube/parse/CandidateFact.java | 367 --- .../apache/lens/cube/parse/CandidateTable.java | 6 +- .../cube/parse/CandidateTablePruneCause.java | 297 +-- .../lens/cube/parse/CandidateTableResolver.java | 322 +-- .../apache/lens/cube/parse/CandidateUtil.java | 319 +++ .../apache/lens/cube/parse/CheckTableNames.java | 1 - .../lens/cube/parse/ColumnLifetimeChecker.java | 125 + .../apache/lens/cube/parse/ColumnResolver.java | 2 +- .../lens/cube/parse/CubeQueryContext.java | 664 +++-- .../lens/cube/parse/CubeQueryRewriter.java | 47 +- .../lens/cube/parse/CubeSemanticAnalyzer.java | 12 +- .../lens/cube/parse/DefaultAliasDecider.java | 4 + .../apache/lens/cube/parse/DefaultQueryAST.java | 15 +- .../cube/parse/DenormalizationResolver.java | 137 +- .../lens/cube/parse/ExpressionResolver.java | 244 +- .../apache/lens/cube/parse/FieldValidator.java | 1 - .../apache/lens/cube/parse/GroupbyResolver.java | 56 +- .../apache/lens/cube/parse/JoinCandidate.java | 142 + .../apache/lens/cube/parse/JoinResolver.java | 14 +- .../lens/cube/parse/LeastPartitionResolver.java | 35 +- .../lens/cube/parse/LightestFactResolver.java | 28 +- .../cube/parse/MaxCoveringFactResolver.java | 98 +- .../lens/cube/parse/MultiFactHQLContext.java | 238 -- .../org/apache/lens/cube/parse/PruneCauses.java | 37 +- .../lens/cube/parse/QueriedPhraseContext.java | 73 +- .../org/apache/lens/cube/parse/QueryAST.java | 6 + .../lens/cube/parse/SimpleHQLContext.java | 62 +- .../parse/SingleFactMultiStorageHQLContext.java | 259 -- .../SingleFactSingleStorageHQLContext.java | 73 - .../lens/cube/parse/StorageCandidate.java | 1040 ++++++++ .../lens/cube/parse/StorageTableResolver.java | 866 ++---- .../org/apache/lens/cube/parse/StorageUtil.java | 132 +- .../lens/cube/parse/TimeRangeChecker.java | 240 -- .../apache/lens/cube/parse/UnionCandidate.java | 295 +++ .../apache/lens/cube/parse/UnionHQLContext.java | 55 - .../lens/cube/parse/UnionQueryWriter.java | 730 +++++ .../lens/cube/parse/join/AutoJoinContext.java | 62 +- .../cube/parse/join/BridgeTableJoinContext.java | 22 +- .../apache/lens/driver/cube/RewriterPlan.java | 21 +- .../apache/lens/cube/metadata/DateFactory.java | 29 +- .../apache/lens/cube/parse/CubeTestSetup.java | 2491 +----------------- .../FieldsCannotBeQueriedTogetherTest.java | 2 +- .../lens/cube/parse/TestAggregateResolver.java | 236 +- .../lens/cube/parse/TestBaseCubeQueries.java | 891 +++---- .../cube/parse/TestBetweenTimeRangeWriter.java | 52 +- .../lens/cube/parse/TestBridgeTableQueries.java | 443 +++- .../lens/cube/parse/TestCubeRewriter.java | 343 +-- .../cube/parse/TestDenormalizationResolver.java | 132 +- .../lens/cube/parse/TestExpressionResolver.java | 98 +- .../lens/cube/parse/TestJoinResolver.java | 117 +- .../lens/cube/parse/TestQueryMetrics.java | 51 +- .../lens/cube/parse/TestRewriterPlan.java | 12 +- .../lens/cube/parse/TestTimeRangeResolver.java | 85 +- .../parse/TestTimeRangeWriterWithQuery.java | 41 +- .../cube/parse/TestUnionAndJoinCandidates.java | 169 ++ .../lens/cube/parse/TestUnionQueries.java | 475 ++-- .../resources/schema/cubes/base/basecube.xml | 972 +++++++ .../resources/schema/cubes/base/testcube.xml | 663 +++++ .../resources/schema/cubes/derived/der1.xml | 42 + .../resources/schema/cubes/derived/der2.xml | 63 + .../resources/schema/cubes/derived/der3.xml | 52 + .../schema/cubes/derived/derivedcube.xml | 44 + .../cubes/derived/union_join_ctx_der1.xml | 45 + .../resources/schema/dimensions/citydim.xml | 122 + .../resources/schema/dimensions/countrydim.xml | 41 + .../resources/schema/dimensions/cycledim1.xml | 71 + .../resources/schema/dimensions/cycledim2.xml | 70 + .../test/resources/schema/dimensions/daydim.xml | 61 + .../resources/schema/dimensions/hourdim.xml | 45 + .../test/resources/schema/dimensions/sports.xml | 45 + .../resources/schema/dimensions/statedim.xml | 73 + .../resources/schema/dimensions/testdim2.xml | 223 ++ .../resources/schema/dimensions/testdim3.xml | 68 + .../resources/schema/dimensions/testdim4.xml | 45 + .../schema/dimensions/unreachabledim.xml | 44 + .../schema/dimensions/user_interests.xml | 50 + .../resources/schema/dimensions/userdim.xml | 78 + .../test/resources/schema/dimensions/zipdim.xml | 50 + .../resources/schema/dimtables/citytable.xml | 88 + .../resources/schema/dimtables/citytable2.xml | 49 + .../resources/schema/dimtables/citytable3.xml | 49 + .../resources/schema/dimtables/citytable4.xml | 48 + .../resources/schema/dimtables/countrytable.xml | 52 + .../dimtables/countrytable_partitioned.xml | 55 + .../resources/schema/dimtables/cycledim1tbl.xml | 69 + .../resources/schema/dimtables/cycledim2tbl.xml | 69 + .../resources/schema/dimtables/daydimtbl.xml | 68 + .../resources/schema/dimtables/hourdimtbl.xml | 68 + .../resources/schema/dimtables/sports_tbl.xml | 68 + .../resources/schema/dimtables/statetable.xml | 74 + .../schema/dimtables/statetable_partitioned.xml | 57 + .../resources/schema/dimtables/testdim2tbl.xml | 70 + .../resources/schema/dimtables/testdim2tbl2.xml | 89 + .../resources/schema/dimtables/testdim2tbl3.xml | 89 + .../resources/schema/dimtables/testdim3tbl.xml | 69 + .../resources/schema/dimtables/testdim4tbl.xml | 68 + .../schema/dimtables/unreachabledimtable.xml | 55 + .../schema/dimtables/user_interests_tbl.xml | 70 + .../resources/schema/dimtables/usertable.xml | 72 + .../resources/schema/dimtables/ziptable.xml | 55 + .../test/resources/schema/facts/cheapfact.xml | 101 + .../test/resources/schema/facts/summary1.xml | 101 + .../test/resources/schema/facts/summary2.xml | 103 + .../test/resources/schema/facts/summary3.xml | 104 + .../test/resources/schema/facts/summary4.xml | 85 + .../test/resources/schema/facts/testfact.xml | 251 ++ .../resources/schema/facts/testfact1_base.xml | 155 ++ .../schema/facts/testfact1_raw_base.xml | 95 + .../test/resources/schema/facts/testfact2.xml | 95 + .../resources/schema/facts/testfact2_base.xml | 140 + .../resources/schema/facts/testfact2_raw.xml | 95 + .../schema/facts/testfact2_raw_base.xml | 66 + .../resources/schema/facts/testfact3_base.xml | 137 + .../schema/facts/testfact3_raw_base.xml | 64 + .../schema/facts/testfact4_raw_base.xml | 65 + .../resources/schema/facts/testfact5_base.xml | 148 ++ .../schema/facts/testfact5_raw_base.xml | 58 + .../resources/schema/facts/testfact6_base.xml | 137 + .../schema/facts/testfact_continuous.xml | 58 + .../schema/facts/testfact_deprecated.xml | 146 + .../resources/schema/facts/testfactmonthly.xml | 66 + .../schema/facts/union_join_ctx_fact1.xml | 59 + .../schema/facts/union_join_ctx_fact2.xml | 58 + .../schema/facts/union_join_ctx_fact3.xml | 58 + .../schema/facts/union_join_ctx_fact5.xml | 58 + .../schema/facts/union_join_ctx_fact6.xml | 58 + .../resources/schema/segmentations/seg1.xml | 45 + .../src/test/resources/schema/storages/c0.xml | 26 + .../src/test/resources/schema/storages/c1.xml | 26 + .../src/test/resources/schema/storages/c2.xml | 26 + .../src/test/resources/schema/storages/c3.xml | 26 + .../src/test/resources/schema/storages/c4.xml | 26 + .../src/test/resources/schema/storages/c5.xml | 26 + .../src/test/resources/schema/storages/c99.xml | 26 + .../metastore/CubeMetastoreServiceImpl.java | 141 +- .../apache/lens/server/metastore/JAXBUtils.java | 1116 -------- .../server/metastore/MetastoreResource.java | 2 +- .../lens/server/common/RestAPITestUtil.java | 11 + .../lens/server/common/TestDataUtils.java | 4 + .../server/query/QueryAPIErrorResponseTest.java | 27 + 161 files changed, 15229 insertions(+), 8297 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-api/src/main/java/org/apache/lens/api/ToXMLString.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/ToXMLString.java b/lens-api/src/main/java/org/apache/lens/api/ToXMLString.java index e74adc9..0058f20 100644 --- a/lens-api/src/main/java/org/apache/lens/api/ToXMLString.java +++ b/lens-api/src/main/java/org/apache/lens/api/ToXMLString.java @@ -24,6 +24,9 @@ import java.util.HashMap; import java.util.Map; import javax.xml.bind.*; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlType; +import javax.xml.namespace.QName; import org.apache.lens.api.jaxb.LensJAXBContext; @@ -31,6 +34,11 @@ public abstract class ToXMLString { protected static final Map<Class<?>, JAXBContext> JAXB_CONTEXTS = new HashMap<>(); public static String toString(Object o) { + if (!(o instanceof JAXBElement) && o.getClass().getAnnotation(XmlRootElement.class) == null + && o.getClass().getAnnotation(XmlType.class)!= null) { + o = new JAXBElement(new QName("uri:lens:cube:0.1", o.getClass().getAnnotation(XmlType.class).name()), + o.getClass(), null, o); + } try { StringWriter stringWriter = new StringWriter(); Class cl = null; http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-api/src/main/java/org/apache/lens/api/jaxb/LensJAXBContext.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/jaxb/LensJAXBContext.java b/lens-api/src/main/java/org/apache/lens/api/jaxb/LensJAXBContext.java index 14fc4aa..8858b95 100644 --- a/lens-api/src/main/java/org/apache/lens/api/jaxb/LensJAXBContext.java +++ b/lens-api/src/main/java/org/apache/lens/api/jaxb/LensJAXBContext.java @@ -24,6 +24,7 @@ package org.apache.lens.api.jaxb; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.Reader; import javax.xml.XMLConstants; import javax.xml.bind.*; @@ -114,17 +115,26 @@ public class LensJAXBContext extends JAXBContext { return UNMARSHALLER; } + public static <T> T unmarshall(File file) throws JAXBException, IOException { + return ((JAXBElement<T>) UNMARSHALLER.unmarshal(file)).getValue(); + } + public static <T> T unmarshall(InputStream inputStream) throws JAXBException, IOException { + return ((JAXBElement<T>) UNMARSHALLER.unmarshal(inputStream)).getValue(); + } + public static <T> T unmarshall(Reader reader) throws JAXBException, IOException { + return ((JAXBElement<T>) UNMARSHALLER.unmarshal(reader)).getValue(); + } public static <T> T unmarshallFromFile(String filename) throws JAXBException, IOException { File file = new File(filename); if (file.exists()) { - return ((JAXBElement<T>) UNMARSHALLER.unmarshal(file)).getValue(); + return unmarshall(file); } else { // load from classpath InputStream stream = Thread.currentThread().getContextClassLoader().getResourceAsStream(filename); if (stream == null) { throw new IOException("File not found:" + filename); } - return ((JAXBElement<T>) UNMARSHALLER.unmarshal(stream)).getValue(); + return unmarshall(stream); } } } http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java b/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java new file mode 100644 index 0000000..76cb8b9 --- /dev/null +++ b/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.api.metastore; + +import java.io.File; +import java.io.FilenameFilter; +import java.util.Map; +import java.util.function.BiConsumer; + +import com.google.common.collect.Maps; + +/* + * Created on 07/03/17. + */ +public class SchemaTraverser implements Runnable { + final File parent; + final Map<String, Class<?>> types = Maps.newLinkedHashMap(); + private final SchemaEntityProcessor action; + { + types.put("storages", XStorage.class); + types.put("cubes/base", XBaseCube.class); + types.put("cubes/derived", XDerivedCube.class); + types.put("dimensions", XDimension.class); + types.put("facts", XFactTable.class); + types.put("dimtables", XDimensionTable.class); + types.put("dimensiontables", XDimensionTable.class); + types.put("dimensiontables", XDimensionTable.class); + types.put("segmentations", XSegmentation.class); + } + private static final FilenameFilter XML_FILTER = (dir, name) -> name.endsWith(".xml"); + + public interface SchemaEntityProcessor extends BiConsumer<File, Class<?>> { + } + + public SchemaTraverser(File parent, SchemaEntityProcessor action) { + this.parent = parent; + this.action = action; + } + + @Override + public void run() { + for (Map.Entry<String, Class<?>> entry : types.entrySet()) { + File f = new File(parent, entry.getKey()); + if (f.exists()) { + assert f.isDirectory(); + File[] files = f.listFiles(XML_FILTER); + if (files != null) { + for (File entityFile : files) { + action.accept(entityFile.getAbsoluteFile(), entry.getValue()); + } + } + } + } + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-api/src/main/resources/cube-0.1.xsd ---------------------------------------------------------------------- diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd index 060eb43..1d8a624 100644 --- a/lens-api/src/main/resources/cube-0.1.xsd +++ b/lens-api/src/main/resources/cube-0.1.xsd @@ -390,7 +390,7 @@ </xs:documentation> </xs:annotation> <xs:sequence> - <xs:element type="x_expr_column" name="expression" maxOccurs="unbounded" minOccurs="1"/> + <xs:element type="x_expr_column" name="expression" maxOccurs="unbounded" minOccurs="0"/> </xs:sequence> </xs:complexType> @@ -707,7 +707,7 @@ <xs:complexType name="x_columns"> <xs:sequence> - <xs:element name="column" type="x_column" maxOccurs="unbounded" minOccurs="1"/> + <xs:element name="column" type="x_column" maxOccurs="unbounded" minOccurs="0"/> </xs:sequence> </xs:complexType> http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-api/src/main/resources/lens-errors.conf ---------------------------------------------------------------------- diff --git a/lens-api/src/main/resources/lens-errors.conf b/lens-api/src/main/resources/lens-errors.conf index 29e24cf..94505ef 100644 --- a/lens-api/src/main/resources/lens-errors.conf +++ b/lens-api/src/main/resources/lens-errors.conf @@ -326,6 +326,19 @@ lensCubeErrorsForQuery = [ errorMsg = "Could not find queried table or chain: %s" } + { + errorCode = 3034 + httpStatusCode = ${BAD_REQUEST} + errorMsg = "%s does not have any facts that can cover the requested time range : %s and queried measure set : %s" + } + + { + errorCode = 3035 + httpStatusCode = ${BAD_REQUEST} + errorMsg = "%s does not have any facts that can cover the queried measure set : %s" + } + + ] lensCubeErrorsForMetastore = [ http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java index feabf9c..befe4e6 100644 --- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java +++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java @@ -18,11 +18,21 @@ */ package org.apache.lens.cli.commands; -import java.io.*; +import java.io.File; +import java.io.FilenameFilter; import java.util.List; +import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; +import org.apache.lens.api.metastore.SchemaTraverser; +import org.apache.lens.api.metastore.XBaseCube; +import org.apache.lens.api.metastore.XDerivedCube; +import org.apache.lens.api.metastore.XDimension; +import org.apache.lens.api.metastore.XDimensionTable; +import org.apache.lens.api.metastore.XFactTable; +import org.apache.lens.api.metastore.XSegmentation; +import org.apache.lens.api.metastore.XStorage; import org.apache.lens.cli.commands.annotations.UserDocumentation; import org.springframework.beans.factory.annotation.Autowired; @@ -35,6 +45,7 @@ import org.springframework.stereotype.Component; import org.springframework.util.Assert; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; @Component @UserDocumentation(title = "Creating schema with one command", @@ -84,15 +95,52 @@ public class LensSchemaCommands implements CommandMarker { logger.setLevel(Level.FINE); } - private static final FilenameFilter XML_FILTER = new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return name.endsWith(".xml"); - } - }; + private static final FilenameFilter XML_FILTER = (dir, name) -> name.endsWith(".xml"); + private static final Map<Class<?>, String> CREATE_COMMAND_MAP = Maps.newHashMap(); + private static final Map<Class<?>, String> UPDATE_COMMAND_MAP = Maps.newHashMap(); + @Autowired private JLineShellComponent shell; + static { + CREATE_COMMAND_MAP.put(XStorage.class, "create storage --path %s"); + UPDATE_COMMAND_MAP.put(XStorage.class, "update storage --name %s --path %s"); + CREATE_COMMAND_MAP.put(XDimension.class, "create dimension --path %s"); + UPDATE_COMMAND_MAP.put(XDimension.class, "update dimension --name %s --path %s"); + CREATE_COMMAND_MAP.put(XBaseCube.class, "create cube --path %s"); + UPDATE_COMMAND_MAP.put(XBaseCube.class, "update cube --name %s --path %s"); + CREATE_COMMAND_MAP.put(XDerivedCube.class, "create cube --path %s"); + UPDATE_COMMAND_MAP.put(XDerivedCube.class, "update cube --name %s --path %s"); + CREATE_COMMAND_MAP.put(XDimensionTable.class, "create dimtable --path %s"); + UPDATE_COMMAND_MAP.put(XDimensionTable.class, "update dimtable --dimtable_name %s --path %s"); + CREATE_COMMAND_MAP.put(XDimensionTable.class, "create dimtable --path %s"); + UPDATE_COMMAND_MAP.put(XDimensionTable.class, "update dimtable --dimtable_name %s --path %s"); + CREATE_COMMAND_MAP.put(XFactTable.class, "create fact --path %s"); + UPDATE_COMMAND_MAP.put(XFactTable.class, "update fact --fact_name %s --path %s"); + CREATE_COMMAND_MAP.put(XSegmentation.class, "create segmentation --path %s"); + UPDATE_COMMAND_MAP.put(XSegmentation.class, "update segmentation --name %s --path %s"); + } + + private final SchemaTraverser.SchemaEntityProcessor processor = (entityFile, type) -> { + String entityName = entityFile.getName().substring(0, entityFile.getName().length() - 4); + String entityPath = entityFile.getAbsolutePath(); + String createCommand = String.format(CREATE_COMMAND_MAP.get(type), entityPath); + String entityType = createCommand.substring(8, createCommand.indexOf(" ", 9)); + logger.fine(createCommand); + if (shell.executeScriptLine(createCommand)) { + logger.info("Created " + entityType + " " + entityName); + } else { + logger.warning("Create failed, trying update"); + String updateCommand = String.format(UPDATE_COMMAND_MAP.get(type), entityName, entityPath); + logger.fine(updateCommand); + if (shell.executeScriptLine(updateCommand)) { + logger.info("Updated " + entityType + " " + entityName); + } else { + logger.severe("Couldn't create or update " + entityType + " " + entityName); + } + } + }; + @CliCommand(value = {"schema", "create schema"}, help = "Parses the specified resource file and executes commands for " + "creation/updation of schema\nExpected structure is " + STRUCTURE) @@ -108,55 +156,10 @@ public class LensSchemaCommands implements CommandMarker { // ignore result. it can fail if database already exists shell.executeCommand("create database " + database); if (shell.executeScriptLine("use " + database)) { - createOrUpdate(new File(schemaDirectory, "storages"), "storage", - "create storage --path %s", "update storage --name %s --path %s"); - createOrUpdate(new File(schemaDirectory, "dimensions"), "dimension", - "create dimension --path %s", "update dimension --name %s --path %s"); - createOrUpdate(new File(new File(schemaDirectory, "cubes"), "base"), "base cube", - "create cube --path %s", "update cube --name %s --path %s"); - createOrUpdate(new File(new File(schemaDirectory, "cubes"), "derived"), "derived cube", - "create cube --path %s", "update cube --name %s --path %s"); - createOrUpdate(new File(schemaDirectory, "dimensiontables"), "dimension table", - "create dimtable --path %s", "update dimtable --dimtable_name %s --path %s"); - createOrUpdate(new File(schemaDirectory, "dimtables"), "dimension table", - "create dimtable --path %s", "update dimtable --dimtable_name %s --path %s"); - createOrUpdate(new File(schemaDirectory, "facts"), "fact", - "create fact --path %s", "update fact --fact_name %s --path %s"); - createOrUpdate(new File(schemaDirectory, "segmentations"), "fact", - "create segmentation --path %s", "update segmentation --name %s --path %s"); + SchemaTraverser schemaTraverser = new SchemaTraverser(schemaDirectory, processor); + schemaTraverser.run(); } else { throw new IllegalStateException("Switching to database " + database + " failed"); } } - - public List<File> createOrUpdate(File parent, String entityType, String createSyntax, String updateSyntax) { - List<File> failedFiles = Lists.newArrayList(); - // Create/update entities - if (parent.exists()) { - Assert.isTrue(parent.isDirectory(), parent.toString() + " must be a directory"); - for (File entityFile : parent.listFiles(XML_FILTER)) { - String entityName = entityFile.getName().substring(0, entityFile.getName().length() - 4); - String entityPath = entityFile.getAbsolutePath(); - String createCommand = String.format(createSyntax, entityPath); - logger.fine(createCommand); - if (shell.executeScriptLine(createCommand)) { - logger.info("Created " + entityType + " " + entityName); - } else { - logger.warning("Create failed, trying update"); - String updateCommand = String.format(updateSyntax, entityName, entityPath); - logger.fine(updateCommand); - if (shell.executeScriptLine(updateCommand)) { - logger.info("Updated " + entityType + " " + entityName); - } else { - logger.severe("Couldn't create or update " + entityType + " " + entityName); - failedFiles.add(entityFile); - } - } - } - } - if (!failedFiles.isEmpty()) { - logger.severe("Failed for " + entityType + ": " + failedFiles); - } - return failedFiles; - } } http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java b/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java index 571b481..d98c4c5 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java @@ -57,6 +57,8 @@ public enum LensCubeErrorCode { STORAGE_UNION_DISABLED(3031, 1500), COULD_NOT_PARSE_EXPRESSION(3032, 1500), QUERIED_TABLE_NOT_FOUND(3033, 0), + NO_UNION_CANDIDATE_AVAILABLE(3034, 1501), + NO_JOIN_CANDIDATE_AVAILABLE(3035, 1502), // Error codes greater than 3100 are errors while doing a metastore operation. ERROR_IN_ENTITY_DEFINITION(3101, 100), TIMELINE_ABSENT(3102, 100), http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-cube/src/main/java/org/apache/lens/cube/error/NoCandidateFactAvailableException.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/error/NoCandidateFactAvailableException.java b/lens-cube/src/main/java/org/apache/lens/cube/error/NoCandidateFactAvailableException.java index b2568ff..21dda16 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/error/NoCandidateFactAvailableException.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/error/NoCandidateFactAvailableException.java @@ -18,18 +18,27 @@ */ package org.apache.lens.cube.error; -import org.apache.lens.cube.metadata.CubeFactTable; +import org.apache.lens.cube.parse.CubeQueryContext; import org.apache.lens.cube.parse.PruneCauses; +import org.apache.lens.cube.parse.StorageCandidate; import org.apache.lens.server.api.error.LensException; +import lombok.Getter; + +/** + * Note: This class is mainly meant for test cases to assert the detailed reasons (stored in + * {@link #briefAndDetailedError} leading to "No Candidate was found" + */ public class NoCandidateFactAvailableException extends LensException { - private final PruneCauses<CubeFactTable> briefAndDetailedError; + @Getter + private final PruneCauses<StorageCandidate> briefAndDetailedError; - public NoCandidateFactAvailableException(PruneCauses<CubeFactTable> briefAndDetailedError) { - super(LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo(), briefAndDetailedError.getBriefCause()); - this.briefAndDetailedError = briefAndDetailedError; + public NoCandidateFactAvailableException(CubeQueryContext cubeql) { + super(LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo(), + cubeql.getStoragePruningMsgs().getBriefCause()); + this.briefAndDetailedError = cubeql.getStoragePruningMsgs(); } public PruneCauses.BriefAndDetailedError getJsonMessage() { http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java index 896a7a1..e00122d 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java @@ -111,21 +111,16 @@ public class CubeFactTable extends AbstractCubeTable { private Map<String, Map<UpdatePeriod, String>> getUpdatePeriodMap(String factName, Map<String, String> props) { Map<String, Map<UpdatePeriod, String>> ret = new HashMap<>(); - for (Map.Entry entry : storageUpdatePeriods.entrySet()) { - String storage = (String) entry.getKey(); - for (UpdatePeriod period : (Set<UpdatePeriod>) entry.getValue()) { + for (Map.Entry<String, Set<UpdatePeriod>> entry : storageUpdatePeriods.entrySet()) { + String storage = entry.getKey(); + for (UpdatePeriod period : entry.getValue()) { String storagePrefixKey = MetastoreUtil .getUpdatePeriodStoragePrefixKey(factName.trim(), storage, period.getName()); String storageTableNamePrefix = props.get(storagePrefixKey); if (storageTableNamePrefix == null) { storageTableNamePrefix = storage; } - Map<UpdatePeriod, String> mapOfUpdatePeriods = ret.get(storage); - if (mapOfUpdatePeriods == null) { - mapOfUpdatePeriods = new HashMap<>(); - ret.put(storage, mapOfUpdatePeriods); - } - mapOfUpdatePeriods.put(period, storageTableNamePrefix); + ret.computeIfAbsent(storage, k -> new HashMap<>()).put(period, storageTableNamePrefix); } } return ret; http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 087c203..b5c4c89 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -20,12 +20,25 @@ package org.apache.lens.cube.metadata; import static org.apache.lens.cube.metadata.DateUtil.resolveDate; +import static org.apache.lens.cube.metadata.JAXBUtils.getStorageTableDescFromHiveTable; +import static org.apache.lens.cube.metadata.JAXBUtils.segmentationFromXSegmentation; import static org.apache.lens.cube.metadata.MetastoreUtil.*; import java.text.ParseException; import java.util.*; import java.util.concurrent.ConcurrentHashMap; +import org.apache.lens.api.metastore.XCube; +import org.apache.lens.api.metastore.XDerivedCube; +import org.apache.lens.api.metastore.XDimension; +import org.apache.lens.api.metastore.XDimensionTable; +import org.apache.lens.api.metastore.XFactTable; +import org.apache.lens.api.metastore.XSegmentation; +import org.apache.lens.api.metastore.XStorage; +import org.apache.lens.api.metastore.XStorageTableElement; +import org.apache.lens.api.metastore.XUpdatePeriod; +import org.apache.lens.api.metastore.XUpdatePeriodTableDescriptor; +import org.apache.lens.api.metastore.XUpdatePeriods; import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.cube.metadata.Storage.LatestInfo; import org.apache.lens.cube.metadata.Storage.LatestPartColumnInfo; @@ -50,6 +63,10 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.util.ReflectionUtils; import org.apache.thrift.TException; +import org.jvnet.jaxb2_commons.lang.Equals; +import org.jvnet.jaxb2_commons.lang.HashCode; +import org.jvnet.jaxb2_commons.lang.ToString; + import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -302,6 +319,62 @@ public class CubeMetastoreClient { } + public <T extends Equals & HashCode & ToString> void createEntity(T entity) throws LensException { + if (entity instanceof XStorage) { + createStorage((XStorage) entity); + } else if (entity instanceof XCube) { + createCube((XCube)entity); + } else if (entity instanceof XDimension) { + createDimension((XDimension) entity); + } else if (entity instanceof XFactTable) { + createCubeFactTable((XFactTable) entity); + } else if (entity instanceof XDimensionTable) { + createCubeDimensionTable((XDimensionTable) entity); + } else if (entity instanceof XSegmentation) { + createSegmentation((XSegmentation) entity); + } else { + throw new LensException("Unable to create entity " + entity + " as it's unrecognizable: "+ entity.getClass()); + } + } + + public <T extends Equals & HashCode & ToString> void updateEntity(String name, T entity) + throws LensException, HiveException { + if (entity instanceof XStorage) { + alterStorage((XStorage) entity); + } else if (entity instanceof XCube) { + alterCube((XCube)entity); + } else if (entity instanceof XDimension) { + alterDimension((XDimension) entity); + } else if (entity instanceof XFactTable) { + alterCubeFactTable((XFactTable) entity); + } else if (entity instanceof XDimensionTable) { + alterCubeDimensionTable((XDimensionTable) entity); + } else if (entity instanceof XSegmentation) { + alterSegmentation((XSegmentation) entity); + } else { + throw new LensException("Unable to alter entity " + entity + " as it's unrecognizable: " + entity.getClass()); + } + } + + + public static Map<String, String> addFactColStartTimePropertyToFactProperties(XFactTable fact) { + Map<String, String> props = new HashMap<String, String>(); + props.putAll(JAXBUtils.mapFromXProperties(fact.getProperties())); + props.putAll(JAXBUtils.columnStartAndEndTimeFromXColumns(fact.getColumns())); + return props; + } + public void createCubeFactTable(XFactTable fact) throws LensException { + createCubeFactTable(fact.getCubeName(), + fact.getName(), + JAXBUtils.fieldSchemaListFromColumns(fact.getColumns()), + JAXBUtils.getFactUpdatePeriodsFromStorageTables(fact.getStorageTables()), + fact.getWeight(), + addFactColStartTimePropertyToFactProperties(fact), + JAXBUtils.tableDescPrefixMapFromXStorageTables(fact.getStorageTables()), + JAXBUtils.storageTablePrefixMapOfStorage(fact.getStorageTables())); + } + + /** * In-memory storage of {@link PartitionTimeline} objects for each valid * storagetable-updateperiod-partitioncolumn tuple. also simultaneously stored in metastore table of the @@ -349,7 +422,7 @@ public class CubeMetastoreClient { public TreeMap<UpdatePeriod, CaseInsensitiveStringHashMap<PartitionTimeline>> get(String fact, String storage) throws HiveException, LensException { // SUSPEND CHECKSTYLE CHECK DoubleCheckedLockingCheck - // Unique key for the timeline cache, based on storageName and fact. + // Unique key for the timeline cache, based on storage and fact. String timeLineKey = (Storage.getPrefix(storage)+ fact).toLowerCase(); synchronized (this) { if (get(timeLineKey) == null) { @@ -478,17 +551,11 @@ public class CubeMetastoreClient { */ public PartitionTimeline ensureEntry(String timeLineKey, String storagTableName, UpdatePeriod updatePeriod, String partitionColumn) { - if (get(timeLineKey) == null) { - put(timeLineKey, new TreeMap<UpdatePeriod, CaseInsensitiveStringHashMap<PartitionTimeline>>()); - } - if (get(timeLineKey).get(updatePeriod) == null) { - get(timeLineKey).put(updatePeriod, new CaseInsensitiveStringHashMap<PartitionTimeline>()); - } - if (get(timeLineKey).get(updatePeriod).get(partitionColumn) == null) { - get(timeLineKey).get(updatePeriod).put(partitionColumn, PartitionTimelineFactory.get( - CubeMetastoreClient.this, storagTableName, updatePeriod, partitionColumn)); - } - return get(timeLineKey).get(updatePeriod).get(partitionColumn); + return this + .computeIfAbsent(timeLineKey, s -> new TreeMap<>()) + .computeIfAbsent(updatePeriod, k -> new CaseInsensitiveStringHashMap<>()) + .computeIfAbsent(partitionColumn, c -> PartitionTimelineFactory.get( + CubeMetastoreClient.this, storagTableName, updatePeriod, c)); } /** check partition existence in the appropriate timeline if it exists */ @@ -625,12 +692,22 @@ public class CubeMetastoreClient { } } + public void createStorage(XStorage storage) throws LensException { + createStorage(JAXBUtils.storageFromXStorage(storage)); + } + public void createStorage(Storage storage) throws LensException { createCubeHiveTable(storage); // do a get to update cache getStorage(storage.getName()); } + public void createCube(XCube cube) throws LensException { + Cube parent = cube instanceof XDerivedCube ? (Cube) getCube( + ((XDerivedCube) cube).getParent()) : null; + createCube(JAXBUtils.hiveCubeFromXCube(cube, parent)); + } + /** * Create cube in metastore defined by {@link Cube} or {@link DerivedCube} object * @@ -720,6 +797,9 @@ public class CubeMetastoreClient { createDimension(dim); } + public void createDimension(XDimension dim) throws LensException { + createDimension(JAXBUtils.dimensionFromXDimension(dim)); + } /** * Create dimension in metastore defined by {@link Dimension} object * @@ -789,6 +869,18 @@ public class CubeMetastoreClient { getSegmentation(segmentationName); } + public void createCubeDimensionTable(XDimensionTable xDimTable) throws LensException { + List<FieldSchema> columns = JAXBUtils.fieldSchemaListFromColumns(xDimTable.getColumns()); + Map<String, UpdatePeriod> updatePeriodMap = + JAXBUtils.dumpPeriodsFromStorageTables(xDimTable.getStorageTables()); + + Map<String, String> properties = JAXBUtils.mapFromXProperties(xDimTable.getProperties()); + Map<String, StorageTableDesc> storageDesc = JAXBUtils.tableDescPrefixMapFromXStorageTables( + xDimTable.getStorageTables()); + log.info("# Columns: " + columns); + createCubeDimensionTable(xDimTable.getDimensionName(), xDimTable.getTableName(), columns, xDimTable.getWeight(), + updatePeriodMap, properties, storageDesc); + } /** * Create a cube dimension table * @@ -852,6 +944,14 @@ public class CubeMetastoreClient { } } + public void createSegmentation(XSegmentation cubeSeg) throws LensException { + createSegmentation( + cubeSeg.getCubeName(), + cubeSeg.getName(), + JAXBUtils.segmentsFromXSegments(cubeSeg.getSegements()), + cubeSeg.getWeight(), + JAXBUtils.mapFromXProperties(cubeSeg.getProperties())); + } public void createSegmentation(Segmentation cubeSeg) throws LensException { // create virtual cube table in metastore @@ -979,14 +1079,14 @@ public class CubeMetastoreClient { } } - private Date getStorageTableStartDate(String storageTable, String factTableName) + public Date getStorageTableStartDate(String storageTable, String factTableName) throws LensException { List<Date> startDates = getStorageTimes(storageTable, MetastoreUtil.getStoragetableStartTimesKey()); startDates.add(getFactTable(factTableName).getStartTime()); return Collections.max(startDates); } - private Date getStorageTableEndDate(String storageTable, String factTableName) + public Date getStorageTableEndDate(String storageTable, String factTableName) throws LensException { List<Date> endDates = getStorageTimes(storageTable, MetastoreUtil.getStoragetableEndTimesKey()); endDates.add(getFactTable(factTableName).getEndTime()); @@ -1624,6 +1724,48 @@ public class CubeMetastoreClient { return CubeTableType.DIMENSION.name().equals(tableType); } + public XFactTable getXFactTable(String tableName) throws LensException { + return getXFactTable(getFactTable(tableName)); + } + public XFactTable getXFactTable(CubeFactTable cft) throws LensException { + + XFactTable factTable = JAXBUtils.factTableFromCubeFactTable(cft); + Map<String, Map<UpdatePeriod, String>> storageMap = cft.getStoragePrefixUpdatePeriodMap(); + for (String storageName : cft.getStorages()) { + Set<UpdatePeriod> updatePeriods = cft.getUpdatePeriods().get(storageName); + // This map tells if there are different tables for different update period. + Map<UpdatePeriod, String> updatePeriodToTableMap = storageMap.get(storageName); + Set<String> tableNames = new HashSet<>(); + for (UpdatePeriod updatePeriod : updatePeriods) { + tableNames.add(updatePeriodToTableMap.get(updatePeriod)); + } + if (tableNames.size() <= 1) { + XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( + getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(), storageName))); + tblElement.setStorageName(storageName); + for (UpdatePeriod p : updatePeriods) { + tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); + } + factTable.getStorageTables().getStorageTable().add(tblElement); + } else { + // Multiple storage tables. + XStorageTableElement tblElement = new XStorageTableElement(); + tblElement.setStorageName(storageName); + XUpdatePeriods xUpdatePeriods = new XUpdatePeriods(); + tblElement.setUpdatePeriods(xUpdatePeriods); + for (Map.Entry entry : updatePeriodToTableMap.entrySet()) { + XUpdatePeriodTableDescriptor updatePeriodTableDescriptor = new XUpdatePeriodTableDescriptor(); + updatePeriodTableDescriptor.setTableDesc(getStorageTableDescFromHiveTable( + this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(), + (String) entry.getValue())))); + updatePeriodTableDescriptor.setUpdatePeriod(XUpdatePeriod.valueOf(((UpdatePeriod)entry.getKey()).name())); + xUpdatePeriods.getUpdatePeriodTableDescriptor().add(updatePeriodTableDescriptor); + } + factTable.getStorageTables().getStorageTable().add(tblElement); + } + } + return factTable; + } /** * Get {@link CubeFactTable} object corresponding to the name * @@ -1640,6 +1782,25 @@ public class CubeMetastoreClient { return new Segmentation(getTableWithTypeFailFast(tableName, CubeTableType.SEGMENTATION)); } + public XDimensionTable getXDimensionTable(String dimTable) throws LensException { + return getXDimensionTable(getDimensionTable(dimTable)); + } + public XDimensionTable getXDimensionTable(CubeDimensionTable dimTable) throws LensException { + XDimensionTable dt = JAXBUtils.dimTableFromCubeDimTable(dimTable); + if (!dimTable.getStorages().isEmpty()) { + for (String storageName : dimTable.getStorages()) { + XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( + this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(dimTable.getName(), storageName))); + tblElement.setStorageName(storageName); + UpdatePeriod p = dimTable.getSnapshotDumpPeriods().get(storageName); + if (p != null) { + tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); + } + dt.getStorageTables().getStorageTable().add(tblElement); + } + } + return dt; + } /** * Get {@link CubeDimensionTable} object corresponding to the name * @@ -2101,17 +2262,37 @@ public class CubeMetastoreClient { return dimTables; } - public boolean partColExists(String tableName, String partCol) throws LensException { - Table tbl = getTable(tableName); - for (FieldSchema f : tbl.getPartCols()) { - if (f.getName().equalsIgnoreCase(partCol)) { - return true; + public boolean partColExists(String fact, String storage, String partCol) throws LensException { + for (String storageTable : getStorageTables(fact, storage)) { + for (FieldSchema f : getTable(storageTable).getPartCols()) { + if (f.getName().equalsIgnoreCase(partCol)) { + return true; + } } } return false; } /** + * Returns storage table names for a storage. + * Note: If each update period in the storage has a different storage table, this method will return N Storage Tables + * where N is the number of update periods in the storage (LENS-1386) + * + * @param fact + * @param storage + * @return + * @throws LensException + */ + public Set<String> getStorageTables(String fact, String storage) throws LensException { + Set<String> uniqueStorageTables = new HashSet<>(); + for (UpdatePeriod updatePeriod : getFactTable(fact).getUpdatePeriods().get(storage)) { + uniqueStorageTables.add(getStorageTableName(fact, storage, updatePeriod)); + } + return uniqueStorageTables; + } + + + /** * * @param table table name * @param hiveTable hive table @@ -2150,6 +2331,11 @@ public class CubeMetastoreClient { } } + public void alterCube(XCube cube) throws HiveException, LensException { + Cube parent = cube instanceof XDerivedCube ? (Cube) getCube( + ((XDerivedCube) cube).getParent()) : null; + alterCube(cube.getName(), JAXBUtils.hiveCubeFromXCube(cube, parent)); + } /** * Alter cube specified by the name to new definition * @@ -2168,10 +2354,13 @@ public class CubeMetastoreClient { /** * Alter dimension specified by the dimension name to new definition * - * @param dimName The cube name to be altered * @param newDim The new dimension definition * @throws HiveException */ + public void alterDimension(XDimension newDim) throws HiveException, LensException { + alterDimension(newDim.getName(), JAXBUtils.dimensionFromXDimension(newDim)); + } + public void alterDimension(String dimName, Dimension newDim) throws HiveException, LensException { Table tbl = getTableWithTypeFailFast(dimName, CubeTableType.DIMENSION); alterCubeTable(dimName, tbl, newDim); @@ -2183,10 +2372,12 @@ public class CubeMetastoreClient { /** * Alter storage specified by the name to new definition * - * @param storageName The storage name to be altered * @param storage The new storage definition * @throws LensException */ + public void alterStorage(XStorage storage) throws LensException, HiveException { + alterStorage(storage.getName(), JAXBUtils.storageFromXStorage(storage)); + } public void alterStorage(String storageName, Storage storage) throws LensException, HiveException { Table storageTbl = getTableWithTypeFailFast(storageName, CubeTableType.STORAGE); alterCubeTable(storageName, storageTbl, storage); @@ -2339,7 +2530,11 @@ public class CubeMetastoreClient { dropHiveTable(dimTblName); allDimTables.remove(dimTblName.trim().toLowerCase()); } - + public void alterCubeFactTable(XFactTable fact) throws LensException, HiveException { + alterCubeFactTable(fact.getName(), JAXBUtils.cubeFactFromFactTable(fact), + JAXBUtils.tableDescPrefixMapFromXStorageTables(fact.getStorageTables()), + JAXBUtils.columnStartAndEndTimeFromXColumns(fact.getColumns())); + } /** * Alter a cubefact with new definition and alter underlying storage tables as well. * @@ -2367,6 +2562,9 @@ public class CubeMetastoreClient { updateFactCache(factTableName); } + public void alterSegmentation(XSegmentation cubeSeg) throws LensException, HiveException { + alterSegmentation(cubeSeg.getName(), segmentationFromXSegmentation(cubeSeg)); + } public void alterSegmentation(String segName, Segmentation seg) throws HiveException, LensException { getTableWithTypeFailFast(segName, CubeTableType.SEGMENTATION); @@ -2394,7 +2592,11 @@ public class CubeMetastoreClient { allDimTables.put(dimTblName.trim().toLowerCase(), getDimensionTable(refreshTable(dimTblName))); } } - + public void alterCubeDimensionTable(XDimensionTable dimensionTable) throws LensException, HiveException { + alterCubeDimensionTable(dimensionTable.getTableName(), + JAXBUtils.cubeDimTableFromDimTable(dimensionTable), + JAXBUtils.tableDescPrefixMapFromXStorageTables(dimensionTable.getStorageTables())); + } /** * Alter dimension table with new dimension definition and underlying storage tables as well * http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java index 7717081..99ad233 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java @@ -30,6 +30,7 @@ import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Stream; import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.server.api.error.LensException; @@ -305,11 +306,11 @@ public final class DateUtil { switch (interval) { case SECONDLY: case CONTINUOUS: - return getMilliSecondCoveringInfo(from, to, 1000); + return getMilliSecondCoveringInfo(from, to, 1000, interval); case MINUTELY: case HOURLY: case DAILY: - return getMilliSecondCoveringInfo(from, to, interval.weight()); + return getMilliSecondCoveringInfo(from, to, interval.weight(), interval); case WEEKLY: return getWeeklyCoveringInfo(from, to); case MONTHLY: @@ -323,18 +324,26 @@ public final class DateUtil { } } - private static CoveringInfo getMilliSecondCoveringInfo(Date from, Date to, long millisInInterval) { + private static CoveringInfo getMilliSecondCoveringInfo(Date from, Date to, long millisInInterval, + UpdatePeriod interval) { long diff = to.getTime() - from.getTime(); - return new CoveringInfo((int) (diff / millisInInterval), diff % millisInInterval == 0); + return new CoveringInfo((int) (diff / millisInInterval), + Stream.of(from, to).allMatch(a->interval.truncate(a).equals(a))); + // start date and end date should lie on boundaries. } + /** + * Whether the range [from,to) is coverable by intervals + * @param from from time + * @param to to time + * @param intervals intervals to check + * @return true if any of the intervals can completely cover the range + */ static boolean isCoverableBy(Date from, Date to, Set<UpdatePeriod> intervals) { - for (UpdatePeriod period : intervals) { - if (getCoveringInfo(from, to, period).isCoverable()) { - return true; - } - } - return false; + return intervals.stream().anyMatch(period->isCoverableBy(from, to, period)); + } + private static boolean isCoverableBy(Date from, Date to, UpdatePeriod period) { + return getCoveringInfo(from, to, period).isCoverable(); } public static int getTimeDiff(Date fromDate, Date toDate, UpdatePeriod updatePeriod) { http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java index 1694b80..ed940cc 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java @@ -65,6 +65,16 @@ public class FactPartition implements Comparable<FactPartition> { } } + /** + * Partition should not be used a indicative of the class itself. + * New Fact partition created includes more final partitions with that creation. + * @return + */ + + public FactPartition withoutContaining() { + return new FactPartition(this.getPartCol(), this.getPartSpec(), this.getPeriod(), null, this + .getPartFormat(), this.getStorageTables()); + } public FactPartition(String partCol, TimePartition timePartition) { this(partCol, timePartition, null, null); }
