LENS-1400: Convert CubeTestSetup to setup using xml files instead of code
Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/112af59c Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/112af59c Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/112af59c Branch: refs/heads/lens-1381 Commit: 112af59cc4b82402a3df2c7b6687d2ee328e96b4 Parents: de464fa Author: Rajat Khandelwal <[email protected]> Authored: Mon Mar 27 14:36:58 2017 +0530 Committer: Rajat Khandelwal <[email protected]> Committed: Mon Mar 27 14:36:58 2017 +0530 ---------------------------------------------------------------------- .../java/org/apache/lens/api/ToXMLString.java | 9 + .../apache/lens/api/jaxb/LensJAXBContext.java | 14 +- .../lens/api/metastore/SchemaTraverser.java | 58 + lens-api/src/main/resources/cube-0.1.xsd | 4 +- .../lens/cli/commands/LensSchemaCommands.java | 111 +- .../lens/cube/metadata/CubeFactTable.java | 13 +- .../lens/cube/metadata/CubeMetastoreClient.java | 195 +- .../apache/lens/cube/metadata/JAXBUtils.java | 1114 ++++++++ .../lens/cube/parse/StorageCandidate.java | 8 +- .../lens/cube/parse/StorageTableResolver.java | 2 +- .../apache/lens/cube/parse/CubeTestSetup.java | 2630 +----------------- .../resources/schema/cubes/base/basecube.xml | 952 +++++++ .../resources/schema/cubes/base/testcube.xml | 640 +++++ .../resources/schema/cubes/derived/der1.xml | 22 + .../resources/schema/cubes/derived/der2.xml | 43 + .../resources/schema/cubes/derived/der3.xml | 32 + .../schema/cubes/derived/derivedcube.xml | 24 + .../cubes/derived/union_join_ctx_der1.xml | 25 + .../resources/schema/dimensions/citydim.xml | 102 + .../resources/schema/dimensions/countrydim.xml | 21 + .../resources/schema/dimensions/cycledim1.xml | 51 + .../resources/schema/dimensions/cycledim2.xml | 50 + .../test/resources/schema/dimensions/daydim.xml | 41 + .../resources/schema/dimensions/hourdim.xml | 25 + .../test/resources/schema/dimensions/sports.xml | 25 + .../resources/schema/dimensions/statedim.xml | 53 + .../resources/schema/dimensions/testdim2.xml | 203 ++ .../resources/schema/dimensions/testdim3.xml | 48 + .../resources/schema/dimensions/testdim4.xml | 25 + .../schema/dimensions/unreachabledim.xml | 24 + .../schema/dimensions/user_interests.xml | 30 + .../resources/schema/dimensions/userdim.xml | 58 + .../test/resources/schema/dimensions/zipdim.xml | 30 + .../resources/schema/dimtables/citytable.xml | 51 + .../resources/schema/dimtables/citytable2.xml | 29 + .../resources/schema/dimtables/citytable3.xml | 29 + .../resources/schema/dimtables/citytable4.xml | 28 + .../resources/schema/dimtables/countrytable.xml | 32 + .../dimtables/countrytable_partitioned.xml | 35 + .../resources/schema/dimtables/cycledim1tbl.xml | 49 + .../resources/schema/dimtables/cycledim2tbl.xml | 49 + .../resources/schema/dimtables/daydimtbl.xml | 48 + .../resources/schema/dimtables/hourdimtbl.xml | 48 + .../resources/schema/dimtables/sports_tbl.xml | 48 + .../resources/schema/dimtables/statetable.xml | 36 + .../schema/dimtables/statetable_partitioned.xml | 37 + .../resources/schema/dimtables/testdim2tbl.xml | 50 + .../resources/schema/dimtables/testdim2tbl2.xml | 69 + .../resources/schema/dimtables/testdim2tbl3.xml | 69 + .../resources/schema/dimtables/testdim3tbl.xml | 49 + .../resources/schema/dimtables/testdim4tbl.xml | 48 + .../schema/dimtables/unreachabledimtable.xml | 35 + .../schema/dimtables/user_interests_tbl.xml | 50 + .../resources/schema/dimtables/usertable.xml | 52 + .../resources/schema/dimtables/ziptable.xml | 35 + .../test/resources/schema/facts/cheapfact.xml | 81 + .../test/resources/schema/facts/summary1.xml | 81 + .../test/resources/schema/facts/summary2.xml | 83 + .../test/resources/schema/facts/summary3.xml | 84 + .../test/resources/schema/facts/summary4.xml | 65 + .../test/resources/schema/facts/testfact.xml | 192 ++ .../resources/schema/facts/testfact1_base.xml | 135 + .../schema/facts/testfact1_raw_base.xml | 75 + .../test/resources/schema/facts/testfact2.xml | 75 + .../resources/schema/facts/testfact2_base.xml | 120 + .../resources/schema/facts/testfact2_raw.xml | 75 + .../schema/facts/testfact2_raw_base.xml | 46 + .../resources/schema/facts/testfact3_base.xml | 117 + .../schema/facts/testfact3_raw_base.xml | 44 + .../schema/facts/testfact4_raw_base.xml | 45 + .../resources/schema/facts/testfact5_base.xml | 128 + .../schema/facts/testfact5_raw_base.xml | 38 + .../resources/schema/facts/testfact6_base.xml | 117 + .../schema/facts/testfact_continuous.xml | 38 + .../schema/facts/testfact_deprecated.xml | 126 + .../resources/schema/facts/testfactmonthly.xml | 46 + .../schema/facts/union_join_ctx_fact1.xml | 39 + .../schema/facts/union_join_ctx_fact2.xml | 38 + .../schema/facts/union_join_ctx_fact3.xml | 38 + .../schema/facts/union_join_ctx_fact5.xml | 38 + .../schema/facts/union_join_ctx_fact6.xml | 38 + .../resources/schema/segmentations/seg1.xml | 45 + .../src/test/resources/schema/storages/c0.xml | 6 + .../src/test/resources/schema/storages/c1.xml | 6 + .../src/test/resources/schema/storages/c2.xml | 6 + .../src/test/resources/schema/storages/c3.xml | 6 + .../src/test/resources/schema/storages/c4.xml | 6 + .../src/test/resources/schema/storages/c5.xml | 6 + .../src/test/resources/schema/storages/c99.xml | 6 + .../metastore/CubeMetastoreServiceImpl.java | 139 +- .../apache/lens/server/metastore/JAXBUtils.java | 1116 -------- 91 files changed, 7176 insertions(+), 3826 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-api/src/main/java/org/apache/lens/api/ToXMLString.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/ToXMLString.java b/lens-api/src/main/java/org/apache/lens/api/ToXMLString.java index e74adc9..746a82b 100644 --- a/lens-api/src/main/java/org/apache/lens/api/ToXMLString.java +++ b/lens-api/src/main/java/org/apache/lens/api/ToXMLString.java @@ -24,6 +24,10 @@ import java.util.HashMap; import java.util.Map; import javax.xml.bind.*; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlSeeAlso; +import javax.xml.bind.annotation.XmlType; +import javax.xml.namespace.QName; import org.apache.lens.api.jaxb.LensJAXBContext; @@ -31,6 +35,11 @@ public abstract class ToXMLString { protected static final Map<Class<?>, JAXBContext> JAXB_CONTEXTS = new HashMap<>(); public static String toString(Object o) { + if (!(o instanceof JAXBElement) && o.getClass().getAnnotation(XmlRootElement.class) == null + && o.getClass().getAnnotation(XmlType.class)!= null) { + o = new JAXBElement(new QName("uri:lens:cube:0.1", o.getClass().getAnnotation(XmlType.class).name()), + o.getClass(), null, o); + } try { StringWriter stringWriter = new StringWriter(); Class cl = null; http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-api/src/main/java/org/apache/lens/api/jaxb/LensJAXBContext.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/jaxb/LensJAXBContext.java b/lens-api/src/main/java/org/apache/lens/api/jaxb/LensJAXBContext.java index 14fc4aa..8858b95 100644 --- a/lens-api/src/main/java/org/apache/lens/api/jaxb/LensJAXBContext.java +++ b/lens-api/src/main/java/org/apache/lens/api/jaxb/LensJAXBContext.java @@ -24,6 +24,7 @@ package org.apache.lens.api.jaxb; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.Reader; import javax.xml.XMLConstants; import javax.xml.bind.*; @@ -114,17 +115,26 @@ public class LensJAXBContext extends JAXBContext { return UNMARSHALLER; } + public static <T> T unmarshall(File file) throws JAXBException, IOException { + return ((JAXBElement<T>) UNMARSHALLER.unmarshal(file)).getValue(); + } + public static <T> T unmarshall(InputStream inputStream) throws JAXBException, IOException { + return ((JAXBElement<T>) UNMARSHALLER.unmarshal(inputStream)).getValue(); + } + public static <T> T unmarshall(Reader reader) throws JAXBException, IOException { + return ((JAXBElement<T>) UNMARSHALLER.unmarshal(reader)).getValue(); + } public static <T> T unmarshallFromFile(String filename) throws JAXBException, IOException { File file = new File(filename); if (file.exists()) { - return ((JAXBElement<T>) UNMARSHALLER.unmarshal(file)).getValue(); + return unmarshall(file); } else { // load from classpath InputStream stream = Thread.currentThread().getContextClassLoader().getResourceAsStream(filename); if (stream == null) { throw new IOException("File not found:" + filename); } - return ((JAXBElement<T>) UNMARSHALLER.unmarshal(stream)).getValue(); + return unmarshall(stream); } } } http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java b/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java new file mode 100644 index 0000000..157ad71 --- /dev/null +++ b/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java @@ -0,0 +1,58 @@ +package org.apache.lens.api.metastore; + +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.util.Map; +import java.util.function.BiConsumer; + +import javax.xml.bind.JAXBException; + +import org.apache.lens.api.jaxb.LensJAXBContext; + +import com.google.common.collect.Maps; + +/* + * Created on 07/03/17. + */ +public class SchemaTraverser implements Runnable { + final File parent; + final Map<String, Class<?>> types = Maps.newLinkedHashMap(); + private final SchemaEntityProcessor action; + { + types.put("storages", XStorage.class); + types.put("cubes/base", XBaseCube.class); + types.put("cubes/derived", XDerivedCube.class); + types.put("dimensions", XDimension.class); + types.put("facts", XFactTable.class); + types.put("dimtables", XDimensionTable.class); + types.put("dimensiontables", XDimensionTable.class); + types.put("dimensiontables", XDimensionTable.class); + types.put("segmentations", XSegmentation.class); + } + private static final FilenameFilter XML_FILTER = (dir, name) -> name.endsWith(".xml"); + + public interface SchemaEntityProcessor extends BiConsumer<File, Class<?>> { + } + + public SchemaTraverser(File parent, SchemaEntityProcessor action) { + this.parent = parent; + this.action = action; + } + + @Override + public void run() { + for (Map.Entry<String, Class<?>> entry : types.entrySet()) { + File f = new File(parent, entry.getKey()); + if (f.exists()) { + assert f.isDirectory(); + File[] files = f.listFiles(XML_FILTER); + if (files != null) { + for (File entityFile : files) { + action.accept(entityFile.getAbsoluteFile(), entry.getValue()); + } + } + } + } + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-api/src/main/resources/cube-0.1.xsd ---------------------------------------------------------------------- diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd index 060eb43..1d8a624 100644 --- a/lens-api/src/main/resources/cube-0.1.xsd +++ b/lens-api/src/main/resources/cube-0.1.xsd @@ -390,7 +390,7 @@ </xs:documentation> </xs:annotation> <xs:sequence> - <xs:element type="x_expr_column" name="expression" maxOccurs="unbounded" minOccurs="1"/> + <xs:element type="x_expr_column" name="expression" maxOccurs="unbounded" minOccurs="0"/> </xs:sequence> </xs:complexType> @@ -707,7 +707,7 @@ <xs:complexType name="x_columns"> <xs:sequence> - <xs:element name="column" type="x_column" maxOccurs="unbounded" minOccurs="1"/> + <xs:element name="column" type="x_column" maxOccurs="unbounded" minOccurs="0"/> </xs:sequence> </xs:complexType> http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java index feabf9c..befe4e6 100644 --- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java +++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java @@ -18,11 +18,21 @@ */ package org.apache.lens.cli.commands; -import java.io.*; +import java.io.File; +import java.io.FilenameFilter; import java.util.List; +import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; +import org.apache.lens.api.metastore.SchemaTraverser; +import org.apache.lens.api.metastore.XBaseCube; +import org.apache.lens.api.metastore.XDerivedCube; +import org.apache.lens.api.metastore.XDimension; +import org.apache.lens.api.metastore.XDimensionTable; +import org.apache.lens.api.metastore.XFactTable; +import org.apache.lens.api.metastore.XSegmentation; +import org.apache.lens.api.metastore.XStorage; import org.apache.lens.cli.commands.annotations.UserDocumentation; import org.springframework.beans.factory.annotation.Autowired; @@ -35,6 +45,7 @@ import org.springframework.stereotype.Component; import org.springframework.util.Assert; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; @Component @UserDocumentation(title = "Creating schema with one command", @@ -84,15 +95,52 @@ public class LensSchemaCommands implements CommandMarker { logger.setLevel(Level.FINE); } - private static final FilenameFilter XML_FILTER = new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return name.endsWith(".xml"); - } - }; + private static final FilenameFilter XML_FILTER = (dir, name) -> name.endsWith(".xml"); + private static final Map<Class<?>, String> CREATE_COMMAND_MAP = Maps.newHashMap(); + private static final Map<Class<?>, String> UPDATE_COMMAND_MAP = Maps.newHashMap(); + @Autowired private JLineShellComponent shell; + static { + CREATE_COMMAND_MAP.put(XStorage.class, "create storage --path %s"); + UPDATE_COMMAND_MAP.put(XStorage.class, "update storage --name %s --path %s"); + CREATE_COMMAND_MAP.put(XDimension.class, "create dimension --path %s"); + UPDATE_COMMAND_MAP.put(XDimension.class, "update dimension --name %s --path %s"); + CREATE_COMMAND_MAP.put(XBaseCube.class, "create cube --path %s"); + UPDATE_COMMAND_MAP.put(XBaseCube.class, "update cube --name %s --path %s"); + CREATE_COMMAND_MAP.put(XDerivedCube.class, "create cube --path %s"); + UPDATE_COMMAND_MAP.put(XDerivedCube.class, "update cube --name %s --path %s"); + CREATE_COMMAND_MAP.put(XDimensionTable.class, "create dimtable --path %s"); + UPDATE_COMMAND_MAP.put(XDimensionTable.class, "update dimtable --dimtable_name %s --path %s"); + CREATE_COMMAND_MAP.put(XDimensionTable.class, "create dimtable --path %s"); + UPDATE_COMMAND_MAP.put(XDimensionTable.class, "update dimtable --dimtable_name %s --path %s"); + CREATE_COMMAND_MAP.put(XFactTable.class, "create fact --path %s"); + UPDATE_COMMAND_MAP.put(XFactTable.class, "update fact --fact_name %s --path %s"); + CREATE_COMMAND_MAP.put(XSegmentation.class, "create segmentation --path %s"); + UPDATE_COMMAND_MAP.put(XSegmentation.class, "update segmentation --name %s --path %s"); + } + + private final SchemaTraverser.SchemaEntityProcessor processor = (entityFile, type) -> { + String entityName = entityFile.getName().substring(0, entityFile.getName().length() - 4); + String entityPath = entityFile.getAbsolutePath(); + String createCommand = String.format(CREATE_COMMAND_MAP.get(type), entityPath); + String entityType = createCommand.substring(8, createCommand.indexOf(" ", 9)); + logger.fine(createCommand); + if (shell.executeScriptLine(createCommand)) { + logger.info("Created " + entityType + " " + entityName); + } else { + logger.warning("Create failed, trying update"); + String updateCommand = String.format(UPDATE_COMMAND_MAP.get(type), entityName, entityPath); + logger.fine(updateCommand); + if (shell.executeScriptLine(updateCommand)) { + logger.info("Updated " + entityType + " " + entityName); + } else { + logger.severe("Couldn't create or update " + entityType + " " + entityName); + } + } + }; + @CliCommand(value = {"schema", "create schema"}, help = "Parses the specified resource file and executes commands for " + "creation/updation of schema\nExpected structure is " + STRUCTURE) @@ -108,55 +156,10 @@ public class LensSchemaCommands implements CommandMarker { // ignore result. it can fail if database already exists shell.executeCommand("create database " + database); if (shell.executeScriptLine("use " + database)) { - createOrUpdate(new File(schemaDirectory, "storages"), "storage", - "create storage --path %s", "update storage --name %s --path %s"); - createOrUpdate(new File(schemaDirectory, "dimensions"), "dimension", - "create dimension --path %s", "update dimension --name %s --path %s"); - createOrUpdate(new File(new File(schemaDirectory, "cubes"), "base"), "base cube", - "create cube --path %s", "update cube --name %s --path %s"); - createOrUpdate(new File(new File(schemaDirectory, "cubes"), "derived"), "derived cube", - "create cube --path %s", "update cube --name %s --path %s"); - createOrUpdate(new File(schemaDirectory, "dimensiontables"), "dimension table", - "create dimtable --path %s", "update dimtable --dimtable_name %s --path %s"); - createOrUpdate(new File(schemaDirectory, "dimtables"), "dimension table", - "create dimtable --path %s", "update dimtable --dimtable_name %s --path %s"); - createOrUpdate(new File(schemaDirectory, "facts"), "fact", - "create fact --path %s", "update fact --fact_name %s --path %s"); - createOrUpdate(new File(schemaDirectory, "segmentations"), "fact", - "create segmentation --path %s", "update segmentation --name %s --path %s"); + SchemaTraverser schemaTraverser = new SchemaTraverser(schemaDirectory, processor); + schemaTraverser.run(); } else { throw new IllegalStateException("Switching to database " + database + " failed"); } } - - public List<File> createOrUpdate(File parent, String entityType, String createSyntax, String updateSyntax) { - List<File> failedFiles = Lists.newArrayList(); - // Create/update entities - if (parent.exists()) { - Assert.isTrue(parent.isDirectory(), parent.toString() + " must be a directory"); - for (File entityFile : parent.listFiles(XML_FILTER)) { - String entityName = entityFile.getName().substring(0, entityFile.getName().length() - 4); - String entityPath = entityFile.getAbsolutePath(); - String createCommand = String.format(createSyntax, entityPath); - logger.fine(createCommand); - if (shell.executeScriptLine(createCommand)) { - logger.info("Created " + entityType + " " + entityName); - } else { - logger.warning("Create failed, trying update"); - String updateCommand = String.format(updateSyntax, entityName, entityPath); - logger.fine(updateCommand); - if (shell.executeScriptLine(updateCommand)) { - logger.info("Updated " + entityType + " " + entityName); - } else { - logger.severe("Couldn't create or update " + entityType + " " + entityName); - failedFiles.add(entityFile); - } - } - } - } - if (!failedFiles.isEmpty()) { - logger.severe("Failed for " + entityType + ": " + failedFiles); - } - return failedFiles; - } } http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java index 896a7a1..e00122d 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java @@ -111,21 +111,16 @@ public class CubeFactTable extends AbstractCubeTable { private Map<String, Map<UpdatePeriod, String>> getUpdatePeriodMap(String factName, Map<String, String> props) { Map<String, Map<UpdatePeriod, String>> ret = new HashMap<>(); - for (Map.Entry entry : storageUpdatePeriods.entrySet()) { - String storage = (String) entry.getKey(); - for (UpdatePeriod period : (Set<UpdatePeriod>) entry.getValue()) { + for (Map.Entry<String, Set<UpdatePeriod>> entry : storageUpdatePeriods.entrySet()) { + String storage = entry.getKey(); + for (UpdatePeriod period : entry.getValue()) { String storagePrefixKey = MetastoreUtil .getUpdatePeriodStoragePrefixKey(factName.trim(), storage, period.getName()); String storageTableNamePrefix = props.get(storagePrefixKey); if (storageTableNamePrefix == null) { storageTableNamePrefix = storage; } - Map<UpdatePeriod, String> mapOfUpdatePeriods = ret.get(storage); - if (mapOfUpdatePeriods == null) { - mapOfUpdatePeriods = new HashMap<>(); - ret.put(storage, mapOfUpdatePeriods); - } - mapOfUpdatePeriods.put(period, storageTableNamePrefix); + ret.computeIfAbsent(storage, k -> new HashMap<>()).put(period, storageTableNamePrefix); } } return ret; http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 78fb6d3..c8a2498 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -20,12 +20,25 @@ package org.apache.lens.cube.metadata; import static org.apache.lens.cube.metadata.DateUtil.resolveDate; +import static org.apache.lens.cube.metadata.JAXBUtils.getStorageTableDescFromHiveTable; +import static org.apache.lens.cube.metadata.JAXBUtils.segmentationFromXSegmentation; import static org.apache.lens.cube.metadata.MetastoreUtil.*; import java.text.ParseException; import java.util.*; import java.util.concurrent.ConcurrentHashMap; +import org.apache.lens.api.metastore.XCube; +import org.apache.lens.api.metastore.XDerivedCube; +import org.apache.lens.api.metastore.XDimension; +import org.apache.lens.api.metastore.XDimensionTable; +import org.apache.lens.api.metastore.XFactTable; +import org.apache.lens.api.metastore.XSegmentation; +import org.apache.lens.api.metastore.XStorage; +import org.apache.lens.api.metastore.XStorageTableElement; +import org.apache.lens.api.metastore.XUpdatePeriod; +import org.apache.lens.api.metastore.XUpdatePeriodTableDescriptor; +import org.apache.lens.api.metastore.XUpdatePeriods; import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.cube.metadata.Storage.LatestInfo; import org.apache.lens.cube.metadata.Storage.LatestPartColumnInfo; @@ -50,6 +63,10 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.util.ReflectionUtils; import org.apache.thrift.TException; +import org.jvnet.jaxb2_commons.lang.Equals; +import org.jvnet.jaxb2_commons.lang.HashCode; +import org.jvnet.jaxb2_commons.lang.ToString; + import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -290,6 +307,61 @@ public class CubeMetastoreClient { } } + + public <T extends Equals & HashCode & ToString> void createEntity(T entity) throws LensException { + if (entity instanceof XStorage) { + createStorage((XStorage) entity); + } else if (entity instanceof XCube) { + createCube((XCube)entity); + } else if (entity instanceof XDimension) { + createDimension((XDimension) entity); + } else if (entity instanceof XFactTable) { + createCubeFactTable((XFactTable) entity); + } else if (entity instanceof XDimensionTable) { + createCubeDimensionTable((XDimensionTable) entity); + } else if (entity instanceof XSegmentation) { + createSegmentation((XSegmentation) entity); + } else { + throw new LensException("Unable to create entity " + entity + " as it's unrecognizable: "+ entity.getClass()); + } + } + + public <T extends Equals & HashCode & ToString> void updateEntity(String name, T entity) + throws LensException, HiveException { + if (entity instanceof XStorage) { + alterStorage((XStorage) entity); + } else if (entity instanceof XCube) { + alterCube((XCube)entity); + } else if (entity instanceof XDimension) { + alterDimension((XDimension) entity); + } else if (entity instanceof XFactTable) { + alterCubeFactTable((XFactTable) entity); + } else if (entity instanceof XDimensionTable) { + alterCubeDimensionTable((XDimensionTable) entity); + } else if (entity instanceof XSegmentation) { + alterSegmentation((XSegmentation) entity); + } else { + throw new LensException("Unable to alter entity " + entity + " as it's unrecognizable: " + entity.getClass()); + } + } + + + public static Map<String, String> addFactColStartTimePropertyToFactProperties(XFactTable fact) { + Map<String, String> props = new HashMap<String, String>(); + props.putAll(JAXBUtils.mapFromXProperties(fact.getProperties())); + props.putAll(JAXBUtils.columnStartAndEndTimeFromXColumns(fact.getColumns())); + return props; + } + public void createCubeFactTable(XFactTable fact) throws LensException { + createCubeFactTable(fact.getCubeName(), + fact.getName(), + JAXBUtils.fieldSchemaListFromColumns(fact.getColumns()), + JAXBUtils.getFactUpdatePeriodsFromStorageTables(fact.getStorageTables()), + fact.getWeight(), + addFactColStartTimePropertyToFactProperties(fact), + JAXBUtils.tableDescPrefixMapFromXStorageTables(fact.getStorageTables()), + JAXBUtils.storageTablePrefixMapOfStorage(fact.getStorageTables())); + } public void createCubeFactTable(String cubeName, String factName, List<FieldSchema> columns, Map<String, Set<UpdatePeriod>> storageAggregatePeriods, double weight, Map<String, String> properties, Map<String, StorageTableDesc> storageTableDescs, Map<String, Map<UpdatePeriod, String>> storageUpdatePeriodMap) @@ -302,6 +374,7 @@ public class CubeMetastoreClient { } + /** * In-memory storage of {@link PartitionTimeline} objects for each valid * storagetable-updateperiod-partitioncolumn tuple. also simultaneously stored in metastore table of the @@ -619,12 +692,22 @@ public class CubeMetastoreClient { } } + public void createStorage(XStorage storage) throws LensException { + createStorage(JAXBUtils.storageFromXStorage(storage)); + } + public void createStorage(Storage storage) throws LensException { createCubeHiveTable(storage); // do a get to update cache getStorage(storage.getName()); } + public void createCube(XCube cube) throws LensException { + Cube parent = cube instanceof XDerivedCube ? (Cube) getCube( + ((XDerivedCube) cube).getParent()) : null; + createCube(JAXBUtils.hiveCubeFromXCube(cube, parent)); + } + /** * Create cube in metastore defined by {@link Cube} or {@link DerivedCube} object * @@ -714,6 +797,9 @@ public class CubeMetastoreClient { createDimension(dim); } + public void createDimension(XDimension dim) throws LensException { + createDimension(JAXBUtils.dimensionFromXDimension(dim)); + } /** * Create dimension in metastore defined by {@link Dimension} object * @@ -783,6 +869,18 @@ public class CubeMetastoreClient { getSegmentation(segmentationName); } + public void createCubeDimensionTable(XDimensionTable xDimTable) throws LensException { + List<FieldSchema> columns = JAXBUtils.fieldSchemaListFromColumns(xDimTable.getColumns()); + Map<String, UpdatePeriod> updatePeriodMap = + JAXBUtils.dumpPeriodsFromStorageTables(xDimTable.getStorageTables()); + + Map<String, String> properties = JAXBUtils.mapFromXProperties(xDimTable.getProperties()); + Map<String, StorageTableDesc> storageDesc = JAXBUtils.tableDescPrefixMapFromXStorageTables( + xDimTable.getStorageTables()); + log.info("# Columns: " + columns); + createCubeDimensionTable(xDimTable.getDimensionName(), xDimTable.getTableName(), columns, xDimTable.getWeight(), + updatePeriodMap, properties, storageDesc); + } /** * Create a cube dimension table * @@ -846,6 +944,14 @@ public class CubeMetastoreClient { } } + public void createSegmentation(XSegmentation cubeSeg) throws LensException { + createSegmentation( + cubeSeg.getCubeName(), + cubeSeg.getName(), + JAXBUtils.segmentsFromXSegments(cubeSeg.getSegements()), + cubeSeg.getWeight(), + JAXBUtils.mapFromXProperties(cubeSeg.getProperties())); + } public void createSegmentation(Segmentation cubeSeg) throws LensException { // create virtual cube table in metastore @@ -1618,6 +1724,47 @@ public class CubeMetastoreClient { return CubeTableType.DIMENSION.name().equals(tableType); } + public XFactTable getXFactTable(String tableName) throws LensException { + return getXFactTable(getFactTable(tableName)); + } + public XFactTable getXFactTable(CubeFactTable cft) throws LensException { + + XFactTable factTable = JAXBUtils.factTableFromCubeFactTable(cft); + Map<String, Map<UpdatePeriod, String>> storageMap = cft.getStoragePrefixUpdatePeriodMap(); + for (String storageName : cft.getStorages()) { + Set<UpdatePeriod> updatePeriods = cft.getUpdatePeriods().get(storageName); + // This map tells if there are different tables for different update period. + Map<UpdatePeriod, String> updatePeriodToTableMap = storageMap.get(storageName); + Set<String> tableNames = new HashSet<>(); + for (UpdatePeriod updatePeriod : updatePeriods) { + tableNames.add(updatePeriodToTableMap.get(updatePeriod)); + } + if (tableNames.size() <= 1) { + XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( + getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(), storageName))); + tblElement.setStorageName(storageName); + for (UpdatePeriod p : updatePeriods) { + tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); + } + factTable.getStorageTables().getStorageTable().add(tblElement); + } else { + // Multiple storage tables. + XStorageTableElement tblElement = new XStorageTableElement(); + tblElement.setStorageName(storageName); + XUpdatePeriods xUpdatePeriods = new XUpdatePeriods(); + tblElement.setUpdatePeriods(xUpdatePeriods); + for (Map.Entry entry : updatePeriodToTableMap.entrySet()) { + XUpdatePeriodTableDescriptor updatePeriodTableDescriptor = new XUpdatePeriodTableDescriptor(); + updatePeriodTableDescriptor.setTableDesc(getStorageTableDescFromHiveTable( + this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(), (String) entry.getValue())))); + updatePeriodTableDescriptor.setUpdatePeriod(XUpdatePeriod.valueOf(((UpdatePeriod)entry.getKey()).name())); + xUpdatePeriods.getUpdatePeriodTableDescriptor().add(updatePeriodTableDescriptor); + } + factTable.getStorageTables().getStorageTable().add(tblElement); + } + } + return factTable; + } /** * Get {@link CubeFactTable} object corresponding to the name * @@ -1634,6 +1781,25 @@ public class CubeMetastoreClient { return new Segmentation(getTableWithTypeFailFast(tableName, CubeTableType.SEGMENTATION)); } + public XDimensionTable getXDimensionTable(String dimTable) throws LensException { + return getXDimensionTable(getDimensionTable(dimTable)); + } + public XDimensionTable getXDimensionTable(CubeDimensionTable dimTable) throws LensException { + XDimensionTable dt = JAXBUtils.dimTableFromCubeDimTable(dimTable); + if (!dimTable.getStorages().isEmpty()) { + for (String storageName : dimTable.getStorages()) { + XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( + this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(dimTable.getName(), storageName))); + tblElement.setStorageName(storageName); + UpdatePeriod p = dimTable.getSnapshotDumpPeriods().get(storageName); + if (p != null) { + tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); + } + dt.getStorageTables().getStorageTable().add(tblElement); + } + } + return dt; + } /** * Get {@link CubeDimensionTable} object corresponding to the name * @@ -2144,6 +2310,11 @@ public class CubeMetastoreClient { } } + public void alterCube(XCube cube) throws HiveException, LensException { + Cube parent = cube instanceof XDerivedCube ? (Cube) getCube( + ((XDerivedCube) cube).getParent()) : null; + alterCube(cube.getName(), JAXBUtils.hiveCubeFromXCube(cube, parent)); + } /** * Alter cube specified by the name to new definition * @@ -2162,10 +2333,13 @@ public class CubeMetastoreClient { /** * Alter dimension specified by the dimension name to new definition * - * @param dimName The cube name to be altered * @param newDim The new dimension definition * @throws HiveException */ + public void alterDimension(XDimension newDim) throws HiveException, LensException { + alterDimension(newDim.getName(), JAXBUtils.dimensionFromXDimension(newDim)); + } + public void alterDimension(String dimName, Dimension newDim) throws HiveException, LensException { Table tbl = getTableWithTypeFailFast(dimName, CubeTableType.DIMENSION); alterCubeTable(dimName, tbl, newDim); @@ -2177,10 +2351,12 @@ public class CubeMetastoreClient { /** * Alter storage specified by the name to new definition * - * @param storageName The storage name to be altered * @param storage The new storage definition * @throws LensException */ + public void alterStorage(XStorage storage) throws LensException, HiveException { + alterStorage(storage.getName(), JAXBUtils.storageFromXStorage(storage)); + } public void alterStorage(String storageName, Storage storage) throws LensException, HiveException { Table storageTbl = getTableWithTypeFailFast(storageName, CubeTableType.STORAGE); alterCubeTable(storageName, storageTbl, storage); @@ -2333,7 +2509,11 @@ public class CubeMetastoreClient { dropHiveTable(dimTblName); allDimTables.remove(dimTblName.trim().toLowerCase()); } - + public void alterCubeFactTable(XFactTable fact) throws LensException, HiveException { + alterCubeFactTable(fact.getName(), JAXBUtils.cubeFactFromFactTable(fact), + JAXBUtils.tableDescPrefixMapFromXStorageTables(fact.getStorageTables()), + JAXBUtils.columnStartAndEndTimeFromXColumns(fact.getColumns())); + } /** * Alter a cubefact with new definition and alter underlying storage tables as well. * @@ -2361,6 +2541,9 @@ public class CubeMetastoreClient { updateFactCache(factTableName); } + public void alterSegmentation(XSegmentation cubeSeg) throws LensException, HiveException { + alterSegmentation(cubeSeg.getName(), segmentationFromXSegmentation(cubeSeg)); + } public void alterSegmentation(String segName, Segmentation seg) throws HiveException, LensException { getTableWithTypeFailFast(segName, CubeTableType.SEGMENTATION); @@ -2388,7 +2571,11 @@ public class CubeMetastoreClient { allDimTables.put(dimTblName.trim().toLowerCase(), getDimensionTable(refreshTable(dimTblName))); } } - + public void alterCubeDimensionTable(XDimensionTable dimensionTable) throws LensException, HiveException { + alterCubeDimensionTable(dimensionTable.getTableName(), + JAXBUtils.cubeDimTableFromDimTable(dimensionTable), + JAXBUtils.tableDescPrefixMapFromXStorageTables(dimensionTable.getStorageTables())); + } /** * Alter dimension table with new dimension definition and underlying storage tables as well * http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java new file mode 100644 index 0000000..e1e3d16 --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java @@ -0,0 +1,1114 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.metadata; + +import java.lang.reflect.Constructor; +import java.text.ParseException; +import java.util.*; + +import javax.ws.rs.WebApplicationException; +import javax.xml.datatype.DatatypeConfigurationException; +import javax.xml.datatype.DatatypeFactory; +import javax.xml.datatype.XMLGregorianCalendar; + +import org.apache.lens.api.metastore.*; +import org.apache.lens.cube.metadata.ExprColumn.ExprSpec; +import org.apache.lens.cube.metadata.ReferencedDimAttribute.ChainRefCol; +import org.apache.lens.server.api.error.LensException; + +import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.Order; +import org.apache.hadoop.hive.ql.io.HiveOutputFormat; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.mapred.InputFormat; + +import com.google.common.base.Optional; +import com.google.common.collect.Maps; +import lombok.extern.slf4j.Slf4j; + +/** + * Utilities for converting to and from JAXB types to hive.ql.metadata.cube types + */ +@Slf4j +public final class JAXBUtils { + private JAXBUtils() { + + } + + private static final ObjectFactory XCF = new ObjectFactory(); + + /** + * Create a hive ql cube object from corresponding JAXB object + * + * @param cube JAXB Cube + * @return {@link Cube} + * @throws LensException + */ + public static CubeInterface hiveCubeFromXCube(XCube cube, Cube parent) throws LensException { + if (cube instanceof XDerivedCube) { + XDerivedCube dcube = (XDerivedCube) cube; + Set<String> dims = new LinkedHashSet<String>(); + dims.addAll(dcube.getDimAttrNames().getAttrName()); + + Set<String> measures = new LinkedHashSet<String>(); + measures.addAll(dcube.getMeasureNames().getMeasureName()); + + Map<String, String> properties = mapFromXProperties(cube.getProperties()); + return new DerivedCube(cube.getName(), measures, dims, properties, 0L, parent); + } else { + XBaseCube bcube = (XBaseCube) cube; + Set<CubeDimAttribute> dims = new LinkedHashSet<CubeDimAttribute>(); + if (bcube.getDimAttributes() != null && !bcube.getDimAttributes().getDimAttribute().isEmpty()) { + for (XDimAttribute xd : bcube.getDimAttributes().getDimAttribute()) { + dims.add(hiveDimAttrFromXDimAttr(xd)); + } + } + + Set<CubeMeasure> measures = new LinkedHashSet<CubeMeasure>(); + for (XMeasure xm : bcube.getMeasures().getMeasure()) { + measures.add(hiveMeasureFromXMeasure(xm)); + } + + Set<ExprColumn> expressions = new LinkedHashSet<ExprColumn>(); + if (bcube.getExpressions() != null && !bcube.getExpressions().getExpression().isEmpty()) { + for (XExprColumn xe : bcube.getExpressions().getExpression()) { + expressions.add(hiveExprColumnFromXExprColumn(xe)); + } + } + + Set<JoinChain> joinchains = new LinkedHashSet<JoinChain>(); + if (bcube.getJoinChains() != null && !bcube.getJoinChains().getJoinChain().isEmpty()) { + for (XJoinChain xj : bcube.getJoinChains().getJoinChain()) { + joinchains.add(joinChainFromXJoinChain(xj)); + } + } + + Map<String, String> properties = mapFromXProperties(cube.getProperties()); + return new Cube(cube.getName(), measures, dims, expressions, joinchains, properties, 0L); + } + } + + /** + * Get XCube from hive.ql.metadata.Cube + * + * @param c + * @return {@link XCube} + */ + public static XCube xCubeFromHiveCube(CubeInterface c) { + XCube xc; + if (c.isDerivedCube()) { + XDerivedCube xdc = XCF.createXDerivedCube(); + xdc.setMeasureNames(new XMeasureNames()); + xdc.setDimAttrNames(new XDimAttrNames()); + xc = xdc; + xdc.getMeasureNames().getMeasureName().addAll(c.getMeasureNames()); + xdc.getDimAttrNames().getAttrName().addAll(c.getDimAttributeNames()); + xdc.setParent(((DerivedCube) c).getParent().getName()); + } else { + XBaseCube xbc = XCF.createXBaseCube(); + xbc.setMeasures(new XMeasures()); + xbc.setDimAttributes(new XDimAttributes()); + xbc.setExpressions(new XExpressions()); + xbc.setJoinChains(new XJoinChains()); + xc = xbc; + for (CubeMeasure cm : c.getMeasures()) { + xbc.getMeasures().getMeasure().add(xMeasureFromHiveMeasure(cm)); + } + + for (ExprColumn ec : c.getExpressions()) { + xbc.getExpressions().getExpression().add(xExprColumnFromHiveExprColumn(ec)); + } + for (CubeDimAttribute cd : c.getDimAttributes()) { + xbc.getDimAttributes().getDimAttribute().add(xDimAttrFromHiveDimAttr(cd, (Cube) c)); + } + for (JoinChain jc : c.getJoinChains()) { + xbc.getJoinChains().getJoinChain().add(getXJoinChainFromJoinChain(jc)); + } + } + xc.setName(c.getName()); + xc.setProperties(new XProperties()); + xc.getProperties().getProperty().addAll(xPropertiesFromMap(((AbstractCubeTable) c).getProperties())); + return xc; + } + + /** + * Create a hive ql CubeDimension from JAXB counterpart + * + * @param xd + * @return {@link org.apache.lens.cube.metadata.CubeDimAttribute} + */ + public static CubeDimAttribute hiveDimAttrFromXDimAttr(XDimAttribute xd) throws LensException { + Date startDate = getDateFromXML(xd.getStartTime()); + Date endDate = getDateFromXML(xd.getEndTime()); + + CubeDimAttribute hiveDim; + + if (xd.getHierarchy() != null) { + List<CubeDimAttribute> hierarchy = new ArrayList<>(); + for (XDimAttribute hd : xd.getHierarchy().getDimAttribute()) { + hierarchy.add(hiveDimAttrFromXDimAttr(hd)); + } + hiveDim = new HierarchicalDimAttribute(xd.getName(), xd.getDescription(), hierarchy); + } else if (xd.getChainRefColumn() != null + && !xd.getChainRefColumn().isEmpty()) { + hiveDim = new ReferencedDimAttribute(new FieldSchema(xd.getName(), xd.getType().toLowerCase(), + xd.getDescription()), + xd.getDisplayString(), + getChainRefColumns(xd.getChainRefColumn()), + startDate, + endDate, + null, + xd.getNumDistinctValues(), + xd.getValues(), + mapFromXProperties(xd.getTags()) + ); + } else { + hiveDim = new BaseDimAttribute(new FieldSchema(xd.getName(), xd.getType().toLowerCase(), + xd.getDescription()), + xd.getDisplayString(), + startDate, + endDate, + null, + xd.getNumDistinctValues(), + xd.getValues(), + mapFromXProperties(xd.getTags()) + ); + } + return hiveDim; + } + + private static List<ChainRefCol> getChainRefColumns(List<XChainColumn> chainCols) { + List<ChainRefCol> chainRefCols = new ArrayList<>(); + for (XChainColumn chainCol : chainCols) { + chainRefCols.add(new ChainRefCol(chainCol.getChainName(), chainCol.getRefCol())); + } + return chainRefCols; + } + + /** + * Get XMLGregorianCalendar from Date. + * + * Useful for converting from java code to XML spec. + * + * @param d Date value + * @return XML value + */ + public static XMLGregorianCalendar getXMLGregorianCalendar(Date d) { + if (d == null) { + return null; + } + + GregorianCalendar c1 = new GregorianCalendar(); + c1.setTime(d); + try { + return DatatypeFactory.newInstance().newXMLGregorianCalendar(c1); + } catch (DatatypeConfigurationException e) { + log.warn("Error converting date " + d, e); + return null; + } + } + + /** + * Get Date from XMLGregorianCalendar + * + * Useful for converting from XML spec to java code. + * + * @param cal XML value + * @return Date value + */ + public static Date getDateFromXML(XMLGregorianCalendar cal) { + if (cal == null) { + return null; + } + return cal.toGregorianCalendar().getTime(); + } + + /** + * Create XMeasure from hive ql cube measure + */ + public static XMeasure xMeasureFromHiveMeasure(CubeMeasure cm) { + if (cm == null) { + return null; + } + + XMeasure xm = XCF.createXMeasure(); + xm.setName(cm.getName()); + xm.setDescription(cm.getDescription()); + xm.setDisplayString(cm.getDisplayString()); + xm.setDefaultAggr(cm.getAggregate()); + xm.setFormatString(cm.getFormatString()); + xm.setType(XMeasureType.valueOf(cm.getType().toUpperCase())); + xm.setUnit(cm.getUnit()); + xm.setStartTime(getXMLGregorianCalendar(cm.getStartTime())); + xm.setEndTime(getXMLGregorianCalendar(cm.getEndTime())); + xm.setMin(cm.getMin()); + xm.setMax(cm.getMax()); + xm.setTags(getXProperties(xPropertiesFromMap(cm.getTags()))); + return xm; + } + + public static XProperties getXProperties(List<XProperty> prop) { + XProperties properties = XCF.createXProperties(); + properties.getProperty().addAll(prop); + return properties; + } + + /** + * Create XExprColumn from hive ExprColum + */ + public static XExprColumn xExprColumnFromHiveExprColumn(ExprColumn ec) { + if (ec == null) { + return null; + } + + XExprColumn xe = XCF.createXExprColumn(); + xe.setName(ec.getName()); + xe.setType(ec.getType()); + xe.setDescription(ec.getDescription()); + xe.setDisplayString(ec.getDisplayString()); + xe.getExprSpec().addAll(xExprSpecFromExprColumn(ec.getExpressionSpecs())); + xe.setTags(getXProperties(xPropertiesFromMap(ec.getTags()))); + return xe; + } + + private static Collection<XExprSpec> xExprSpecFromExprColumn(Collection<ExprSpec> esSet) { + List<XExprSpec> xes = new ArrayList<XExprSpec>(); + for (ExprSpec es : esSet) { + XExprSpec e = new XExprSpec(); + e.setExpr(es.getExpr()); + e.setStartTime(getXMLGregorianCalendar(es.getStartTime())); + e.setEndTime(getXMLGregorianCalendar(es.getEndTime())); + xes.add(e); + } + return xes; + } + + private static ExprSpec[] exprSpecFromXExprColumn(Collection<XExprSpec> xesList) throws LensException { + List<ExprSpec> esArray = new ArrayList<ExprSpec>(xesList.size()); + for (XExprSpec xes : xesList) { + esArray.add(new ExprSpec(xes.getExpr(), getDateFromXML(xes.getStartTime()), getDateFromXML(xes.getEndTime()))); + } + return esArray.toArray(new ExprSpec[0]); + } + + /** + * Create XDimAttribute from CubeDimAttribute + */ + public static XDimAttribute xDimAttrFromHiveDimAttr(CubeDimAttribute cd, AbstractBaseTable baseTable) { + XDimAttribute xd = XCF.createXDimAttribute(); + xd.setName(cd.getName()); + xd.setDescription(cd.getDescription()); + xd.setDisplayString(cd.getDisplayString()); + xd.setStartTime(getXMLGregorianCalendar(cd.getStartTime())); + xd.setEndTime(getXMLGregorianCalendar(cd.getEndTime())); + xd.setTags(getXProperties(xPropertiesFromMap(cd.getTags()))); + if (cd instanceof ReferencedDimAttribute) { + ReferencedDimAttribute rd = (ReferencedDimAttribute) cd; + if (!rd.getChainRefColumns().isEmpty()) { + for (ChainRefCol crCol : rd.getChainRefColumns()) { + XChainColumn xcc = new XChainColumn(); + xcc.setChainName(crCol.getChainName()); + xcc.setRefCol(crCol.getRefColumn()); + if (baseTable.getChainByName(crCol.getChainName()) == null) { + log.error("Missing chain definition for " + crCol.getChainName()); + } else { + xcc.setDestTable(baseTable.getChainByName(crCol.getChainName()).getDestTable()); + } + xd.getChainRefColumn().add(xcc); + } + } + xd.setType(rd.getType()); + Optional<Long> numOfDistinctValues = rd.getNumOfDistinctValues(); + if (numOfDistinctValues.isPresent()) { + xd.setNumDistinctValues(numOfDistinctValues.get()); + } + if (rd.getValues().isPresent()) { + xd.getValues().addAll(rd.getValues().get()); + } + } else if (cd instanceof BaseDimAttribute) { + BaseDimAttribute bd = (BaseDimAttribute) cd; + xd.setType(bd.getType()); + Optional<Long> numOfDistinctValues = bd.getNumOfDistinctValues(); + if (numOfDistinctValues.isPresent()) { + xd.setNumDistinctValues(numOfDistinctValues.get()); + } + if (bd.getValues().isPresent()) { + xd.getValues().addAll(bd.getValues().get()); + } + } else if (cd instanceof HierarchicalDimAttribute) { + HierarchicalDimAttribute hd = (HierarchicalDimAttribute) cd; + XDimAttributes hierarchy = new XDimAttributes(); + for (CubeDimAttribute hdDim : hd.getHierarchy()) { + hierarchy.getDimAttribute().add(xDimAttrFromHiveDimAttr(hdDim, baseTable)); + } + xd.setHierarchy(hierarchy); + } + return xd; + } + + /** + * Create XJoinChain from cube join chain + */ + public static XJoinChain getXJoinChainFromJoinChain(JoinChain jc) { + XJoinChain xjc = XCF.createXJoinChain(); + xjc.setName(jc.getName()); + xjc.setDescription(jc.getDescription()); + xjc.setDisplayString(jc.getDisplayString()); + xjc.setDestTable(jc.getDestTable()); + xjc.setPaths(new XJoinPaths()); + + for (JoinChain.Path path : jc.getPaths()) { + xjc.getPaths().getPath().add(xJoinPathFromJoinPath(path)); + } + return xjc; + } + + public static XJoinPath xJoinPathFromJoinPath(JoinChain.Path path) { + XJoinPath xjp = XCF.createXJoinPath(); + xjp.setEdges(new XJoinEdges()); + for (JoinChain.Edge edge : path.getLinks()) { + XJoinEdge xje = XCF.createXJoinEdge(); + xje.setFrom(xTabReferenceFromTabReference(edge.getFrom())); + xje.setTo(xTabReferenceFromTabReference(edge.getTo())); + xjp.getEdges().getEdge().add(xje); + } + return xjp; + } + + public static List<XTableReference> xTabReferencesFromHiveTabReferences(List<TableReference> hiveRefs) { + List<XTableReference> xrefList = new ArrayList<XTableReference>(); + + for (TableReference hRef : hiveRefs) { + xrefList.add(xTabReferenceFromTabReference(hRef)); + } + return xrefList; + } + + private static XTableReference xTabReferenceFromTabReference(TableReference ref) { + XTableReference xref = XCF.createXTableReference(); + xref.setTable(ref.getDestTable()); + xref.setColumn(ref.getDestColumn()); + xref.setMapsToMany(ref.isMapsToMany()); + return xref; + } + + /** + * Create hive ql CubeMeasure from JAXB counterpart + * + * @param xm + * @return {@link CubeMeasure} + */ + public static CubeMeasure hiveMeasureFromXMeasure(XMeasure xm) { + Date startDate = xm.getStartTime() == null ? null : xm.getStartTime().toGregorianCalendar().getTime(); + Date endDate = xm.getEndTime() == null ? null : xm.getEndTime().toGregorianCalendar().getTime(); + CubeMeasure cm = new ColumnMeasure(new FieldSchema(xm.getName(), xm.getType().name().toLowerCase(), + xm.getDescription()), + xm.getDisplayString(), + xm.getFormatString(), + xm.getDefaultAggr(), + xm.getUnit(), + startDate, + endDate, + null, + xm.getMin(), + xm.getMax(), + mapFromXProperties(xm.getTags()) + ); + return cm; + } + + /** + * Create cube's JoinChain from JAXB counterpart + * + * @param xj + * @return {@link JoinChain} + */ + public static JoinChain joinChainFromXJoinChain(XJoinChain xj) { + JoinChain jc = new JoinChain(xj.getName(), xj.getDisplayString(), xj.getDescription()); + for (int i = 0; i < xj.getPaths().getPath().size(); i++) { + XJoinPath xchain = xj.getPaths().getPath().get(i); + List<TableReference> chain = new ArrayList<TableReference>(xchain.getEdges().getEdge().size() * 2); + + for (XJoinEdge xRef : xchain.getEdges().getEdge()) { + chain.add(new TableReference(xRef.getFrom().getTable(), xRef.getFrom().getColumn(), + xRef.getFrom().isMapsToMany())); + chain.add(new TableReference(xRef.getTo().getTable(), xRef.getTo().getColumn(), xRef.getTo().isMapsToMany())); + } + jc.addPath(chain); + } + return jc; + } + + public static ExprColumn hiveExprColumnFromXExprColumn(XExprColumn xe) throws LensException { + ExprColumn ec = new ExprColumn(new FieldSchema(xe.getName(), xe.getType().toLowerCase(), + xe.getDescription()), + xe.getDisplayString(), + mapFromXProperties(xe.getTags()), + exprSpecFromXExprColumn(xe.getExprSpec())); + return ec; + } + + /** + * Convert JAXB properties to Map<String, String> + * + * @param xProperties + * @return {@link Map} + */ + public static Map<String, String> mapFromXProperties(XProperties xProperties) { + Map<String, String> properties = new HashMap<String, String>(); + if (xProperties != null && !xProperties.getProperty().isEmpty()) { + for (XProperty xp : xProperties.getProperty()) { + properties.put(xp.getName(), xp.getValue()); + } + } + return properties; + } + + /** + * Convert string map to XProperties + */ + public static List<XProperty> xPropertiesFromMap(Map<String, String> map) { + List<XProperty> xpList = new ArrayList<XProperty>(); + if (map != null && !map.isEmpty()) { + for (Map.Entry<String, String> e : map.entrySet()) { + XProperty property = XCF.createXProperty(); + property.setName(e.getKey()); + property.setValue(e.getValue()); + xpList.add(property); + } + } + return xpList; + } + + public static Set<XSegment> xSegmentsFromSegments(Set<Segment> segs) { + Set<XSegment> xsegs = new HashSet<XSegment>(); + if (segs != null && !segs.isEmpty()) { + for (Segment seg : segs) { + XSegment xcubeSeg = XCF.createXSegment(); + xcubeSeg.setCubeName(seg.getName()); + xcubeSeg.setSegmentParameters(getXpropertiesFromSegment(seg)); + xsegs.add(xcubeSeg); + } + } + return xsegs; + } + + public static XProperties getXpropertiesFromSegment(Segment cseg) { + XProperties xproperties = XCF.createXProperties(); + for (String prop : cseg.getProperties().keySet()) { + String segPrefix = MetastoreUtil.getSegmentPropertyKey(cseg.getName()); + if (prop.startsWith(segPrefix)){ + XProperty xprop = XCF.createXProperty(); + xprop.setName(prop.replace(segPrefix, "")); + xprop.setValue(cseg.getProperties().get(prop)); + xproperties.getProperty().add(xprop); + } + } + return xproperties; + } + + + public static FieldSchema fieldSchemaFromColumn(XColumn c) { + if (c == null) { + return null; + } + + return new FieldSchema(c.getName(), c.getType().toLowerCase(), c.getComment()); + } + + public static XColumn columnFromFieldSchema(FieldSchema fs) { + if (fs == null) { + return null; + } + XColumn c = XCF.createXColumn(); + c.setName(fs.getName()); + c.setType(fs.getType()); + c.setComment(fs.getComment()); + return c; + } + + public static ArrayList<FieldSchema> fieldSchemaListFromColumns(XColumns columns) { + if (columns != null && !columns.getColumn().isEmpty()) { + ArrayList<FieldSchema> fsList = new ArrayList<FieldSchema>(columns.getColumn().size()); + for (XColumn c : columns.getColumn()) { + fsList.add(fieldSchemaFromColumn(c)); + } + return fsList; + } + return null; + } + + public static Map<String, String> columnStartAndEndTimeFromXColumns(XColumns columns) { + if (columns != null && !columns.getColumn().isEmpty()) { + Map<String, String> colStartTimeMap = new HashMap<String, String>(); + for (XColumn c : columns.getColumn()) { + if (!(c.getStartTime() == null)) { + colStartTimeMap.put(MetastoreConstants.FACT_COL_START_TIME_PFX.concat(c.getName()), c.getStartTime()); + } + if (!(c.getEndTime() == null)) { + colStartTimeMap.put(MetastoreConstants.FACT_COL_END_TIME_PFX.concat(c.getName()), c.getEndTime()); + } + } + return colStartTimeMap; + } + return null; + } + + public static List<XColumn> columnsFromFieldSchemaList(List<FieldSchema> fslist) { + List<XColumn> cols = new ArrayList<XColumn>(); + if (fslist == null || fslist.isEmpty()) { + return cols; + } + + for (FieldSchema fs : fslist) { + cols.add(columnFromFieldSchema(fs)); + } + return cols; + } + + public static Map<String, Set<UpdatePeriod>> getFactUpdatePeriodsFromStorageTables(XStorageTables storageTables) { + if (storageTables != null && !storageTables.getStorageTable().isEmpty()) { + Map<String, Set<UpdatePeriod>> factUpdatePeriods = new LinkedHashMap<String, Set<UpdatePeriod>>(); + + for (XStorageTableElement ste : storageTables.getStorageTable()) { + Set<UpdatePeriod> updatePeriods = new TreeSet<>(); + // Check if the update period array is empty. + List<XUpdatePeriod> updatePeriodList = ste.getUpdatePeriods().getUpdatePeriod(); + if (updatePeriodList.isEmpty()) { + List<XUpdatePeriodTableDescriptor> tableDescriptorList = ste.getUpdatePeriods() + .getUpdatePeriodTableDescriptor(); + for (XUpdatePeriodTableDescriptor tableDescriptor : tableDescriptorList) { + updatePeriodList.add(tableDescriptor.getUpdatePeriod()); + } + } + for (XUpdatePeriod upd : updatePeriodList) { + updatePeriods.add(UpdatePeriod.valueOf(upd.name())); + } + factUpdatePeriods.put(ste.getStorageName(), updatePeriods); + } + return factUpdatePeriods; + } + return null; + } + + public static Map<String, UpdatePeriod> dumpPeriodsFromStorageTables(XStorageTables storageTables) { + if (storageTables != null && !storageTables.getStorageTable().isEmpty()) { + Map<String, UpdatePeriod> dumpPeriods = new LinkedHashMap<String, UpdatePeriod>(); + + for (XStorageTableElement ste : storageTables.getStorageTable()) { + UpdatePeriod dumpPeriod = null; + if (ste.getUpdatePeriods() != null && !ste.getUpdatePeriods().getUpdatePeriod().isEmpty()) { + dumpPeriod = UpdatePeriod.valueOf(ste.getUpdatePeriods().getUpdatePeriod().get(0).name()); + } + dumpPeriods.put(ste.getStorageName(), dumpPeriod); + } + return dumpPeriods; + } + return null; + } + + public static Storage storageFromXStorage(XStorage xs) { + if (xs == null) { + return null; + } + + Storage storage; + try { + Class<?> clazz = Class.forName(xs.getClassname()); + Constructor<?> constructor = clazz.getConstructor(String.class); + storage = (Storage) constructor.newInstance(xs.getName()); + storage.addProperties(mapFromXProperties(xs.getProperties())); + return storage; + } catch (Exception e) { + log.error("Could not create storage class" + xs.getClassname() + "with name:" + xs.getName(), e); + throw new WebApplicationException(e); + } + } + + public static XStorage xstorageFromStorage(Storage storage) { + if (storage == null) { + return null; + } + + XStorage xstorage = null; + xstorage = XCF.createXStorage(); + xstorage.setProperties(new XProperties()); + xstorage.setName(storage.getName()); + xstorage.setClassname(storage.getClass().getCanonicalName()); + xstorage.getProperties().getProperty().addAll(xPropertiesFromMap(storage.getProperties())); + return xstorage; + } + + public static XDimensionTable dimTableFromCubeDimTable(CubeDimensionTable cubeDimTable) { + if (cubeDimTable == null) { + return null; + } + + XDimensionTable dimTab = XCF.createXDimensionTable(); + dimTab.setDimensionName(cubeDimTable.getDimName()); + dimTab.setTableName(cubeDimTable.getName()); + dimTab.setWeight(cubeDimTable.weight()); + dimTab.setColumns(new XColumns()); + dimTab.setProperties(new XProperties()); + dimTab.setStorageTables(new XStorageTables()); + + for (FieldSchema column : cubeDimTable.getColumns()) { + dimTab.getColumns().getColumn().add(columnFromFieldSchema(column)); + } + dimTab.getProperties().getProperty().addAll(xPropertiesFromMap(cubeDimTable.getProperties())); + + return dimTab; + } + + public static List<? extends XTableReference> dimRefListFromTabRefList( + List<TableReference> tabRefs) { + if (tabRefs != null && !tabRefs.isEmpty()) { + List<XTableReference> xTabRefs = new ArrayList<XTableReference>(tabRefs.size()); + for (TableReference ref : tabRefs) { + XTableReference xRef = XCF.createXTableReference(); + xRef.setColumn(ref.getDestColumn()); + xRef.setTable(ref.getDestTable()); + xRef.setMapsToMany(ref.isMapsToMany()); + xTabRefs.add(xRef); + } + return xTabRefs; + } + + return null; + } + + public static CubeDimensionTable cubeDimTableFromDimTable(XDimensionTable dimensionTable) throws LensException { + + return new CubeDimensionTable(dimensionTable.getDimensionName(), + dimensionTable.getTableName(), + fieldSchemaListFromColumns(dimensionTable.getColumns()), + dimensionTable.getWeight(), + dumpPeriodsFromStorageTables(dimensionTable.getStorageTables()), + mapFromXProperties(dimensionTable.getProperties())); + } + + public static CubeFactTable cubeFactFromFactTable(XFactTable fact) throws LensException { + List<FieldSchema> columns = fieldSchemaListFromColumns(fact.getColumns()); + + Map<String, Set<UpdatePeriod>> storageUpdatePeriods = getFactUpdatePeriodsFromStorageTables( + fact.getStorageTables()); + Map<String, Map<UpdatePeriod, String>> storageTablePrefixMap = storageTablePrefixMapOfStorage( + fact.getStorageTables()); + return new CubeFactTable(fact.getCubeName(), fact.getName(), columns, storageUpdatePeriods, fact.getWeight(), + mapFromXProperties(fact.getProperties()), storageTablePrefixMap); + } + + public static Segmentation segmentationFromXSegmentation(XSegmentation seg) throws LensException { + + Map<String, String> props = new HashMap<>(); + // Skip properties with keyword internal. These properties are internal to lens + // and users are not supposed to see them. + for(String prop : mapFromXProperties(seg.getProperties()).keySet()) { + if (!(prop.toLowerCase().startsWith(MetastoreConstants.SEGMENTATION_KEY_PFX))) { + props.put(prop, mapFromXProperties(seg.getProperties()).get(prop)); + } + } + return new Segmentation(seg.getCubeName(), + seg.getName(), + segmentsFromXSegments(seg.getSegements()), + seg.getWeight(), + props); + } + + + public static XFactTable factTableFromCubeFactTable(CubeFactTable cFact) { + XFactTable fact = XCF.createXFactTable(); + fact.setName(cFact.getName()); + fact.setColumns(new XColumns()); + fact.setProperties(new XProperties()); + fact.setStorageTables(new XStorageTables()); + fact.getProperties().getProperty().addAll(xPropertiesFromMap(cFact.getProperties())); + fact.getColumns().getColumn().addAll(columnsFromFieldSchemaList(cFact.getColumns())); + fact.setWeight(cFact.weight()); + fact.setCubeName(cFact.getCubeName()); + return fact; + } + + public static XSegmentation xsegmentationFromSegmentation(Segmentation cSeg) { + XSegmentation seg = XCF.createXSegmentation(); + seg.setName(cSeg.getName()); + seg.setProperties(new XProperties()); + seg.setSegements(new XSegments()); + seg.setWeight(cSeg.weight()); + seg.setCubeName(cSeg.getBaseCube()); + if (xPropertiesFromMap(cSeg.getProperties()) != null) { + seg.getProperties().getProperty().addAll(xPropertiesFromMap(cSeg.getProperties())); + } + seg.getSegements().getSegment(). + addAll(xSegmentsFromSegments(cSeg.getSegments())); + return seg; + } + + public static StorageTableDesc storageTableDescFromXStorageTableDesc( + XStorageTableDesc xtableDesc) { + StorageTableDesc tblDesc = new StorageTableDesc(); + tblDesc.setTblProps(mapFromXProperties(xtableDesc.getTableParameters())); + tblDesc.setSerdeProps(mapFromXProperties(xtableDesc.getSerdeParameters())); + tblDesc.setPartCols(fieldSchemaListFromColumns(xtableDesc.getPartCols())); + tblDesc.setTimePartCols(xtableDesc.getTimePartCols()); + tblDesc.setExternal(xtableDesc.isExternal()); + tblDesc.setLocation(xtableDesc.getTableLocation()); + tblDesc.setInputFormat(xtableDesc.getInputFormat()); + tblDesc.setOutputFormat(xtableDesc.getOutputFormat()); + tblDesc.setFieldDelim(xtableDesc.getFieldDelimiter()); + tblDesc.setFieldEscape(xtableDesc.getEscapeChar()); + tblDesc.setCollItemDelim(xtableDesc.getCollectionDelimiter()); + tblDesc.setLineDelim(xtableDesc.getLineDelimiter()); + tblDesc.setMapKeyDelim(xtableDesc.getMapKeyDelimiter()); + tblDesc.setSerName(xtableDesc.getSerdeClassName()); + tblDesc.setStorageHandler(xtableDesc.getStorageHandlerName()); + return tblDesc; + } + + public static StorageTableDesc storageTableDescFromXStorageTableElement( + XStorageTableElement storageTableElement) { + return storageTableDescFromXStorageTableDesc(storageTableElement.getTableDesc()); + } + + public static XStorageTableElement getXStorageTableFromHiveTable(Table tbl) { + XStorageTableElement tblElement = new XStorageTableElement(); + tblElement.setUpdatePeriods(new XUpdatePeriods()); + tblElement.setTableDesc(getStorageTableDescFromHiveTable(tbl)); + return tblElement; + } + + public static XStorageTableDesc getStorageTableDescFromHiveTable(Table tbl) { + XStorageTableDesc tblDesc = new XStorageTableDesc(); + tblDesc.setPartCols(new XColumns()); + tblDesc.setTableParameters(new XProperties()); + tblDesc.setSerdeParameters(new XProperties()); + tblDesc.getPartCols().getColumn().addAll(columnsFromFieldSchemaList(tbl.getPartCols())); + String timePartCols = tbl.getParameters().get(MetastoreConstants.TIME_PART_COLUMNS); + if (timePartCols != null) { + tblDesc.getTimePartCols().addAll(Arrays.asList(org.apache.commons.lang.StringUtils.split(timePartCols, ","))); + } + tblDesc.setNumBuckets(tbl.getNumBuckets()); + tblDesc.getBucketCols().addAll(tbl.getBucketCols()); + List<String> sortCols = new ArrayList<String>(); + List<Integer> sortOrders = new ArrayList<Integer>(); + for (Order order : tbl.getSortCols()) { + sortCols.add(order.getCol()); + sortOrders.add(order.getOrder()); + } + tblDesc.getSortCols().addAll(sortCols); + tblDesc.getSortColOrder().addAll(sortOrders); + + XSkewedInfo xskewinfo = new XSkewedInfo(); + xskewinfo.getColNames().addAll(tbl.getSkewedColNames()); + for (List<String> value : tbl.getSkewedColValues()) { + XSkewColList colVallist = new XSkewColList(); + colVallist.getElements().addAll(value); + xskewinfo.getColValues().add(colVallist); + XSkewedValueLocation valueLocation = new XSkewedValueLocation(); + if (tbl.getSkewedColValueLocationMaps().get(value) != null) { + valueLocation.setValue(colVallist); + valueLocation.setLocation(tbl.getSkewedColValueLocationMaps().get(value)); + xskewinfo.getValueLocationMap().add(valueLocation); + } + } + + tblDesc.getTableParameters().getProperty().addAll(xPropertiesFromMap(tbl.getParameters())); + tblDesc.getSerdeParameters().getProperty().addAll(xPropertiesFromMap( + tbl.getTTable().getSd().getSerdeInfo().getParameters())); + tblDesc.setExternal(tbl.getTableType().equals(TableType.EXTERNAL_TABLE)); + tblDesc.setCompressed(tbl.getTTable().getSd().isCompressed()); + tblDesc.setTableLocation(tbl.getDataLocation().toString()); + tblDesc.setInputFormat(tbl.getInputFormatClass().getCanonicalName()); + tblDesc.setOutputFormat(tbl.getOutputFormatClass().getCanonicalName()); + tblDesc.setFieldDelimiter(tbl.getSerdeParam(serdeConstants.FIELD_DELIM)); + tblDesc.setLineDelimiter(tbl.getSerdeParam(serdeConstants.LINE_DELIM)); + tblDesc.setCollectionDelimiter(tbl.getSerdeParam(serdeConstants.COLLECTION_DELIM)); + tblDesc.setMapKeyDelimiter(tbl.getSerdeParam(serdeConstants.MAPKEY_DELIM)); + tblDesc.setEscapeChar(tbl.getSerdeParam(serdeConstants.ESCAPE_CHAR)); + tblDesc.setSerdeClassName(tbl.getSerializationLib()); + tblDesc.setStorageHandlerName(tbl.getStorageHandler() != null + ? tbl.getStorageHandler().getClass().getCanonicalName() : ""); + return tblDesc; + } + + public static Map<String, StorageTableDesc> tableDescPrefixMapFromXStorageTables(XStorageTables storageTables) { + Map<String, StorageTableDesc> storageTablePrefixToDescMap = new HashMap<>(); + if (storageTables != null && !storageTables.getStorageTable().isEmpty()) { + for (XStorageTableElement sTbl : storageTables.getStorageTable()) { + if (sTbl.getUpdatePeriods() != null && sTbl.getUpdatePeriods().getUpdatePeriodTableDescriptor() != null && !sTbl + .getUpdatePeriods().getUpdatePeriodTableDescriptor().isEmpty()) { + for (XUpdatePeriodTableDescriptor updatePeriodTable : sTbl.getUpdatePeriods() + .getUpdatePeriodTableDescriptor()) { + // Get table name with update period as the prefix. + storageTablePrefixToDescMap.put(updatePeriodTable.getUpdatePeriod() + "_" + sTbl.getStorageName(), + storageTableDescFromXStorageTableDesc(updatePeriodTable.getTableDesc())); + } + } else { + storageTablePrefixToDescMap.put(sTbl.getStorageName(), storageTableDescFromXStorageTableElement(sTbl)); + } + } + } + return storageTablePrefixToDescMap; + } + + public static Map<String, Map<UpdatePeriod, String>> storageTablePrefixMapOfStorage(XStorageTables storageTables) { + Map<String, Map<UpdatePeriod, String>> storageTableMap = new HashMap<>(); + if (storageTables != null && !storageTables.getStorageTable().isEmpty()) { + for (XStorageTableElement sTbl : storageTables.getStorageTable()) { + Map<UpdatePeriod, String> storageNameMap = new HashMap<>(); + if (sTbl.getUpdatePeriods() != null && sTbl.getUpdatePeriods().getUpdatePeriodTableDescriptor() != null && !sTbl + .getUpdatePeriods().getUpdatePeriodTableDescriptor().isEmpty()) { + for (XUpdatePeriodTableDescriptor updatePeriodTable : sTbl.getUpdatePeriods() + .getUpdatePeriodTableDescriptor()) { + // Get table name with update period as the prefix. + storageNameMap.put(UpdatePeriod.valueOf(updatePeriodTable.getUpdatePeriod().value()), + updatePeriodTable.getUpdatePeriod() + "_" + sTbl.getStorageName()); + } + } else { + for (XUpdatePeriod updatePeriod : sTbl.getUpdatePeriods().getUpdatePeriod()) { + storageNameMap.put(UpdatePeriod.valueOf(updatePeriod.value()), sTbl.getStorageName()); + } + } + storageTableMap.put(sTbl.getStorageName(), storageNameMap); + } + } + return storageTableMap; + } + + public static Set<Segment> segmentsFromXSegments(XSegments segs) { + Set<Segment> cubeSegs = new HashSet<>(); + for (XSegment xcube : segs.getSegment()){ + Map<String, String> segProp = new HashMap<>(); + if (xcube.getSegmentParameters() != null) { + for (XProperty prop : xcube.getSegmentParameters().getProperty()) { + segProp.put(prop.getName(), prop.getValue()); + } + } + cubeSegs.add(new Segment(xcube.getCubeName(), segProp)); + } + return cubeSegs; + } + + public static Map<String, Date> timePartSpecfromXTimePartSpec( + XTimePartSpec xtimePartSpec) { + Map<String, Date> timePartSpec = new HashMap<String, Date>(); + if (xtimePartSpec != null && !xtimePartSpec.getPartSpecElement().isEmpty()) { + for (XTimePartSpecElement xtimePart : xtimePartSpec.getPartSpecElement()) { + timePartSpec.put(xtimePart.getKey(), getDateFromXML(xtimePart.getValue())); + } + } + return timePartSpec; + } + + public static Map<String, String> nonTimePartSpecfromXNonTimePartSpec( + XPartSpec xnonTimePartSpec) { + Map<String, String> nonTimePartSpec = new HashMap<String, String>(); + if (xnonTimePartSpec != null && !xnonTimePartSpec.getPartSpecElement().isEmpty()) { + for (XPartSpecElement xPart : xnonTimePartSpec.getPartSpecElement()) { + nonTimePartSpec.put(xPart.getKey(), xPart.getValue()); + } + } + return nonTimePartSpec; + } + + public static XPartitionList xpartitionListFromPartitionList(String cubeTableName, List<Partition> partitions, + List<String> timePartCols) throws HiveException { + XPartitionList xPartitionList = new XPartitionList(); + xPartitionList.getPartition(); + if (partitions != null) { + for (Partition partition : partitions) { + xPartitionList.getPartition().add(xpartitionFromPartition(cubeTableName, partition, timePartCols)); + } + } + return xPartitionList; + } + + public static XPartition xpartitionFromPartition(String cubeTableName, Partition p, List<String> timePartCols) + throws HiveException { + XPartition xp = new XPartition(); + xp.setFactOrDimensionTableName(cubeTableName); + xp.setPartitionParameters(new XProperties()); + xp.setSerdeParameters(new XProperties()); + xp.setName(p.getCompleteName()); + xp.setLocation(p.getLocation()); + xp.setInputFormat(p.getInputFormatClass().getCanonicalName()); + xp.setOutputFormat(p.getOutputFormatClass().getCanonicalName()); + xp.getPartitionParameters().getProperty().addAll(xPropertiesFromMap(p.getParameters())); + String upParam = p.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD); + xp.setUpdatePeriod(XUpdatePeriod.valueOf(upParam)); + LinkedHashMap<String, String> partSpec = p.getSpec(); + xp.setFullPartitionSpec(new XPartSpec()); + for (Map.Entry<String, String> entry : partSpec.entrySet()) { + XPartSpecElement e = new XPartSpecElement(); + e.setKey(entry.getKey()); + e.setValue(entry.getValue()); + xp.getFullPartitionSpec().getPartSpecElement().add(e); + } + try { + xp.setTimePartitionSpec(new XTimePartSpec()); + xp.setNonTimePartitionSpec(new XPartSpec()); + for (Map.Entry<String, String> entry : partSpec.entrySet()) { + if (timePartCols.contains(entry.getKey())) { + XTimePartSpecElement timePartSpecElement = new XTimePartSpecElement(); + timePartSpecElement.setKey(entry.getKey()); + timePartSpecElement + .setValue(getXMLGregorianCalendar(UpdatePeriod.valueOf(xp.getUpdatePeriod().name()).parse( + entry.getValue()))); + xp.getTimePartitionSpec().getPartSpecElement().add(timePartSpecElement); + } else { + XPartSpecElement partSpecElement = new XPartSpecElement(); + partSpecElement.setKey(entry.getKey()); + partSpecElement.setValue(entry.getValue()); + xp.getNonTimePartitionSpec().getPartSpecElement().add(partSpecElement); + } + } + } catch (ParseException exc) { + log.debug("can't form time part spec from " + partSpec, exc); + xp.setTimePartitionSpec(null); + xp.setNonTimePartitionSpec(null); + } + xp.setSerdeClassname(p.getTPartition().getSd().getSerdeInfo().getSerializationLib()); + xp.getSerdeParameters().getProperty().addAll(xPropertiesFromMap( + p.getTPartition().getSd().getSerdeInfo().getParameters())); + return xp; + } + + public static void updatePartitionFromXPartition(Partition partition, XPartition xp) throws ClassNotFoundException { + partition.getParameters().putAll(mapFromXProperties(xp.getPartitionParameters())); + partition.getTPartition().getSd().getSerdeInfo().setParameters(mapFromXProperties(xp.getSerdeParameters())); + partition.setLocation(xp.getLocation()); + if (xp.getInputFormat() != null) { + partition.setInputFormatClass(Class.forName(xp.getInputFormat()).asSubclass(InputFormat.class)); + } + if (xp.getOutputFormat() != null) { + Class<? extends HiveOutputFormat> outputFormatClass = + Class.forName(xp.getOutputFormat()).asSubclass(HiveOutputFormat.class); + partition.setOutputFormatClass(outputFormatClass); + } + partition.getParameters().put(MetastoreConstants.PARTITION_UPDATE_PERIOD, xp.getUpdatePeriod().name()); + partition.getTPartition().getSd().getSerdeInfo().setSerializationLib(xp.getSerdeClassname()); + } + + public static StoragePartitionDesc storagePartSpecFromXPartition( + XPartition xpart) { + StoragePartitionDesc partDesc = new StoragePartitionDesc( + xpart.getFactOrDimensionTableName(), + timePartSpecfromXTimePartSpec(xpart.getTimePartitionSpec()), + nonTimePartSpecfromXNonTimePartSpec(xpart.getNonTimePartitionSpec()), + UpdatePeriod.valueOf(xpart.getUpdatePeriod().name())); + partDesc.setPartParams(mapFromXProperties(xpart.getPartitionParameters())); + partDesc.setSerdeParams(mapFromXProperties(xpart.getSerdeParameters())); + partDesc.setLocation(xpart.getLocation()); + partDesc.setInputFormat(xpart.getInputFormat()); + partDesc.setOutputFormat(xpart.getOutputFormat()); + partDesc.setSerializationLib(xpart.getSerdeClassname()); + return partDesc; + } + + public static List<StoragePartitionDesc> storagePartSpecListFromXPartitionList( + final XPartitionList xpartList) { + ArrayList<StoragePartitionDesc> ret = new ArrayList<StoragePartitionDesc>(); + for (XPartition xpart : xpartList.getPartition()) { + ret.add(storagePartSpecFromXPartition(xpart)); + } + return ret; + } + + public static Dimension dimensionFromXDimension(XDimension dimension) throws LensException { + Set<CubeDimAttribute> dims = new LinkedHashSet<CubeDimAttribute>(); + for (XDimAttribute xd : dimension.getAttributes().getDimAttribute()) { + dims.add(hiveDimAttrFromXDimAttr(xd)); + } + + Set<ExprColumn> expressions = new LinkedHashSet<ExprColumn>(); + if (dimension.getExpressions() != null && !dimension.getExpressions().getExpression().isEmpty()) { + for (XExprColumn xe : dimension.getExpressions().getExpression()) { + expressions.add(hiveExprColumnFromXExprColumn(xe)); + } + } + + Set<JoinChain> joinchains = new LinkedHashSet<JoinChain>(); + if (dimension.getJoinChains() != null && !dimension.getJoinChains().getJoinChain().isEmpty()) { + for (XJoinChain xj : dimension.getJoinChains().getJoinChain()) { + joinchains.add(joinChainFromXJoinChain(xj)); + } + } + + Map<String, String> properties = mapFromXProperties(dimension.getProperties()); + return new Dimension(dimension.getName(), dims, expressions, joinchains, properties, 0L); + } + + public static XDimension xdimensionFromDimension(Dimension dimension) { + XDimension xd = XCF.createXDimension(); + xd.setName(dimension.getName()); + xd.setAttributes(new XDimAttributes()); + xd.setExpressions(new XExpressions()); + xd.setJoinChains(new XJoinChains()); + xd.setProperties(new XProperties()); + + xd.getProperties().getProperty().addAll(xPropertiesFromMap(((AbstractCubeTable) dimension).getProperties())); + for (CubeDimAttribute cd : dimension.getAttributes()) { + xd.getAttributes().getDimAttribute().add(xDimAttrFromHiveDimAttr(cd, dimension)); + } + + for (ExprColumn ec : dimension.getExpressions()) { + xd.getExpressions().getExpression().add(xExprColumnFromHiveExprColumn(ec)); + } + + for (JoinChain jc : dimension.getJoinChains()) { + xd.getJoinChains().getJoinChain().add(getXJoinChainFromJoinChain(jc)); + } + + return xd; + } + + public static XNativeTable nativeTableFromMetaTable(Table table) { + XNativeTable xtable = XCF.createXNativeTable(); + xtable.setColumns(new XColumns()); + xtable.setName(table.getTableName()); + xtable.setDbname(table.getDbName()); + xtable.setOwner(table.getOwner()); + xtable.setCreatetime(table.getTTable().getCreateTime()); + xtable.setLastAccessTime(table.getTTable().getLastAccessTime()); + xtable.getColumns().getColumn().addAll(columnsFromFieldSchemaList(table.getCols())); + xtable.setStorageDescriptor(getStorageTableDescFromHiveTable(table)); + xtable.setTableType(table.getTableType().name()); + return xtable; + } + + public static Map<String, String> getFullPartSpecAsMap(XPartition partition) { + Map<String, String> spec = Maps.newHashMap(); + if (partition.getTimePartitionSpec() != null) { + for (XTimePartSpecElement timePartSpecElement : partition.getTimePartitionSpec().getPartSpecElement()) { + spec.put(timePartSpecElement.getKey(), UpdatePeriod.valueOf(partition.getUpdatePeriod().name()).format() + .format(getDateFromXML(timePartSpecElement.getValue()))); + } + } + if (partition.getNonTimePartitionSpec() != null) { + for (XPartSpecElement partSpecElement : partition.getNonTimePartitionSpec().getPartSpecElement()) { + spec.put(partSpecElement.getKey(), partSpecElement.getValue()); + } + } + return spec; + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index 25acb01..e6e9f8f 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -635,12 +635,8 @@ public class StorageCandidate implements Candidate, CandidateTable { log.info("Completeness for the measure_tag {} is {}, threshold: {}, for the hour {}", tag, completenessResult.getValue(), completenessThreshold, formatter.format(completenessResult.getKey())); String measureorExprFromTag = tagToMeasureOrExprMap.get(tag); - Map<String, Float> incompletePartition = dataCompletenessMap.get(measureorExprFromTag); - if (incompletePartition == null) { - incompletePartition = new HashMap<>(); - dataCompletenessMap.put(measureorExprFromTag, incompletePartition); - } - incompletePartition.put(formatter.format(completenessResult.getKey()), completenessResult.getValue()); + dataCompletenessMap.computeIfAbsent(measureorExprFromTag, k -> new HashMap<>()) + .put(formatter.format(completenessResult.getKey()), completenessResult.getValue()); isDataComplete = false; } } http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java index bc008ae..22e2e09 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java @@ -59,7 +59,7 @@ class StorageTableResolver implements ContextRewriter { validDimTables = StringUtils.isBlank(str) ? null : Arrays.asList(StringUtils.split(str.toLowerCase(), ",")); String maxIntervalStr = conf.get(CubeQueryConfUtil.QUERY_MAX_INTERVAL); if (maxIntervalStr != null) { - this.maxInterval = UpdatePeriod.valueOf(maxIntervalStr); + this.maxInterval = UpdatePeriod.valueOf(maxIntervalStr.toUpperCase()); } else { this.maxInterval = null; }
