LENS-975 : Add cube segmentation specification
Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/e4f26aae Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/e4f26aae Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/e4f26aae Branch: refs/heads/master Commit: e4f26aae8293432833a6bfca03ebcf79cd71d13d Parents: a2b6125 Author: Amareshwari Sriramadasu <amareshw...@apache.org> Authored: Mon Mar 28 15:42:33 2016 +0530 Committer: Amareshwari Sriramadasu <amareshw...@apache.org> Committed: Mon Mar 28 15:42:33 2016 +0530 ---------------------------------------------------------------------- lens-api/src/main/resources/cube-0.1.xsd | 97 ++++++++++ .../lens/cli/commands/BaseTableCrudCommand.java | 37 ++++ .../LensCubeSegmentationCommands.java | 137 +++++++++++++ .../cli/TestLensCubeSegmentationCommands.java | 156 +++++++++++++++ lens-cli/src/test/resources/seg1.xml | 45 +++++ .../java/org/apache/lens/client/LensClient.java | 25 +++ .../apache/lens/client/LensMetadataClient.java | 86 +++++++++ .../lens/cube/metadata/AbstractCubeTable.java | 25 +++ .../lens/cube/metadata/CubeFactTable.java | 20 +- .../lens/cube/metadata/CubeMetastoreClient.java | 145 ++++++++++++++ .../apache/lens/cube/metadata/CubeSegment.java | 35 ++++ .../lens/cube/metadata/CubeSegmentation.java | 190 +++++++++++++++++++ .../lens/cube/metadata/CubeTableType.java | 2 +- .../lens/cube/metadata/MetastoreConstants.java | 9 + .../lens/cube/metadata/MetastoreUtil.java | 20 ++ .../lens/cube/metadata/CubeFactTableTest.java | 1 + .../cube/metadata/TestCubeMetastoreClient.java | 47 +++++ .../apache/lens/examples/SampleMetastore.java | 11 ++ lens-examples/src/main/resources/seg1.xml | 44 +++++ lens-examples/src/test/resources/yaml/seg1.yaml | 33 ++++ .../lens/server/api/LensConfConstants.java | 5 + .../api/metastore/CubeMetastoreService.java | 49 +++++ .../metastore/CubeMetastoreServiceImpl.java | 84 ++++++++ .../apache/lens/server/metastore/JAXBUtils.java | 75 ++++++++ .../server/metastore/MetastoreResource.java | 157 ++++++++++++++- .../server/metastore/TestMetastoreService.java | 147 ++++++++++++++ 26 files changed, 1661 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-api/src/main/resources/cube-0.1.xsd ---------------------------------------------------------------------- diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd index 0f4dbea..f221a6d 100644 --- a/lens-api/src/main/resources/cube-0.1.xsd +++ b/lens-api/src/main/resources/cube-0.1.xsd @@ -1010,6 +1010,39 @@ </xs:sequence> </xs:complexType> + <xs:complexType name="x_cube_segment"> + <xs:annotation> + <xs:documentation> + cube segment belong to the segmentation + </xs:documentation> + </xs:annotation> + <xs:sequence> + <xs:element name="segment_parameters" type="x_properties" minOccurs="0" maxOccurs="1"> + <xs:annotation> + <xs:documentation> + Segment properties. + The following properties can be specified at a cube segment level : + 1. lens.metastore.cube.column.mapping : The column mapping for columns of cube segment if they are + different in underlying cube. The value is speciified with comma separated map entries specified with + key-values separated by equalto. Example value: id=id1,name=name1 + </xs:documentation> + </xs:annotation> + </xs:element> + </xs:sequence> + <xs:attribute name="cube_name" type="xs:string" use="required"/> + </xs:complexType> + + <xs:complexType name="x_cube_segments"> + <xs:annotation> + <xs:documentation> + Segments in segmentation. There can be two or more such cube segments. + </xs:documentation> + </xs:annotation> + <xs:sequence> + <xs:element name="cube_segment" minOccurs="2" maxOccurs="unbounded" type="x_cube_segment"/> + </xs:sequence> + </xs:complexType> + <xs:element name="x_partition_list" type="x_partition_list"/> <xs:complexType name="x_partition_list"> <xs:annotation> @@ -1191,6 +1224,9 @@ e.g. now.year or now.day - 6 hour etc. 7. cube.fact.absolute.end.time: If you're deprecating this fact, put the final date till which the data of the fact will be valid here. Format same as absolute start time. + 8. cube.fact.relative.end.time: You can specify the end date for fact table + relative to current date e.g. fact table is valid for next 90days starting from today. + This can be specified as just a time difference e.g. "+90 days" </xs:documentation> </xs:annotation> </xs:element> @@ -1225,6 +1261,67 @@ </xs:attribute> </xs:complexType> + <xs:element name="x_cube_segmentation" type="x_cube_segmentation"/> + + <xs:complexType name="x_cube_segmentation"> + <xs:annotation> + <xs:documentation> + Cube segmentation is associated to a base cube and consists of two or more cube segments. All + cube segments together make cube segmentation complete. + </xs:documentation> + </xs:annotation> + <xs:sequence> + <xs:element type="x_properties" name="properties" maxOccurs="1" minOccurs="0"> + <xs:annotation> + <xs:documentation> + Properties that can be set for cubesegmentation are + 1. cube.segmentation.absolute.start.time: start time of the cube segmentation. + For queries that ask for time before this, this cubesegmentation is not a candidate. Time format + can be specified in yyyy[-mm[-dd[-hh[:MM[:ss[,SSS]]]]]] format. + 2. cube.segmentation.relative.start.time: Here you can specify cubesegmentations's relative validity + relative to current time. Useful if you want to specify e.g. this cubesegmentation is valid + for today - 90 days. Can be specified as just a time difference e.g. "-90 days". + Or can be specified in relative syntax. + e.g. now.year or now.day - 6 hour etc. + 3. cube.segmentation.absolute.end.time: If you're deprecating a cubesegmentation, put the final date till + which the data of the cubesegmentation will be valid here. Format same as absolute start time. + 4. cube.segmentation.relative.end.time: You can specify the end date for cubesegmentation + relative to current date e.g. cubesegmentationas is valid for next 90 days starting from today. + This can be specified as just a time difference e.g. "+90 days". + </xs:documentation> + </xs:annotation> + </xs:element> + <xs:element name="cube_segements" type="x_cube_segments" maxOccurs="1" minOccurs="0"/> + </xs:sequence> + <xs:attribute name="name" type="xs:string" use="required"> + <xs:annotation> + <xs:documentation> + Name of cube segmentation + </xs:documentation> + </xs:annotation> + </xs:attribute> + <xs:attribute name="cube_name" type="xs:string" use="required"> + <xs:annotation> + <xs:documentation> + The base or parent cube to which the cube segmentation is associated. + </xs:documentation> + </xs:annotation> + </xs:attribute> + <xs:attribute name="weight" use="required" > + <xs:annotation> + <xs:documentation> + The weight of the cube segmentation. LENS will use this attribute to decide the lightest table to + query when there are more than one eligible tables. + </xs:documentation> + </xs:annotation> + <xs:simpleType> + <xs:restriction base="xs:double"> + <xs:minInclusive value="0"></xs:minInclusive> + </xs:restriction> + </xs:simpleType> + </xs:attribute> + </xs:complexType> + <xs:element name="x_native_table" type="x_native_table"/> <xs:complexType name="x_native_table"> <xs:annotation> http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseTableCrudCommand.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseTableCrudCommand.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseTableCrudCommand.java new file mode 100644 index 0000000..767a78c --- /dev/null +++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseTableCrudCommand.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cli.commands; + +import java.util.List; + +import com.google.common.base.Joiner; + + +public abstract class BaseTableCrudCommand<T> extends LensCRUDCommand<T> { + public String showAll(String filter) { + List<String> all = getAll(filter); + if (all == null || all.isEmpty()) { + return "No " + getSingleObjectName() + " found" + (filter == null ? "" : " for " + filter); + } + return Joiner.on("\n").join(all); + } + + protected abstract List<String> getAll(String filter); + +} http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cli/src/main/java/org/apache/lens/cli/commands/annotations/LensCubeSegmentationCommands.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/annotations/LensCubeSegmentationCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/annotations/LensCubeSegmentationCommands.java new file mode 100644 index 0000000..ed2ad7a --- /dev/null +++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/annotations/LensCubeSegmentationCommands.java @@ -0,0 +1,137 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cli.commands.annotations; + + +import java.io.File; +import java.util.List; + +import org.apache.lens.api.APIResult; +import org.apache.lens.api.metastore.XCubeSegmentation; +import org.apache.lens.cli.commands.BaseTableCrudCommand; + +import org.springframework.shell.core.annotation.CliCommand; +import org.springframework.shell.core.annotation.CliOption; +import org.springframework.stereotype.Component; + +import lombok.NonNull; + +@Component +@UserDocumentation(title = "Commands for CubeSegmentation Management", + description = "These command provide CRUD for CubeSegmentation") +public class LensCubeSegmentationCommands extends BaseTableCrudCommand<XCubeSegmentation> { + + /** + * Show cube segmentation + * + * @return the string + */ + @CliCommand(value = "show cubesegmentations", + help = "display list of cubesegmentations in current database. " + + "If optional <cube_name> is supplied, only cubesegmentations " + + "belonging to cube <cube_name> will be displayed") + public String showCubeSegmentations( + @CliOption(key = {"", "cube_name"}, mandatory = false, help = "<cube_name>") String cubeName) { + return showAll(cubeName); + } + /** + * Creates the cubesegmentation + * + * @param path the cubesegmentation spec + * @return the string + */ + @CliCommand(value = "create cubesegmentation", + help = "create a new cubesegmentation, taking spec from <path-to-cubesegmentation-spec file>") + public String createCubeSegmentation( + @CliOption(key = {"", "path"}, mandatory = true, help = + "<path-to-cubesegmentation-spec file>") @NonNull final File path) { + return create(path, false); + } + + /** + * Describe cubesegmentation. + * + * @param name the cubesegmentation name + * @return the string + */ + @CliCommand(value = "describe cubesegmentation", help = "describe cubesegmentation <cubesegmentation_name>") + public String describeCubeSegmentation( + @CliOption(key = {"", "name"}, mandatory = true, help = "<cubesegmentation_name>") String name) { + return formatJson(getClient().getCubeSegmentation(name)); + } + + + /** + * Update cubesegmentation. + * + * @param name the cubesegmentation to be updated + * @param path path to spec file + * @return the string + */ + @CliCommand(value = "update cubesegmentation", + help = "update cubesegmentation <cubesegmentation_name>, taking spec from <path-to-cubesegmentation-spec file>") + public String updateCubeSegmentation( + @CliOption(key = {"", "name"}, mandatory = true, help = "<cubesegmentation_name>") String name, + @CliOption(key = {"", "path"}, mandatory = true, help = "<path-to-cubesegmentation-spec-file>") + @NonNull final File path) { + return update(name, path); + } + + /** + * Drop cubesegmentation. + * + * @param name the cubesegmentation to be dropped + * @return the string + */ + @CliCommand(value = "drop cubesegmentation", help = "drop cubesegmentation <cubesegmentation_name>") + public String dropCubeSegmentation( + @CliOption(key = {"", "name"}, mandatory = true, help = "<cubesegmentation_name>") String name) { + return drop(name, false); + } + + @Override + public List<String> getAll() { + return getClient().getAllCubeSegmentations(); + } + + @Override + protected APIResult doCreate(String path, boolean ignoreIfExists) { + return getClient().createCubeSegmentation(path); + } + + @Override + protected XCubeSegmentation doRead(String name) { + return getClient().getCubeSegmentation(name); + } + + @Override + public APIResult doUpdate(String name, String path) { + return getClient().updateCubeSegmentation(name, path); + } + + @Override + protected APIResult doDelete(String name, boolean cascade) { + return getClient().dropCubeSegmentation(name); + } + + @Override + protected List<String> getAll(String filter) { + return getClient().getAllCubeSegmentations(filter); + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeSegmentationCommands.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeSegmentationCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeSegmentationCommands.java new file mode 100644 index 0000000..53a52de --- /dev/null +++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeSegmentationCommands.java @@ -0,0 +1,156 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cli; + +import static org.testng.Assert.*; + +import java.io.*; +import java.net.URISyntaxException; +import java.net.URL; + +import javax.ws.rs.NotFoundException; + +import org.apache.lens.cli.commands.LensCubeCommands; +import org.apache.lens.cli.commands.annotations.LensCubeSegmentationCommands; +import org.apache.lens.client.LensClient; + +import org.testng.annotations.Test; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class TestLensCubeSegmentationCommands extends LensCliApplicationTest { + + private static LensCubeSegmentationCommands command = null; + private static LensCubeCommands cubeCommands = null; + + private void createSampleCube() throws URISyntaxException { + URL cubeSpec = TestLensCubeCommands.class.getClassLoader().getResource("sample-cube.xml"); + String cubeList = getCubeCommand().showCubes(); + assertFalse(cubeList.contains("sample_cube"), cubeList); + getCubeCommand().createCube(new File(cubeSpec.toURI())); + cubeList = getCubeCommand().showCubes(); + assertTrue(cubeList.contains("sample_cube"), cubeList); + } + + private static LensCubeSegmentationCommands getCommand() { + if (command == null) { + LensClient client = new LensClient(); + command = new LensCubeSegmentationCommands(); + command.setClient(client); + } + return command; + } + + private static LensCubeCommands getCubeCommand() { + if (cubeCommands == null) { + LensClient client = new LensClient(); + cubeCommands = new LensCubeCommands(); + cubeCommands.setClient(client); + } + return cubeCommands; + } + + public static void testCreateSegmentation() throws IOException { + LensCubeSegmentationCommands command = getCommand(); + String segList = command.showCubeSegmentations(null); + assertEquals(command.showCubeSegmentations("sample_cube"), "No cubesegmentation found for sample_cube"); + assertEquals(segList, "No cubesegmentation found"); + URL segSpec = TestLensCubeSegmentationCommands.class.getClassLoader().getResource("seg1.xml"); + try { + command.createCubeSegmentation(new File(segSpec.toURI())); + } catch (Exception e) { + fail("Unable to create cubesegmentation" + e.getMessage()); + } + segList = command.showCubeSegmentations(null); + assertEquals(command.showCubeSegmentations("sample_cube"), segList); + try { + assertEquals(command.showCubeSegmentations("blah"), segList); + fail(); + } catch (NotFoundException e) { + log.info("blah is not a cubesegmentation", e); + } + } + + public static void testUpdateSegmentation() { + try { + LensCubeSegmentationCommands command = getCommand(); + URL segSpec = TestLensCubeSegmentationCommands.class.getClassLoader().getResource("seg1.xml"); + StringBuilder sb = new StringBuilder(); + BufferedReader bufferedReader = new BufferedReader(new FileReader(segSpec.getFile())); + String s; + while ((s = bufferedReader.readLine()) != null) { + sb.append(s).append("\n"); + } + + bufferedReader.close(); + String xmlContent = sb.toString(); + xmlContent = xmlContent.replace("<property name=\"seg1.prop\" value=\"s1\"/>\n", + "<property name=\"seg1.prop\" value=\"s1\"/>" + "\n<property name=\"seg1.prop1\" value=\"s2\"/>\n"); + + File newFile = new File("target/seg2.xml"); + Writer writer = new OutputStreamWriter(new FileOutputStream(newFile)); + writer.write(xmlContent); + writer.close(); + + String desc = command.describeCubeSegmentation("seg1"); + log.debug(desc); + String propString = "seg1.prop: s1"; + String propString1 = "seg1.prop1: s2"; + + assertTrue(desc.contains(propString)); + + command.updateCubeSegmentation("seg1", new File("target/seg2.xml")); + desc = command.describeCubeSegmentation("seg1"); + log.debug(desc); + assertTrue(desc.contains(propString), "The sample property value is not set"); + assertTrue(desc.contains(propString1), "The sample property value is not set"); + + newFile.delete(); + + } catch (Throwable t) { + log.error("Updating of the cubesegmentation failed with ", t); + fail("Updating of the cubesegmentation failed with " + t.getMessage()); + } + + } + + public static void testDropSegmentation() { + LensCubeSegmentationCommands command = getCommand(); + String segList = command.showCubeSegmentations(null); + assertEquals("seg1", segList, "seg1 segmentation should be found"); + command.dropCubeSegmentation("seg1"); + segList = command.showCubeSegmentations(null); + assertEquals(segList, "No cubesegmentation found"); + } + + private void dropSampleCube() { + getCubeCommand().dropCube("sample_cube"); + } + + @Test + public void testCubeSegmentationCommands() throws IOException, URISyntaxException { + createSampleCube(); + testCreateSegmentation(); + testUpdateSegmentation(); + testDropSegmentation(); + dropSampleCube(); + } + +} http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cli/src/test/resources/seg1.xml ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/resources/seg1.xml b/lens-cli/src/test/resources/seg1.xml new file mode 100644 index 0000000..848ac10 --- /dev/null +++ b/lens-cli/src/test/resources/seg1.xml @@ -0,0 +1,45 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<x_cube_segmentation cube_name="sample_cube" name="seg1" weight="100.0" xmlns="uri:lens:cube:0.1" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd "> + <properties> + <property name="seg1.prop" value="s1"/> + <property name="cube.segmentation.relative.start.time" value="now -10days"/> + </properties> + <cube_segements> + <cube_segment cube_name="cube11"> + <segment_parameters> + <property name="lens.metastore.cube.column.mapping" value="foo=bar"/> + </segment_parameters> + </cube_segment> + <cube_segment cube_name="cube22"> + <segment_parameters> + <property name="lens.metastore.cube.column.mapping" value="foo1=bar1"/> + </segment_parameters> + </cube_segment> + <cube_segment cube_name="cube33"> + <segment_parameters> + <property name="lens.metastore.cube.column.mapping" value="foo2=bar2"/> + </segment_parameters> + </cube_segment> + </cube_segements> +</x_cube_segmentation> http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-client/src/main/java/org/apache/lens/client/LensClient.java ---------------------------------------------------------------------- diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClient.java b/lens-client/src/main/java/org/apache/lens/client/LensClient.java index ea0cd76..39d771c 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensClient.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensClient.java @@ -612,4 +612,29 @@ public class LensClient { public Response getLogs(String logFile) { return this.connection.getLogs(logFile); } + + public XCubeSegmentation getCubeSegmentation(String segName) { + return mc.getCubeSegmentation(segName); + } + + public List<String> getAllCubeSegmentations() { + return mc.getAllCubeSegmentations(); + } + + public List<String> getAllCubeSegmentations(String filter) { + return mc.getAllCubeSegmentations(filter); + } + + public APIResult createCubeSegmentation(String segSpec) { + return mc.createCubeSegmentation(segSpec); + } + + public APIResult updateCubeSegmentation(String segName, String segSpec) { + return mc.updateCubeSegmentation(segName, segSpec); + } + + public APIResult dropCubeSegmentation(String segName) { + return mc.dropCubeSegmentation(segName); + } + } http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java ---------------------------------------------------------------------- diff --git a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java index d0f2b57..f2d42ca 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java @@ -359,6 +359,20 @@ public class LensMetadataClient { return factTables.getElements(); } + public List<String> getAllCubeSegmentations(String cubeName) { + if (cubeName == null) { + return getAllCubeSegmentations(); + } + WebTarget target = getMetastoreWebTarget(); + StringList cubeSegmentations; + cubeSegmentations = target.path("cubes").path(cubeName).path("cubesegmentations") + .queryParam("sessionid", this.connection.getSessionHandle()) + .request(MediaType.APPLICATION_XML) + .get(StringList.class); + return cubeSegmentations.getElements(); + } + + public List<String> getAllFactTables() { WebTarget target = getMetastoreWebTarget(); StringList factTables; @@ -370,6 +384,17 @@ public class LensMetadataClient { return factTables.getElements(); } + public List<String> getAllCubeSegmentations() { + WebTarget target = getMetastoreWebTarget(); + StringList cubeSegmentations; + cubeSegmentations = target.path("cubesegmentations") + .queryParam("sessionid", this.connection.getSessionHandle()) + .request(MediaType.APPLICATION_XML) + .get(StringList.class); + + return cubeSegmentations.getElements(); + } + public APIResult deleteAllFactTables(boolean cascade) { WebTarget target = getMetastoreWebTarget(); @@ -381,6 +406,14 @@ public class LensMetadataClient { } + public APIResult deleteAllCubeSegmentations() { + WebTarget target = getMetastoreWebTarget(); + return target.path("cubesegmentations") + .queryParam("sessionid", this.connection.getSessionHandle()) + .request(MediaType.APPLICATION_XML) + .delete(APIResult.class); + } + public XFactTable getFactTable(String factTableName) { WebTarget target = getMetastoreWebTarget(); JAXBElement<XFactTable> table = target.path("facts").path(factTableName) @@ -391,6 +424,16 @@ public class LensMetadataClient { return table.getValue(); } + public XCubeSegmentation getCubeSegmentation(String segName) { + WebTarget target = getMetastoreWebTarget(); + JAXBElement<XCubeSegmentation> seg = target.path("cubesegmentations").path(segName) + .queryParam("sessionid", this.connection.getSessionHandle()) + .request(MediaType.APPLICATION_XML) + .get(new GenericType<JAXBElement<XCubeSegmentation>>() { + }); + return seg.getValue(); + } + public APIResult createFactTable(XFactTable f) { WebTarget target = getMetastoreWebTarget(); return target.path("facts") @@ -407,6 +450,23 @@ public class LensMetadataClient { } } + public APIResult createCubeSegmentation(XCubeSegmentation seg) { + WebTarget target = getMetastoreWebTarget(); + return target.path("cubesegmentations") + .queryParam("sessionid", this.connection.getSessionHandle()) + .request(MediaType.APPLICATION_XML) + .post(Entity.xml(new GenericEntity<JAXBElement<XCubeSegmentation>>(objFact + .createXCubeSegmentation(seg)){}), APIResult.class); + } + + public APIResult createCubeSegmentation(String segSpec) { + try { + return createCubeSegmentation(this.<XCubeSegmentation>readFromXML(segSpec)); + } catch (JAXBException | IOException e) { + return failureAPIResult(e); + } + } + public APIResult updateFactTable(String factName, XFactTable table) { WebTarget target = getMetastoreWebTarget(); return target.path("facts").path(factName) @@ -423,6 +483,24 @@ public class LensMetadataClient { } } + public APIResult updateCubeSegmentation(String segName, XCubeSegmentation seg) { + WebTarget target = getMetastoreWebTarget(); + return target.path("cubesegmentations").path(segName) + .queryParam("sessionid", this.connection.getSessionHandle()) + .request(MediaType.APPLICATION_XML_TYPE) + .put(Entity.xml(new GenericEntity<JAXBElement<XCubeSegmentation>>(objFact. + createXCubeSegmentation(seg)){}), APIResult.class); + } + + public APIResult updateCubeSegmentation(String segName, String seg) { + try { + return updateCubeSegmentation(segName, this.<XCubeSegmentation>readFromXML(seg)); + } catch (JAXBException | IOException e) { + return failureAPIResult(e); + } + } + + public APIResult dropFactTable(String factName, boolean cascade) { WebTarget target = getMetastoreWebTarget(); return target.path("facts").path(factName) @@ -436,6 +514,14 @@ public class LensMetadataClient { return dropFactTable(factName, false); } + public APIResult dropCubeSegmentation(String segName) { + WebTarget target = getMetastoreWebTarget(); + return target.path("cubesegmentations").path(segName) + .queryParam("sessionid", this.connection.getSessionHandle()) + .request(MediaType.APPLICATION_XML) + .delete(APIResult.class); + } + public List<String> getAllStoragesOfFactTable(String factName) { WebTarget target = getMetastoreWebTarget(); StringList storageList = target.path("facts").path(factName).path("storages") http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java index 01098c4..67aaff8 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java @@ -20,6 +20,9 @@ package org.apache.lens.cube.metadata; import java.util.*; +import org.apache.lens.server.api.error.LensException; + +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.metadata.Table; @@ -179,6 +182,23 @@ public abstract class AbstractCubeTable implements Named { return true; } + public Date getDateFromProperty(String propKey, boolean relative, boolean start) { + String prop = getProperties().get(propKey); + try { + if (StringUtils.isNotBlank(prop)) { + if (relative) { + return DateUtil.resolveRelativeDate(prop, now()); + } else { + return DateUtil.resolveAbsoluteDate(prop); + } + } + } catch (LensException e) { + log.error("unable to parse {} {} date: {}", relative ? "relative" : "absolute", start ? "start" : "end", prop); + } + return start ? DateUtil.MIN_DATE : DateUtil.MAX_DATE; + } + + @Override public String toString() { return getName(); @@ -200,4 +220,9 @@ public abstract class AbstractCubeTable implements Named { } return columns; } + + public Date now() { + return new Date(); + } + } http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java index 643bcfe..fb958c3 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java @@ -323,22 +323,6 @@ public class CubeFactTable extends AbstractCubeTable { getProperties().put(MetastoreConstants.FACT_AGGREGATED_PROPERTY, Boolean.toString(isAggregated)); } - public Date getDateFromProperty(String propKey, boolean relative, boolean start) { - String prop = getProperties().get(propKey); - try { - if (StringUtils.isNotBlank(prop)) { - if (relative) { - return DateUtil.resolveRelativeDate(prop, now()); - } else { - return DateUtil.resolveAbsoluteDate(prop); - } - } - } catch (LensException e) { - log.error("unable to parse {} {} date: {}", relative ? "relative" : "absolute", start ? "start" : "end", prop); - } - return start ? DateUtil.MIN_DATE : DateUtil.MAX_DATE; - } - public Date getAbsoluteStartTime() { return getDateFromProperty(MetastoreConstants.FACT_ABSOLUTE_START_TIME, false, true); } @@ -363,7 +347,5 @@ public class CubeFactTable extends AbstractCubeTable { return Collections.min(Lists.newArrayList(getRelativeEndTime(), getAbsoluteEndTime())); } - public Date now() { - return new Date(); - } + } http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 02bbbbd..a29f42e 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -80,6 +80,9 @@ public class CubeMetastoreClient { // map from fact name to fact table private final Map<String, CubeFactTable> allFactTables = Maps.newConcurrentMap(); private volatile boolean allFactTablesPopulated = false; + //map from segmentation name to segmentation + private final Map<String, CubeSegmentation> allCubeSegmentations = Maps.newConcurrentMap(); + private volatile boolean allCubeSegmentationPopulated = false; // map from storage name to storage private final Map<String, Storage> allStorages = Maps.newConcurrentMap(); private volatile boolean allStoragesPopulated = false; @@ -715,6 +718,24 @@ public class CubeMetastoreClient { } /** + * + * @param baseCubeName The cube name ot which segmentation belong to + * @param segmentationName The segmentation name + * @param cubeSegments Participating cube segements + * @param weight Weight of segmentation + * @param properties Properties of segmentation + * @throws HiveException + */ + public void createCubeSegmentation(String baseCubeName, String segmentationName, Set<CubeSegment> cubeSegments, + double weight, Map<String, String> properties) throws HiveException { + CubeSegmentation cubeSeg = + new CubeSegmentation(baseCubeName, segmentationName, cubeSegments, weight, properties); + createCubeSegmentation(cubeSeg); + // do a get to update cache + getCubeSegmentation(segmentationName); + } + + /** * Create a cube dimension table * * @param dimName The dimension name to which the dim-table belongs to @@ -777,6 +798,12 @@ public class CubeMetastoreClient { } } + public void createCubeSegmentation(CubeSegmentation cubeSeg) + throws HiveException { + // create virtual cube table in metastore + createCubeHiveTable(cubeSeg); + } + /** * Adds storage to fact and creates corresponding storage table * @@ -1402,6 +1429,17 @@ public class CubeMetastoreClient { return CubeTableType.FACT.name().equals(tableType); } + public boolean isCubeSegmentation(String segName) throws HiveException { + Table tbl = getTable(segName); + return isCubeSegmentation(tbl); + } + + boolean isCubeSegmentation(Table tbl) { + String tableType = tbl.getParameters().get(MetastoreConstants.TABLE_TYPE_KEY); + return CubeTableType.SEGMENTATION.name().equals(tableType); + } + + boolean isFactTableForCube(Table tbl, String cube) { return isFactTable(tbl) && CubeFactTable.getCubeName(tbl.getTableName(), tbl.getParameters()) .equalsIgnoreCase(cube.toLowerCase()); @@ -1526,6 +1564,14 @@ public class CubeMetastoreClient { return isFactTable(tbl) ? new CubeFactTable(tbl) : null; } + public CubeSegmentation getCubeSegmentationTable(String tableName) throws HiveException { + return getCubeSegmentationTable(getTable(tableName)); + } + + private CubeSegmentation getCubeSegmentationTable(Table tbl) throws HiveException { + return isCubeSegmentation(tbl) ? new CubeSegmentation(tbl) : null; + } + /** * Get {@link CubeDimensionTable} object corresponding to the name * @@ -1690,6 +1736,25 @@ public class CubeMetastoreClient { return fact; } + public CubeSegmentation getCubeSegmentation(String segName) throws HiveException { + segName = segName.trim().toLowerCase(); + CubeSegmentation seg = allCubeSegmentations.get(segName); + if (seg == null) { + synchronized (allCubeSegmentations) { + if (!allCubeSegmentations.containsKey(segName)) { + seg = getCubeSegmentationTable(segName); + if (enableCaching && seg != null) { + allCubeSegmentations.put(segName, seg); + } + } else { + seg = allCubeSegmentations.get(segName); + } + } + } + return seg; + } + + private CubeInterface getCube(Table tbl) throws HiveException { String parentCube = tbl.getParameters().get(getParentCubeNameKey(tbl.getTableName())); if (parentCube != null) { @@ -1833,6 +1898,34 @@ public class CubeMetastoreClient { } } + /** + * Get all cube segmentations in metastore + * + * @return List of cube segmentation objects + * @throws HiveException + */ + public Collection<CubeSegmentation> getAllCubeSegmentations() throws HiveException { + if (!allCubeSegmentationPopulated) { + List<CubeSegmentation> segs = new ArrayList<>(); + try { + for (String table : getAllHiveTableNames()) { + CubeSegmentation seg = getCubeSegmentation(table); + if (seg != null) { + segs.add(seg); + } + } + } catch (HiveException e) { + throw new HiveException("Could not get all fact tables", e); + } + allFactTablesPopulated = enableCaching; + return segs; + } else { + return allCubeSegmentations.values(); + } + } + + + private Collection<String> getAllHiveTableNames() throws HiveException { if (!allTablesPopulated) { List<String> allTables = getClient().getAllTables(); @@ -1875,6 +1968,28 @@ public class CubeMetastoreClient { return cubeFacts; } + public List<CubeSegmentation> getAllCubeSegmentations(CubeInterface cube) throws HiveException { + String cubeName = null; + if (cube != null) { + if (cube instanceof DerivedCube) { + cube = ((DerivedCube) cube).getParent(); + } + cubeName = cube.getName(); + } + List<CubeSegmentation> cubeSegs = new ArrayList<>(); + try { + for (CubeSegmentation seg : getAllCubeSegmentations()) { + if (cubeName == null || seg.getBaseCube().equalsIgnoreCase(cubeName)) { + cubeSegs.add(seg); + } + } + } catch (HiveException e) { + throw new HiveException("Could not get all segmentations of " + cube, e); + } + return cubeSegs; + } + + /** * Get all derived cubes of the cube, that have all fields queryable together * @@ -2093,6 +2208,16 @@ public class CubeMetastoreClient { } } + + public void dropCubeSegmentation(String segName) throws HiveException { + if (isCubeSegmentation(segName)) { + dropHiveTable(segName); + allCubeSegmentations.remove(segName.trim().toLowerCase()); + } else { + throw new HiveException(segName + " is not a CubeSegmentation"); + } + } + /** * Drop a storage from fact * @@ -2191,6 +2316,26 @@ public class CubeMetastoreClient { } } + public void alterCubeSegmentation(String segName, CubeSegmentation seg) + throws HiveException { + Table segTbl = getTable(segName); + if (isCubeSegmentation(segTbl)) { + if (!(getCubeSegmentation(segName) == seg)) { + dropCubeSegmentation(segName); + createCubeSegmentation(seg); + updateSegmentationCache(segName); + } + } else { + throw new HiveException(segName + " is not a cube segment"); + } + } + + private void updateSegmentationCache(String segmentName) throws HiveException { + if (enableCaching) { + allCubeSegmentations.put(segmentName.trim().toLowerCase(), getCubeSegmentationTable(refreshTable(segmentName))); + } + } + private void updateFactCache(String factTableName) throws HiveException { if (enableCaching) { allFactTables.put(factTableName.trim().toLowerCase(), getFactTable(refreshTable(factTableName))); http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeSegment.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeSegment.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeSegment.java new file mode 100644 index 0000000..8143c3b --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeSegment.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.metadata; + +import java.util.Map; + +import lombok.Getter; + +public class CubeSegment implements Named { + @Getter + private String name; + @Getter + private Map<String, String> properties; + + public CubeSegment(String name, Map<String, String> properties) { + this.name = name; + this.properties = properties; + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeSegmentation.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeSegmentation.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeSegmentation.java new file mode 100644 index 0000000..a93fafe --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeSegmentation.java @@ -0,0 +1,190 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lens.cube.metadata; + +import java.util.*; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.metadata.Table; + +import com.google.common.collect.Lists; + +public class CubeSegmentation extends AbstractCubeTable { + + private String cubeName; + private final Set<CubeSegment> cubeSegments; + private static final List<FieldSchema> COLUMNS = new ArrayList<>(); + + static { + COLUMNS.add(new FieldSchema("dummy", "string", "dummy column")); + } + + public CubeSegmentation(Table hiveTable) { + super(hiveTable); + this.cubeSegments = getCubeSegments(getName(), getProperties()); + this.cubeName = getCubeName(getName(), getProperties()); + } + + public CubeSegmentation(String cubeName, String segmentName, Set<CubeSegment> cubeSegments) { + this(cubeName, segmentName, cubeSegments, 0L); + } + + public CubeSegmentation(String cubeName, String segmentName, Set<CubeSegment> cubeSegments, double weight) { + this(cubeName, segmentName, cubeSegments, weight, new HashMap<String, String>()); + } + + public CubeSegmentation(String baseCube, String segmentName, Set<CubeSegment> cubeSegments, + double weight, Map<String, String> properties) { + super(segmentName, COLUMNS, properties, weight); + this.cubeName = baseCube; + this.cubeSegments = cubeSegments; + addProperties(); + } + + public static Set<String> getSegmentNames(Set<CubeSegment> segments) { + Set<String> names = new HashSet<>(); + for (CubeSegment seg : segments) { + names.add(seg.getName()); + } + return names; + } + + @Override + protected void addProperties() { + super.addProperties(); + addCubeName(getName(), getProperties(), cubeName); + addCubeSegmentProperties(getName(), getProperties(), cubeSegments); + } + + private static void addCubeName(String segName, Map<String, String> props, String cubeName) { + props.put(MetastoreUtil.getSegmentationCubeNameKey(segName), cubeName); + } + + private static void addCubeSegmentProperties(String name, Map<String, String> props, + Set<CubeSegment> cubeSegments) { + if (cubeSegments != null){ + props.put(MetastoreUtil.getSegmentsListKey(name), + MetastoreUtil.getNamedStr(cubeSegments)); + for (CubeSegment cubeSegment : cubeSegments) { + for (Map.Entry<String, String> segProp : cubeSegment.getProperties().entrySet()) { + if (!segProp.getKey().startsWith(MetastoreUtil.getSegmentPropertyKey(cubeSegment.getName()))) { + props.put(MetastoreUtil.getSegmentPropertyKey(cubeSegment.getName()).concat(segProp.getKey()), + segProp.getValue()); + } + } + } + } + } + + private static Set<CubeSegment> getCubeSegments(String name, Map<String, String> props) { + Set<CubeSegment> cubeSegments = new HashSet<>(); + String segmentsString = MetastoreUtil.getNamedStringValue(props, MetastoreUtil.getSegmentsListKey(name)); + if (!StringUtils.isBlank(segmentsString)) { + String[] segments = segmentsString.split(","); + for (String seg : segments) { + Map<String, String> segProps = new HashMap<>(); + String segmentPropStr = MetastoreUtil.getSegmentPropertyKey(seg); + for (String key : props.keySet()) { + if (key.startsWith(segmentPropStr)){ + segProps.put(key, props.get(key)); + } + } + cubeSegments.add(new CubeSegment(seg, segProps)); + } + } + return cubeSegments; + } + + public void addCubeSegment(CubeSegment cubeSeg) { + if (!cubeSegments.contains(cubeSeg)) { + cubeSegments.add(cubeSeg); + addCubeSegmentProperties(getName(), getProperties(), cubeSegments); + } + } + + public void dropCubeSegment(CubeSegment cubeSeg) { + if (cubeSegments.contains(cubeSeg)) { + cubeSegments.remove(cubeSeg); + addCubeSegmentProperties(getName(), getProperties(), cubeSegments); + } + } + + public void alterCubeSegment(Set<CubeSegment> cubeSegs) { + if (!cubeSegments.equals(cubeSegs)) { + cubeSegments.clear(); + cubeSegments.addAll(cubeSegs); + addCubeSegmentProperties(getName(), getProperties(), cubeSegments); + } + } + + public void alterBaseCubeName(String cubeName) { + this.cubeName = cubeName; + addCubeName(getName(), getProperties(), cubeName); + } + + public String getBaseCube() { + return cubeName; + } + + public Set<CubeSegment> getCubeSegments() { + return cubeSegments; + } + + @Override + public CubeTableType getTableType() { + return CubeTableType.SEGMENTATION; + } + + @Override + public Set<String> getStorages() { + return null; + } + + + public Date getAbsoluteStartTime() { + return getDateFromProperty(MetastoreConstants.SEGMENTATION_ABSOLUTE_START_TIME, false, true); + } + + public Date getRelativeStartTime() { + return getDateFromProperty(MetastoreConstants.SEGMENTATION_RELATIVE_START_TIME, true, true); + } + + public Date getStartTime() { + return Collections.max(Lists.newArrayList(getRelativeStartTime(), getAbsoluteStartTime())); + } + + public Date getAbsoluteEndTime() { + return getDateFromProperty(MetastoreConstants.SEGMENTATION_ABSOLUTE_END_TIME, false, false); + } + + public Date getRelativeEndTime() { + return getDateFromProperty(MetastoreConstants.SEGMENTATION_RELATIVE_END_TIME, true, false); + } + + public Date getEndTime() { + return Collections.min(Lists.newArrayList(getRelativeEndTime(), getAbsoluteEndTime())); + } + static String getCubeName(String segName, Map<String, String> props) { + return props.get(MetastoreUtil.getSegmentationCubeNameKey(segName)); + } + + +} http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeTableType.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeTableType.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeTableType.java index ec1c837..6dd35b3 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeTableType.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeTableType.java @@ -20,5 +20,5 @@ package org.apache.lens.cube.metadata; public enum CubeTableType { - CUBE, DIMENSION, FACT, DIM_TABLE, STORAGE + CUBE, DIMENSION, FACT, DIM_TABLE, STORAGE, SEGMENTATION } http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java index ed78d74..cd4862c 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java @@ -56,6 +56,15 @@ public final class MetastoreConstants { public static final String FACT_ABSOLUTE_END_TIME = "cube.fact.absolute.end.time"; public static final String FACT_RELATIVE_END_TIME = "cube.fact.relative.end.time"; + // cube segmentation constants + public static final String SEGMENTATION_KEY_PFX = "cube.segmentation.internal."; + public static final String SEGMENTATION_ABSOLUTE_START_TIME = "cube.segmentation.absolute.start.time"; + public static final String SEGMENTATION_RELATIVE_START_TIME = "cube.segmentation.relative.start.time"; + public static final String SEGMENTATION_ABSOLUTE_END_TIME = "cube.segmentation.absolute.end.time"; + public static final String SEGMENTATION_RELATIVE_END_TIME = "cube.segmentation.relative.end.time"; + public static final String SEGMENTATION_CUBE_SEGMENT_SFX = ".cubesegments"; + public static final String SEGMENT_PROP_SFX = ".props."; + // dim table constants // TODO: remove this and move to "dimtable." public static final String DIM_TBL_PFX = "dimtble."; http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java index deb5368..399d5a7 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java @@ -359,6 +359,26 @@ public class MetastoreUtil { } // ////////////////////////// + // Segmentation propertes /// + // ///////////////////////// + + public static String getSegmentationKeyPrefix(String segName) { + return SEGMENTATION_KEY_PFX + segName.toLowerCase(); + } + + public static String getSegmentationCubeNameKey(String name) { + return getSegmentationKeyPrefix(name) + CUBE_NAME_SFX; + } + + public static String getSegmentsListKey(String name) { + return getSegmentationKeyPrefix(name) + SEGMENTATION_CUBE_SEGMENT_SFX; + } + + public static String getSegmentPropertyKey(String segName) { + return getSegmentationKeyPrefix(segName) + SEGMENT_PROP_SFX; + } + + // ////////////////////////// // Utils /// // ///////////////////////// public static <E extends Named> String getNamedStr(Collection<E> set) { http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java index 0935509..42e32cb 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java @@ -64,6 +64,7 @@ public class CubeFactTableTest { private CubeFactTable getMockCubeFactTable(Map<String, String> properties) { CubeFactTable cubeFactTable = mock(CubeFactTable.class); + when(cubeFactTable.now()).thenReturn(now); when(cubeFactTable.getProperties()).thenReturn(properties); http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index 65387c6..18575f6 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -980,6 +980,53 @@ public class TestCubeMetastoreClient { assertFalse(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); } + @Test(priority = 1) + public void testCubeSegmentation() throws Exception { + String segmentName = "testMetastoreCubeSegmentation"; + + Table cubeTbl = client.getHiveTable(CUBE_NAME); + assertTrue(client.isCube(cubeTbl)); + + Map<String, String> props = new HashMap<String, String>(){{put("foo", "bar"); }}; + Map<String, String> prop1 = new HashMap<String, String>(){{put("foo1", "bar1"); put("foo11", "bar11"); }}; + Map<String, String> prop2 = new HashMap<String, String>(){{put("foo2", "bar2"); }}; + Map<String, String> prop3 = new HashMap<String, String>(){{put("foo3", "bar3"); }}; + Map<String, String> prop5 = new HashMap<String, String>(){{put("foo5", "bar5"); }}; + + CubeSegment seg1 = new CubeSegment("cube1", prop1); + CubeSegment seg2 = new CubeSegment("cube2", prop2); + CubeSegment seg3 = new CubeSegment("cube3", prop3); + CubeSegment seg5 = new CubeSegment("cube5", prop5); + + Set<CubeSegment> cubeSegs = Sets.newHashSet(seg1, seg2, seg3); + + //create cube segmentation + client.createCubeSegmentation(CUBE_NAME, segmentName, cubeSegs, 0L, props); + assertEquals(client.getCubeSegmentation(segmentName).getCubeSegments().size(), 3); + + //Alter cube segmentation + CubeSegmentation segmentation = new CubeSegmentation(Hive.get(conf).getTable(segmentName)); + segmentation.addCubeSegment(seg5); + segmentation.addProperties(new HashMap<String, String>(){{put("new_key", "new_val"); }}); + segmentation.alterBaseCubeName("segCubeAltered"); + segmentation.alterWeight(100.0); + client.alterCubeSegmentation(segmentName, segmentation); + + assertNotNull(client.getCubeSegmentation(segmentName)); + assertEquals(client.getCubeSegmentation(segmentName).getCubeSegments().size(), 4); + assertEquals(client.getCubeSegmentation(segmentName).getBaseCube(), "segCubeAltered"); + assertEquals(client.getCubeSegmentation(segmentName).weight(), 100.0); + + //drop cubesegment to segmentation + segmentation.dropCubeSegment(seg5); + client.alterCubeSegmentation(segmentName, segmentation); + assertEquals(client.getCubeSegmentation(segmentName).getCubeSegments().size(), 3); + + //drop segmentation + client.dropCubeSegmentation(segmentName); + assertFalse(client.tableExists(segmentName)); + } + private void assertRangeValidityForStorageTable(String storageTable) throws HiveException, LensException { Object[][] testCases = new Object[][] { {"now - 15 days", "now - 11 days", false}, http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java b/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java index 0c2301b..626655d 100644 --- a/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java +++ b/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java @@ -117,6 +117,7 @@ public class SampleMetastore { createCubes(); createDimensions(); createFacts(); + createCubeSegmentations(); createDimensionTables(); try { DatabaseUtil.initializeDatabaseStorage(); @@ -166,6 +167,15 @@ public class SampleMetastore { createFact("sales-aggr-continuous-fact.xml"); } + private void createCubeSegmentations() throws JAXBException, IOException { + result = metaClient.createCubeSegmentation("seg1.xml"); + if (result.getStatus().equals(APIResult.Status.FAILED)) { + System.err.println("Creating cubesegmentation from : " + "seg1.xml" + + " failed, reason:" + result.getMessage()); + retCode = 1; + } + } + public static void main(String[] args) throws Exception { SampleMetastore metastore = null; try { @@ -185,6 +195,7 @@ public class SampleMetastore { System.out.println("Dimensions:" + metastore.metaClient.getAllDimensions()); System.out.println("Fact tables:" + metastore.metaClient.getAllFactTables()); System.out.println("Dimension tables:" + metastore.metaClient.getAllDimensionTables()); + System.out.println("CubeSegmentations:" + metastore.metaClient.getAllCubeSegmentations()); if (metastore.retCode != 0) { System.exit(metastore.retCode); } http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-examples/src/main/resources/seg1.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/seg1.xml b/lens-examples/src/main/resources/seg1.xml new file mode 100644 index 0000000..c1576d7 --- /dev/null +++ b/lens-examples/src/main/resources/seg1.xml @@ -0,0 +1,44 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<x_cube_segmentation cube_name="sample_cube" name="seg1" weight="100.0" xmlns="uri:lens:cube:0.1" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd "> + <properties> + <property name= "cube.segmentation.relative.start.time" value="now -10days"/> + </properties> + <cube_segements> + <cube_segment cube_name="cube11"> + <segment_parameters> + <property name="lens.metastore.cube.column.mapping" value="foo=bar"/> + </segment_parameters> + </cube_segment> + <cube_segment cube_name="cube22"> + <segment_parameters> + <property name="lens.metastore.cube.column.mapping" value="foo1=bar1"/> + </segment_parameters> + </cube_segment> + <cube_segment cube_name="cube33"> + <segment_parameters> + <property name="lens.metastore.cube.column.mapping" value="foo2=bar2"/> + </segment_parameters> + </cube_segment> + </cube_segements> +</x_cube_segmentation> http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-examples/src/test/resources/yaml/seg1.yaml ---------------------------------------------------------------------- diff --git a/lens-examples/src/test/resources/yaml/seg1.yaml b/lens-examples/src/test/resources/yaml/seg1.yaml new file mode 100644 index 0000000..dd2f339 --- /dev/null +++ b/lens-examples/src/test/resources/yaml/seg1.yaml @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +properties: + cube.segmentation.relative.start.time: now -10days +cubeSegements: + - + segmentParameters: + lens.metastore.cube.column.mapping: foo=bar + cubeName: cube11 + - + segmentParameters: + lens.metastore.cube.column.mapping: foo1=bar1 + cubeName: cube22 + - + segmentParameters: + lens.metastore.cube.column.mapping: foo2=bar2 + cubeName: cube33 +name: seg1 +cubeName: sample_cube +weight: 100.0 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java index 52a7ccc..f8a936a 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java @@ -412,6 +412,11 @@ public final class LensConfConstants { public static final String NATIVE_TABLE_COLUMN_MAPPING = METASTORE_PFX + "native.table.column.mapping"; /** + * The property name for setting the column mapping, if column names in cubes are different + */ + public static final String CUBE_COLUMN_MAPPING = METASTORE_PFX + "cube.column.mapping"; + + /** * The Constant ES_INDEX_NAME. */ public static final String ES_INDEX_NAME = METASTORE_PFX + "es.index.name"; http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java index c11fd83..431164f 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java @@ -570,4 +570,53 @@ public interface CubeMetastoreService extends LensService { void updatePartitions(LensSessionHandle sessionid, String tblName, String storageName, XPartitionList partitions) throws LensException; + + /** + * + * @param sessionid The session id + * @param cubeSeg The cube segmentation + * @throws LensException + */ + void createCubeSegmentation(LensSessionHandle sessionid, XCubeSegmentation cubeSeg) throws LensException; + + /** + * Create cube segmentation + * + * @param sessionid The session id + * @param segName Cube segmentation name + * @return {@link XCubeSegmentation} + * @throws LensException + */ + XCubeSegmentation getCubeSegmentation(LensSessionHandle sessionid, String segName) throws LensException; + + /** + * Get cube segmentation given by name + * + * @param sessionid The session id + * @param cubeSeg The cube segmentation + * @throws LensException + */ + + void updateCubeSegmentation(LensSessionHandle sessionid, XCubeSegmentation cubeSeg) throws LensException; + + /** + * Update cube segmentation + * + * @param sessionid The session id + * @param cubeSegName Cube segmentation name + * @throws LensException + */ + void dropCubeSegmentation(LensSessionHandle sessionid, String cubeSegName) throws LensException; + + /** + * Get all cube segmentations belong to Cube + * + * @param sessionid The session id + * @param cubeName The cube Name + * @return + * @throws LensException + */ + List<String> getAllCubeSegmentations(LensSessionHandle sessionid, String cubeName) throws LensException; + + } http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java index a1acd1a..0127ef8 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java @@ -507,6 +507,21 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet } @Override + public XCubeSegmentation getCubeSegmentation(LensSessionHandle sessionid, String cubeSegName) throws LensException { + try { + acquire(sessionid); + CubeMetastoreClient msClient = getClient(sessionid); + CubeSegmentation cubeSeg = msClient.getCubeSegmentation(cubeSegName); + return JAXBUtils.xsegmentationFromCubeSegmentation(cubeSeg); + } catch (HiveException e) { + throw new LensException(e); + } finally { + release(sessionid); + } + } + + + @Override public void createFactTable(LensSessionHandle sessionid, XFactTable fact) throws LensException { try { acquire(sessionid); @@ -526,6 +541,25 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet } @Override + public void createCubeSegmentation(LensSessionHandle sessionid, XCubeSegmentation cubeSeg) throws LensException { + try { + acquire(sessionid); + getClient(sessionid).createCubeSegmentation( + cubeSeg.getCubeName(), + cubeSeg.getName(), + JAXBUtils.cubeSegmentsFromXCubeSegments(cubeSeg.getCubeSegements()), + cubeSeg.getWeight(), + JAXBUtils.mapFromXProperties(cubeSeg.getProperties())); + log.info("Created cube segmentation " + cubeSeg.getName()); + } catch (HiveException e) { + throw new LensException(e); + } finally { + release(sessionid); + } + } + + + @Override public void updateFactTable(LensSessionHandle sessionid, XFactTable fact) throws LensException { try { acquire(sessionid); @@ -540,6 +574,19 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet } @Override + public void updateCubeSegmentation(LensSessionHandle sessionid, XCubeSegmentation cubeSeg) throws LensException { + try { + acquire(sessionid); + getClient(sessionid).alterCubeSegmentation(cubeSeg.getName(), cubeSegmentationFromXCubeSegmentation(cubeSeg)); + log.info("Updated cube segmentation " + cubeSeg.getName()); + } catch (HiveException e) { + throw new LensException(e); + } finally { + release(sessionid); + } + } + + @Override public void dropFactTable(LensSessionHandle sessionid, String fact, boolean cascade) throws LensException { try { acquire(sessionid); @@ -553,6 +600,20 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet } @Override + public void dropCubeSegmentation(LensSessionHandle sessionid, String cubeSegName) throws LensException { + try { + acquire(sessionid); + getClient(sessionid).dropCubeSegmentation(cubeSegName); + log.info("Dropped cube segemntation " + cubeSegName); + } catch (HiveException e) { + throw new LensException(e); + } finally { + release(sessionid); + } + + } + + @Override public List<String> getAllFactNames(LensSessionHandle sessionid, String cubeName) throws LensException { try { acquire(sessionid); @@ -574,6 +635,29 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet } } + @Override + public List<String> getAllCubeSegmentations(LensSessionHandle sessionid, String cubeName) throws LensException { + try { + acquire(sessionid); + CubeMetastoreClient client = getClient(sessionid); + CubeInterface seg = client.getCube(cubeName); + if (cubeName != null && seg == null) { + throw new LensException("Could not get table: " + cubeName + " as a cube"); + } + Collection<CubeSegmentation> segs = client.getAllCubeSegmentations(seg); + List<String> segNames = new ArrayList<String>(segs.size()); + for (CubeSegmentation cs : segs) { + segNames.add(cs.getName()); + } + return segNames; + } catch (HiveException e) { + throw new LensException(e); + } finally { + release(sessionid); + } + + } + @Override public List<String> getStoragesOfFact(LensSessionHandle sessionid, String fact) throws LensException { http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java index 6fd19a0..6571dae 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java @@ -490,6 +490,34 @@ public final class JAXBUtils { return xpList; } + public static Set<XCubeSegment> xCubeSegmentsFromCubeSegments(Set<CubeSegment> segs) { + Set<XCubeSegment> xsegs = new HashSet<XCubeSegment>(); + if (segs != null && !segs.isEmpty()) { + for (CubeSegment seg : segs) { + XCubeSegment xcubeSeg = XCF.createXCubeSegment(); + xcubeSeg.setCubeName(seg.getName()); + xcubeSeg.setSegmentParameters(getXpropertiesFromCubeSegment(seg)); + xsegs.add(xcubeSeg); + } + } + return xsegs; + } + + public static XProperties getXpropertiesFromCubeSegment(CubeSegment cseg) { + XProperties xproperties = XCF.createXProperties(); + for (String prop : cseg.getProperties().keySet()) { + String segPrefix = MetastoreUtil.getSegmentPropertyKey(cseg.getName()); + if (prop.startsWith(segPrefix)){ + XProperty xprop = XCF.createXProperty(); + xprop.setName(prop.replace(segPrefix, "")); + xprop.setValue(cseg.getProperties().get(prop)); + xproperties.getProperty().add(xprop); + } + } + return xproperties; + } + + public static FieldSchema fieldSchemaFromColumn(XColumn c) { if (c == null) { return null; @@ -659,6 +687,24 @@ public final class JAXBUtils { mapFromXProperties(fact.getProperties())); } + public static CubeSegmentation cubeSegmentationFromXCubeSegmentation(XCubeSegmentation seg) throws LensException { + + Map<String, String> props = new HashMap<>(); + // Skip properties with keyword internal. These properties are internal to lens + // and users are not supposed to see them. + for(String prop : mapFromXProperties(seg.getProperties()).keySet()) { + if (!(prop.toLowerCase().startsWith(MetastoreConstants.SEGMENTATION_KEY_PFX))) { + props.put(prop, mapFromXProperties(seg.getProperties()).get(prop)); + } + } + return new CubeSegmentation(seg.getCubeName(), + seg.getName(), + cubeSegmentsFromXCubeSegments(seg.getCubeSegements()), + seg.getWeight(), + props); + } + + public static XFactTable factTableFromCubeFactTable(CubeFactTable cFact) { XFactTable fact = XCF.createXFactTable(); fact.setName(cFact.getName()); @@ -673,6 +719,21 @@ public final class JAXBUtils { return fact; } + public static XCubeSegmentation xsegmentationFromCubeSegmentation(CubeSegmentation cSeg) { + XCubeSegmentation seg = XCF.createXCubeSegmentation(); + seg.setName(cSeg.getName()); + seg.setProperties(new XProperties()); + seg.setCubeSegements(new XCubeSegments()); + seg.setWeight(cSeg.weight()); + seg.setCubeName(cSeg.getBaseCube()); + if (xPropertiesFromMap(cSeg.getProperties()) != null) { + seg.getProperties().getProperty().addAll(xPropertiesFromMap(cSeg.getProperties())); + } + seg.getCubeSegements().getCubeSegment(). + addAll(xCubeSegmentsFromCubeSegments(cSeg.getCubeSegments())); + return seg; + } + public static StorageTableDesc storageTableDescFromXStorageTableDesc( XStorageTableDesc xtableDesc) { StorageTableDesc tblDesc = new StorageTableDesc(); @@ -770,6 +831,20 @@ public final class JAXBUtils { return storageTableMap; } + public static Set<CubeSegment> cubeSegmentsFromXCubeSegments(XCubeSegments segs) { + Set<CubeSegment> cubeSegs = new HashSet<>(); + for (XCubeSegment xcube : segs.getCubeSegment()){ + Map<String, String> segProp = new HashMap<>(); + if (xcube.getSegmentParameters() != null) { + for (XProperty prop : xcube.getSegmentParameters().getProperty()) { + segProp.put(prop.getName(), prop.getValue()); + } + } + cubeSegs.add(new CubeSegment(xcube.getCubeName(), segProp)); + } + return cubeSegs; + } + public static Map<String, Date> timePartSpecfromXTimePartSpec( XTimePartSpec xtimePartSpec) { Map<String, Date> timePartSpec = new HashMap<String, Date>(); http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java index 684e2ed..7395c83 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java @@ -142,7 +142,23 @@ public class MetastoreResource { getSvc().dropFactTable(sessionid, entityName, cascade); } } - }, DIMENSION { + }, + SEGMENTATION { + @Override + public List<String> doGetAll(LensSessionHandle sessionid) throws LensException { + return getSvc().getAllCubeSegmentations(sessionid, null); + } + + @Override + public void doDelete(LensSessionHandle sessionid, String entityName, Boolean cascade) throws LensException { + if (cascade == null) { + getSvc().dropCubeSegmentation(sessionid, entityName); + } else { + throw new NotImplementedException(); + } + } + } + , DIMENSION { @Override public List<String> doGetAll(LensSessionHandle sessionid) throws LensException { return getSvc().getAllDimensionNames(sessionid); @@ -791,6 +807,28 @@ public class MetastoreResource { } /** + * Get all cube segmentations that belong to a cube in the metastore + * + * @param sessionid The sessionid in which user is working + * @param cubeName name of the base cube or derived cube + * @return List of {@link XCubeSegmentation} objects + */ + @GET + @Path("/cubes/{cubeName}/cubesegmentations") + public StringList getAllCubeSegmentationsOfCube( + @QueryParam("sessionid") LensSessionHandle sessionid, @PathParam("cubeName") String cubeName) + throws LensException { + checkSessionId(sessionid); + try { + return new StringList(getSvc().getAllCubeSegmentations(sessionid, cubeName)); + } catch (LensException exc) { + checkTableNotFound(exc, cubeName); + throw exc; + } + } + + + /** * Get all fact tables in the metastore in the current database * * @param sessionid The sessionid in which user is working @@ -803,6 +841,22 @@ public class MetastoreResource { return Entity.FACT.getAll(sessionid); } + + /** + * Get all cube segmentations in the current database + * + * @param sessionid The sessionid in which user is working + * @return StringList consisting of all cube segmentations + */ + @GET + @Path("/cubesegmentations") + public StringList getAllCubeSegmentations(@QueryParam("sessionid") LensSessionHandle sessionid) + throws LensException { + checkSessionId(sessionid); + return Entity.SEGMENTATION.getAll(sessionid); + } + + /** * Delete all fact tables * @@ -820,6 +874,19 @@ public class MetastoreResource { } /** + * Delete all cube segmentations + * + * @param sessionid The sessionid in which user is working + * @return APIResult with state {@link Status#SUCCEEDED} in case of successful delete. APIResult with state {@link + * Status#FAILED} in case of delete failure. APIResult with state {@link Status#PARTIAL} in case of partial delete. + */ + @DELETE + @Path("cubesegmentations") + public APIResult deleteAllCubeSegmentations(@QueryParam("sessionid") LensSessionHandle sessionid) { + return Entity.SEGMENTATION.deleteAll(sessionid, null); + } + + /** * Get the fact table specified by name * * @param sessionid The sessionid in which user is working @@ -841,6 +908,28 @@ public class MetastoreResource { } /** + * Get the cube segmentation specified by name + * + * @param sessionid The sessionid in which user is working + * @param cubeSegmentationName The cube segmentation name + * @return JAXB representation of {@link XCubeSegmentation} + */ + @GET + @Path("/cubesegmentations/{cubeSegmentationName}") + public JAXBElement<XCubeSegmentation> getCubeSegmentation(@QueryParam("sessionid") LensSessionHandle sessionid, + @PathParam("cubeSegmentationName") String cubeSegmentationName) + throws LensException { + checkSessionId(sessionid); + try { + return X_CUBE_OBJECT_FACTORY.createXCubeSegmentation(getSvc(). + getCubeSegmentation(sessionid, cubeSegmentationName)); + } catch (LensException exc) { + checkTableNotFound(exc, cubeSegmentationName); + throw exc; + } + } + + /** * Create a new fact tabble * * @param sessionid The sessionid in which user is working @@ -864,6 +953,30 @@ public class MetastoreResource { } /** + * Create a new cube segmentation + * + * @param sessionid The sessionid in which user is working + * @param seg The {@link XCubeSegmentation} representation of the cube segmentation + * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if create was successful. {@link APIResult} with + * state {@link Status#FAILED}, if create has failed + */ + @POST + @Path("/cubesegmentations") + public APIResult createCubeSegmentation(@QueryParam("sessionid") LensSessionHandle sessionid, XCubeSegmentation seg) + throws LensException { + checkSessionId(sessionid); + try { + log.info("Create cube segmentation"); + getSvc().createCubeSegmentation(sessionid, seg); + } catch (LensException exc) { + log.error("Exception creating cube segmentation:", exc); + return failure(processLensException(exc)); + } + return success(); + } + + + /** * Update fact table definition * * @param sessionid The sessionid in which user is working @@ -889,6 +1002,31 @@ public class MetastoreResource { } /** + * Update cube segmentation + * + * @param sessionid The sessionid in which user is working + * @param cubeSegmentationName name of cube segmentation + * @param seg The {@link XCubeSegmentation} representation of the updated fact table definition + * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if update was successful. {@link APIResult} with + * state {@link Status#FAILED}, if update has failed + */ + @PUT + @Path("/cubesegmentations/{cubeSegmentationName}") + public APIResult updateCubeSegmentation(@QueryParam("sessionid") LensSessionHandle sessionid, + @PathParam("cubeSegmentationName") String cubeSegmentationName, XCubeSegmentation seg) + throws LensException { + checkSessionId(sessionid); + try { + getSvc().updateCubeSegmentation(sessionid, seg); + } catch (LensException exc) { + checkTableNotFound(exc, cubeSegmentationName); + log.error("Error updating segmentation {}", cubeSegmentationName, exc); + return failure(processLensException(exc)); + } + return success(); + } + + /** * Drop the fact table, specified by name * * @param sessionid The sessionid in which user is working @@ -906,6 +1044,23 @@ public class MetastoreResource { return Entity.FACT.delete(sessionid, factName, cascade); } + + /** + * Drop the cube segmentation, specified by name + * + * @param sessionid The sessionid in which user is working + * @param cubeSegmentationName The cube segmentation name + * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if drop was successful. {@link APIResult} with state + * {@link Status#FAILED}, if drop has failed + */ + @DELETE + @Path("/cubesegmentations/{cubeSegmentationName}") + public APIResult dropCubeSegmentation(@QueryParam("sessionid") LensSessionHandle sessionid, + @PathParam("cubeSegmentationName") String cubeSegmentationName) + throws LensException { + return Entity.SEGMENTATION.delete(sessionid, cubeSegmentationName, null); + } + /** * Get all storages of the fact table in the metastore *
