LENS-1389: Back Merge with master
Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/8868b063 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/8868b063 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/8868b063 Branch: refs/heads/lens-1381 Commit: 8868b063838050dd8746cefa491f143533e145c9 Parents: d45c538 d1b43d6 Author: Rajat Khandelwal <[email protected]> Authored: Fri Mar 31 15:16:10 2017 +0530 Committer: Rajat Khandelwal <[email protected]> Committed: Fri Mar 31 15:16:10 2017 +0530 ---------------------------------------------------------------------- contrib/clients/python/lens/client/main.py | 7 +++- contrib/clients/python/lens/client/query.py | 43 +++++++++++++++----- contrib/clients/python/setup.py | 2 + contrib/clients/python/test/test_lensclient.py | 13 ++++-- .../lens/cube/metadata/CubeMetastoreClient.java | 22 +++++----- .../cube/parse/DenormalizationResolver.java | 1 + .../lens/cube/parse/ExpressionResolver.java | 2 +- .../apache/lens/cube/parse/CubeTestSetup.java | 2 - .../metastore/CubeMetastoreServiceImpl.java | 2 - tools/conf/server/lens-site.xml | 6 +++ 10 files changed, 69 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/8868b063/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java ---------------------------------------------------------------------- diff --cc lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 7608a43,087c203..b445447 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@@ -307,74 -290,18 +307,74 @@@ public class CubeMetastoreClient } } + public void createCubeFactTable(String cubeName, String factName, List<FieldSchema> columns, + Map<String, Set<UpdatePeriod>> storageAggregatePeriods, double weight, Map<String, String> properties, + Map<String, StorageTableDesc> storageTableDescs, Map<String, Map<UpdatePeriod, String>> storageUpdatePeriodMap) + throws LensException { + CubeFactTable factTable = new CubeFactTable(cubeName, factName, columns, storageAggregatePeriods, weight, + properties, storageUpdatePeriodMap); + createCubeTable(factTable, storageTableDescs); + // do a get to update cache + getCubeFact(factName); + + } + public <T extends Equals & HashCode & ToString> void createEntity(T entity) throws LensException { + if (entity instanceof XStorage) { + createStorage((XStorage) entity); + } else if (entity instanceof XCube) { + createCube((XCube)entity); + } else if (entity instanceof XDimension) { + createDimension((XDimension) entity); + } else if (entity instanceof XFactTable) { + createCubeFactTable((XFactTable) entity); + } else if (entity instanceof XDimensionTable) { + createCubeDimensionTable((XDimensionTable) entity); + } else if (entity instanceof XSegmentation) { + createSegmentation((XSegmentation) entity); + } else { + throw new LensException("Unable to create entity " + entity + " as it's unrecognizable: "+ entity.getClass()); + } + } + + public <T extends Equals & HashCode & ToString> void updateEntity(String name, T entity) + throws LensException, HiveException { + if (entity instanceof XStorage) { + alterStorage((XStorage) entity); + } else if (entity instanceof XCube) { + alterCube((XCube)entity); + } else if (entity instanceof XDimension) { + alterDimension((XDimension) entity); + } else if (entity instanceof XFactTable) { + alterCubeFactTable((XFactTable) entity); + } else if (entity instanceof XDimensionTable) { + alterCubeDimensionTable((XDimensionTable) entity); + } else if (entity instanceof XSegmentation) { + alterSegmentation((XSegmentation) entity); + } else { + throw new LensException("Unable to alter entity " + entity + " as it's unrecognizable: " + entity.getClass()); + } + } + + + public static Map<String, String> addFactColStartTimePropertyToFactProperties(XFactTable fact) { + Map<String, String> props = new HashMap<String, String>(); + props.putAll(JAXBUtils.mapFromXProperties(fact.getProperties())); + props.putAll(JAXBUtils.columnStartAndEndTimeFromXColumns(fact.getColumns())); + return props; + } + public void createCubeFactTable(XFactTable fact) throws LensException { + createCubeFactTable(fact.getCubeName(), + fact.getName(), + JAXBUtils.fieldSchemaListFromColumns(fact.getColumns()), + JAXBUtils.getFactUpdatePeriodsFromStorageTables(fact.getStorageTables()), + fact.getWeight(), + addFactColStartTimePropertyToFactProperties(fact), + JAXBUtils.tableDescPrefixMapFromXStorageTables(fact.getStorageTables()), + JAXBUtils.storageTablePrefixMapOfStorage(fact.getStorageTables())); + } - public void createCubeFactTable(String cubeName, String factName, List<FieldSchema> columns, - Map<String, Set<UpdatePeriod>> storageAggregatePeriods, double weight, Map<String, String> properties, - Map<String, StorageTableDesc> storageTableDescs, Map<String, Map<UpdatePeriod, String>> storageUpdatePeriodMap) - throws LensException { - CubeFactTable factTable = new CubeFactTable(cubeName, factName, columns, storageAggregatePeriods, weight, - properties, storageUpdatePeriodMap); - createCubeTable(factTable, storageTableDescs); - // do a get to update cache - getCubeFact(factName); - - } + + /** * In-memory storage of {@link PartitionTimeline} objects for each valid * storagetable-updateperiod-partitioncolumn tuple. also simultaneously stored in metastore table of the http://git-wip-us.apache.org/repos/asf/lens/blob/8868b063/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java ---------------------------------------------------------------------- diff --cc lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java index 76e5f23,cb26878..e5cf916 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java @@@ -138,9 -161,10 +138,10 @@@ public class DenormalizationResolver im return null; } - public Set<Dimension> rewriteDenormctx(CubeQueryContext cubeql, CandidateFact cfact, Map<Dimension, - CandidateDim> dimsToQuery, boolean replaceFact) throws LensException { + Set<Dimension> rewriteDenormctx(CubeQueryContext cubeql, + StorageCandidate sc, Map<Dimension, CandidateDim> dimsToQuery, boolean replaceFact) throws LensException { Set<Dimension> refTbls = new HashSet<>(); - log.info("Doing denorm changes for fact :{}", cfact); ++ log.info("Doing denorm changes for fact :{}", sc); if (!tableToRefCols.isEmpty()) { // pick referenced columns for fact http://git-wip-us.apache.org/repos/asf/lens/blob/8868b063/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/8868b063/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java ---------------------------------------------------------------------- diff --cc lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java index 860db28,9b29083..62d7386 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java @@@ -525,16 -511,1143 +525,14 @@@ public class CubeTestSetup return expected.toString(); } - private Set<ExprColumn> exprs; - Set<ExprColumn> exprs; - - private void createCube(CubeMetastoreClient client) throws HiveException, ParseException, LensException { - cubeMeasures = new HashSet<CubeMeasure>(); - Map<String, String> tags = new HashMap<>(); - tags.put(MetastoreConstants.MEASURE_DATACOMPLETENESS_TAG, "tag1"); - cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr1", "int", "first measure"), null, null, null, null, null, - null, null, null, null, tags)); - cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr2", "float", "second measure"), "Measure2", null, "SUM", - "RS")); - cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr21", "float", "second measure"), "Measure22", null, "SUM", - "RS")); - cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr22", "float", "second measure"), "Measure22", null, "SUM", - "RS")); - cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr3", "double", "third measure"), "Measure3", null, "MAX", - null)); - cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr4", "bigint", "fourth measure"), "Measure4", null, "COUNT", - null)); - cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr9", "bigint", "ninth measure"), null, null, null, null, - null, null, null, null, null, tags)); - cubeMeasures.add(new ColumnMeasure(new FieldSchema("noAggrMsr", "bigint", "measure without a default aggregate"), - "No aggregateMsr", null, null, null)); - cubeMeasures.add(new ColumnMeasure(new FieldSchema("newmeasure", "bigint", "measure available from now"), - "New measure", null, null, null, NOW, null, 100.0)); - cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr15", "int", "fifteenth measure"), "Measure15", null, "SUM", - "RS")); - - cubeDimensions = new HashSet<CubeDimAttribute>(); - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("d_time", "timestamp", "d time"))); - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("processing_time", "timestamp", "processing time"))); - List<CubeDimAttribute> locationHierarchy = new ArrayList<CubeDimAttribute>(); - locationHierarchy.add(new BaseDimAttribute(new FieldSchema("zipcode", "int", "zip"))); - locationHierarchy.add(new BaseDimAttribute(new FieldSchema("cityid", "int", "city"))); - locationHierarchy.add(new BaseDimAttribute(new FieldSchema("stateid", "int", "state"))); - locationHierarchy.add(new BaseDimAttribute(new FieldSchema("countryid", "int", "country"))); - List<String> regions = Arrays.asList("APAC", "EMEA", "USA"); - locationHierarchy.add(new BaseDimAttribute(new FieldSchema("regionname", "string", "region"), "regionname", null, - null, null, null, regions)); - - cubeDimensions.add(new HierarchicalDimAttribute("location", "Location hierarchy", locationHierarchy)); - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("dim1", "string", "basedim"))); - // Added for ambiguity test - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("ambigdim1", "string", "used in testColumnAmbiguity"))); - cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("dim2", "int", "ref dim"), "dim2 refer", - "dim2chain", "id", null, null, 0.0)); - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("cdim2", "int", "ref dim"), "Dim2 refer", NOW, null, null)); - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("urdimid", "int", "ref dim"), "urdim refer", - null, null, 10.0)); - cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("unreachableName", "string", ""), "urdim name", - "unreachableDim_chain", "name", null, null, 10.0)); - // denormalized reference - cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("dim2big1", "bigint", "ref dim"), "dim2 refer", - "dim2chain", "bigid1", null, null, 0.0)); - cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("dim2big2", "bigint", "ref dim"), "dim2 refer", - "dim2chain", "bigid2", null, null, 0.0)); - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("dim2bignew", "bigint", "ref dim"), "Dim2 refer", - NOW, null, null)); - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("test_time_dim_hour_id", "int", "ref dim"), - "Timedim reference", null, null, null)); - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("test_time_dim_day_id", "int", "ref dim"), - "Timedim reference", null, null, null)); - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("test_time_dim_hour_id2", "int", "ref dim"))); - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("test_time_dim_day_id2", "int", "ref dim"))); - cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("testDim3id", "string", "direct id to testdim3"), - "dim3 refer", "dim3chain", "id", null, null, 0.0)); - cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("cityname", "string", "city name"), - "city name", "cubecity", "name", null, null, 0.0)); - cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("statename_cube", "string", "state name"), - "state name", "cubestate", "name", null, null, 0.0)); - List<ChainRefCol> references = new ArrayList<>(); - references.add(new ChainRefCol("timedatechain1", "full_date")); - references.add(new ChainRefCol("timehourchain1", "full_hour")); - cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("test_time_dim", "date", "ref dim"), - "Timedim full date", references, null, null, null, null)); - List<ChainRefCol> chainRefs = new ArrayList<>(); - chainRefs.add(new ChainRefCol("timehourchain2", "full_hour")); - chainRefs.add(new ChainRefCol("timedatechain2", "full_date")); - cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("test_time_dim2", "date", "chained dim"), - "Timedim full date", chainRefs, null, null, null, null)); - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("cityid1", "int", "id to city"), - "City1", null, null, null)); - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("cityid2", "int", "id to city"), - "City2", null, null, null)); - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("concatedcitystate", "string", "citystate"), - "CityState", null, null, null)); - - Map<String, JoinChain> joinChains = new HashMap<>(); - addCubeChains(joinChains, TEST_CUBE_NAME); - - exprs = new HashSet<ExprColumn>(); - exprs.add(new ExprColumn(new FieldSchema("avgmsr", "double", "avg measure"), "Avg Msr", "avg(msr1 + msr2)")); - exprs.add(new ExprColumn(new FieldSchema("singlecolmsr2expr", "double", "measure2"), "Msr2", "msr2)")); - exprs.add(new ExprColumn(new FieldSchema("singlecolmsr2qualifiedexpr", "double", "testcube.measure2"), - "Msr2", "testcube.msr2")); - exprs.add(new ExprColumn(new FieldSchema("singlecoldim1expr", "string", "dim1"), "dim1", "dim1)")); - exprs.add(new ExprColumn(new FieldSchema("singlecoldim1qualifiedexpr", "string", "testcube.dim1"), - "dim1", "testcube.dim1")); - exprs.add(new ExprColumn(new FieldSchema("singlecolchainid", "string", "dim3chain.id"), - "dim3chainid", "dim3chain.id)")); - exprs.add(new ExprColumn(new FieldSchema("singlecolchainrefexpr", "string", "testcube.testDim3id"), - "dim3chainid", "testcube.testDim3id")); - exprs.add(new ExprColumn(new FieldSchema("singlecolchainfield", "string", "cubecity.name"), - "cubecityname", "cubecity.name")); - exprs.add(new ExprColumn(new FieldSchema("summsrs", "double", "sum measures"), "Sum Msrs", - "(1000 + sum(msr1) + sum(msr2))/100")); - exprs.add(new ExprColumn(new FieldSchema("msr5", "double", "materialized in some facts"), "Fifth Msr", - "msr2 + msr3")); - exprs.add(new ExprColumn(new FieldSchema("msr8", "double", "measure expression"), "Sixth Msr", - "msr2 + msr3")); - exprs.add(new ExprColumn(new FieldSchema("msr7", "double", "measure expression"), "Seventh Msr", - "case when sum(msr2) = 0 then 0 else sum(case when cityid='x' then msr21 else msr22 end)/sum(msr2) end")); - exprs.add(new ExprColumn(new FieldSchema("equalsums", "double", "sums are equals"), "equalsums", - new ExprSpec("msr3 + msr4", null, null), new ExprSpec("(msr3 + msr2)/100", null, null))); - exprs.add(new ExprColumn(new FieldSchema("roundedmsr1", "double", "rounded measure1"), "Rounded msr1", - "round(msr1/1000)")); - exprs.add(new ExprColumn(new FieldSchema("roundedmsr2", "double", "rounded measure2"), "Rounded msr2", - "round(msr2/1000)")); - exprs.add(new ExprColumn(new FieldSchema("flooredmsr12", "double", "floored measure12"), "Floored msr12", - "floor(msr12)")); - exprs.add(new ExprColumn(new FieldSchema("nestedexpr", "double", "nested expr"), "Nested expr", - new ExprSpec("avg(roundedmsr2)", null, null), new ExprSpec("avg(equalsums)", null, null), - new ExprSpec("case when substrexpr = 'xyz' then avg(msr5) when substrexpr = 'abc' then avg(msr4)/100 end", - null, null))); - exprs.add(new ExprColumn(new FieldSchema("msr2expr", "double", "nested expr"), "Nested expr", - new ExprSpec("case when cityStateName = 'xyz' then msr2 else 0 end", null, null))); - exprs.add(new ExprColumn(new FieldSchema("nestedExprWithTimes", "double", "nested expr"), "Nested expr", - new ExprSpec("avg(roundedmsr2)", null, null), new ExprSpec("avg(equalsums)", null, null), - new ExprSpec("case when substrexpr = 'xyz' then avg(msr5) when substrexpr = 'abc' then avg(msr4)/100 end", - NOW, null), new ExprSpec("avg(newmeasure)", null, null))); - exprs.add(new ExprColumn(new FieldSchema("msr6", "bigint", "sixth measure"), "Measure6", - "sum(msr2) + max(msr3)/ count(msr4)")); - exprs.add(new ExprColumn(new FieldSchema("booleancut", "boolean", "a boolean expression"), "Boolean cut", - "(dim1 != 'x' AND dim2 != 10)")); - exprs.add(new ExprColumn(new FieldSchema("substrexpr", "string", "a sub-string expression"), "Substr expr", - new ExprSpec("substr(dim1, 3))", null, null), new ExprSpec("substr(ascii(dim2chain.name), 3)", null, null))); - exprs.add(new ExprColumn(new FieldSchema("substrexprdim2", "string", "a sub-string expression"), "Substr expr", - new ExprSpec("substr(dim2, 3))", null, null), new ExprSpec("substr(ascii(dim2chain.name), 3)", null, null))); - exprs.add(new ExprColumn(new FieldSchema("indiasubstr", "boolean", "nested sub string expression"), "Nested expr", - "substrexpr = 'INDIA'")); - exprs.add(new ExprColumn(new FieldSchema("refexpr", "string", "expression which facts and dimensions"), - "Expr with cube and dim fields", "concat(dim1, \":\", citydim.name)")); - exprs.add(new ExprColumn(new FieldSchema("nocolexpr", "string", "expression which non existing colun"), - "No col expr", "myfun(nonexist)")); - exprs.add(new ExprColumn(new FieldSchema("newexpr", "string", "expression which non existing colun"), - "new measure expr", "myfun(newmeasure)")); - exprs.add(new ExprColumn(new FieldSchema("cityAndState", "String", "city and state together"), "City and State", - new ExprSpec("concat(cityname, \":\", statename_cube)", null, null), - new ExprSpec("substr(concatedcitystate, 10)", null, null))); - exprs.add(new ExprColumn(new FieldSchema("cityAndStateNew", "String", "city and state together"), "City and State", - new ExprSpec("concat(cityname, \":\", statename_cube)", null, TWO_MONTHS_BACK), - new ExprSpec("substr(concatedcitystate, 10)", null, null))); - exprs.add(new ExprColumn(new FieldSchema("cityStateName", "String", "city state"), "City State", - "concat('CityState:', cubecity.statename)")); - exprs.add(new ExprColumn(new FieldSchema("isIndia", "String", "is indian city/state"), "Is Indian City/state", - "cubecity.name == 'DELHI' OR cubestate.name == 'KARNATAKA' OR cubestate.name == 'MAHARASHTRA'")); - exprs.add(new ExprColumn(new FieldSchema("cubeStateName", "String", "statename from cubestate"), "CubeState Name", - "substr(cubestate.name, 5)")); - exprs.add(new ExprColumn(new FieldSchema("substrdim2big1", "String", "substr of dim2big1"), "dim2big1 substr", - "substr(dim2big1, 5)")); - exprs.add(new ExprColumn(new FieldSchema("asciicity", "String", "ascii cityname"), "ascii cityname substr", - "ascii(cityname)")); - exprs.add(new ExprColumn(new FieldSchema("countofdistinctcityid", "int", "Count of Distinct CityId"), - "Count of Distinct CityId Expr", "count(distinct(cityid))")); - exprs.add(new ExprColumn(new FieldSchema("notnullcityid", "int", "Not null cityid"), - "Not null cityid Expr", "case when cityid is null then 0 else cityid end")); - - Map<String, String> cubeProperties = new HashMap<String, String>(); - cubeProperties.put(MetastoreUtil.getCubeTimedDimensionListKey(TEST_CUBE_NAME), - "d_time,pt,it,et,test_time_dim,test_time_dim2"); - cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "test_time_dim", "ttd"); - cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "test_time_dim2", "ttd2"); - cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "d_time", "dt"); - cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "it", "it"); - cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "et", "et"); - cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "pt", "pt"); - cubeProperties.put(MetastoreConstants.TIMEDIM_RELATION + "d_time", "test_time_dim+[-10 days,10 days]"); - - client.createCube(TEST_CUBE_NAME, cubeMeasures, cubeDimensions, exprs, Sets.newHashSet(joinChains.values()), - cubeProperties); - - Set<String> measures = new HashSet<String>(); - measures.add("msr1"); - measures.add("msr2"); - measures.add("msr3"); - measures.add("msr9"); - Set<String> dimensions = new HashSet<String>(); - dimensions.add("dim1"); - dimensions.add("dim2"); - dimensions.add("dim2big1"); - dimensions.add("dim2big2"); - dimensions.add("dim2bignew"); - // Try creating derived cube with non existant dim/measures - try{ - client.createDerivedCube(TEST_CUBE_NAME, DERIVED_CUBE_NAME, - Sets.newHashSet("random_measure"), Sets.newHashSet("random_dim_attribute"), - new HashMap<String, String>(), 5L); - } catch(LensException e) { - assertTrue(e.getMessage().contains("random_measure")); - assertTrue(e.getMessage().contains("random_dim_attribute")); - assertTrue(e.getMessage().contains("not present")); - } - client.createDerivedCube(TEST_CUBE_NAME, DERIVED_CUBE_NAME, - measures, dimensions, new HashMap<String, String>(), 5L); - } - - private void addCubeChains(Map<String, JoinChain> joinChains, final String cubeName) { - joinChains.put("timehourchain1", new JoinChain("timehourchain1", "time chain", "time dim thru hour dim") { - { - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "test_time_dim_hour_id")); - add(new TableReference("hourdim", "id")); - } - }); - } - }); - joinChains.put("timedatechain1", new JoinChain("timedatechain1", "time chain", "time dim thru date dim") { - { - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "test_time_dim_day_id")); - add(new TableReference("daydim", "id")); - } - }); - } - }); - joinChains.put("timehourchain2", new JoinChain("timehourchain2", "time chain", "time dim thru hour dim") { - { - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "test_time_dim_hour_id2")); - add(new TableReference("hourdim", "id")); - } - }); - } - }); - joinChains.put("timedatechain2", new JoinChain("timedatechain2", "time chain", "time dim thru date dim") { - { - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "test_time_dim_day_id2")); - add(new TableReference("daydim", "id")); - } - }); - } - }); - joinChains.put("cubeCity", new JoinChain("cubeCity", "cube-city", "city thru cube") { - { - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "cityid")); - add(new TableReference("citydim", "id")); - } - }); - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "dim2")); - add(new TableReference("testdim2", "id")); - add(new TableReference("testdim2", "cityid")); - add(new TableReference("citydim", "id")); - } - }); - } - }); - joinChains.put("cubeCity1", new JoinChain("cubeCity1", "cube-city", "city thru cube") { - { - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "cityid1")); - add(new TableReference("citydim", "id")); - } - }); - } - }); - joinChains.put("cubeCity2", new JoinChain("cubeCity2", "cube-city", "city thru cube") { - { - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "cityid2")); - add(new TableReference("citydim", "id")); - } - }); - } - }); - joinChains.put("cubeState", new JoinChain("cubeState", "cube-state", "state thru cube") { - { - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "stateid")); - add(new TableReference("statedim", "id")); - } - }); - } - }); - joinChains.put("cubeZip", new JoinChain("cubeZip", "cube-zip", "Zipcode thru cube") { - { - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "zipcode")); - add(new TableReference("zipdim", "code")); - } - }); - } - }); - joinChains.put("cubeCountry", new JoinChain("cubeCountry", "cube-country", "country thru cube") { - { - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "countryid")); - add(new TableReference("countrydim", "id")); - } - }); - } - }); - joinChains.put("dim2chain", new JoinChain("dim2chain", "cube-testdim2", "testdim2 thru cube") { - { - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "dim2")); - add(new TableReference("testdim2", "id")); - } - }); - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "dim2big1")); - add(new TableReference("testdim2", "bigid1")); - } - }); - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "dim2big2")); - add(new TableReference("testdim2", "bigid2")); - } - }); - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "dim2bignew")); - add(new TableReference("testdim2", "bigidnew")); - } - }); - } - }); - joinChains.put("dim3chain", new JoinChain("dim3chain", "cube-testdim3", "cyclicdim thru cube") { - { - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "dim2")); - add(new TableReference("testdim2", "id")); - add(new TableReference("testdim2", "testdim3id")); - add(new TableReference("testdim3", "id")); - } - }); - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "dim2big1")); - add(new TableReference("testdim2", "bigid1")); - add(new TableReference("testdim2", "testdim3id")); - add(new TableReference("testdim3", "id")); - } - }); - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "dim2big2")); - add(new TableReference("testdim2", "bigid2")); - add(new TableReference("testdim2", "testdim3id")); - add(new TableReference("testdim3", "id")); - } - }); - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "dim2bignew")); - add(new TableReference("testdim2", "bigidnew")); - add(new TableReference("testdim2", "testdim3id")); - add(new TableReference("testdim3", "id")); - } - }); - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "testdim3id")); - add(new TableReference("testdim3", "id")); - } - }); - } - }); - joinChains.put("dim4chain", new JoinChain("dim4chain", "cube-testdim3", "cyclicdim thru cube") { - { - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "dim2")); - add(new TableReference("testdim2", "id")); - add(new TableReference("testdim2", "testdim3id")); - add(new TableReference("testdim3", "id")); - add(new TableReference("testdim3", "testdim4id")); - add(new TableReference("testdim4", "id")); - } - }); - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "dim2big1")); - add(new TableReference("testdim2", "bigid1")); - add(new TableReference("testdim2", "testdim3id")); - add(new TableReference("testdim3", "id")); - add(new TableReference("testdim3", "testdim4id")); - add(new TableReference("testdim4", "id")); - } - }); - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "dim2big2")); - add(new TableReference("testdim2", "bigid2")); - add(new TableReference("testdim2", "testdim3id")); - add(new TableReference("testdim3", "id")); - add(new TableReference("testdim3", "testdim4id")); - add(new TableReference("testdim4", "id")); - } - }); - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "dim2bignew")); - add(new TableReference("testdim2", "bigidnew")); - add(new TableReference("testdim2", "testdim3id")); - add(new TableReference("testdim3", "id")); - add(new TableReference("testdim3", "testdim4id")); - add(new TableReference("testdim4", "id")); - } - }); - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "testdim3id")); - add(new TableReference("testdim3", "id")); - add(new TableReference("testdim3", "testdim4id")); - add(new TableReference("testdim4", "id")); - } - }); - } - }); - joinChains.put("cdimChain", new JoinChain("cdimChain", "cube-cyclicdim", "cyclicdim thru cube") { - { - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "cdim2")); - add(new TableReference("cycledim1", "id")); - } - }); - } - }); - joinChains.put("unreachableDim_chain", new JoinChain("unreachableDim_chain", "cube-unreachableDim", - "unreachableDim thru cube") { - { - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "urdimid")); - add(new TableReference("unreachableDim", "id")); - } - }); - } - }); - joinChains.put("cubeCountry", new JoinChain("cubeCountry", "cube-country", "country thru cube") { - { - addPath(new ArrayList<TableReference>() { - { - add(new TableReference(cubeName, "countryid")); - add(new TableReference("countrydim", "id")); - } - }); - } - }); - } - private void createBaseAndDerivedCubes(CubeMetastoreClient client) - throws HiveException, ParseException, LensException { - Set<CubeMeasure> cubeMeasures2 = new HashSet<>(cubeMeasures); - Set<CubeDimAttribute> cubeDimensions2 = new HashSet<>(cubeDimensions); - cubeMeasures2.add(new ColumnMeasure(new FieldSchema("msr11", "int", "first measure"))); - cubeMeasures2.add(new ColumnMeasure(new FieldSchema("msr12", "float", "second measure"), "Measure2", null, "SUM", - "RS")); - cubeMeasures2.add(new ColumnMeasure(new FieldSchema("msr13", "double", "third measure"), "Measure3", null, "MAX", - null)); - cubeMeasures2.add(new ColumnMeasure(new FieldSchema("msr14", "bigint", "fourth measure"), "Measure4", null, - "COUNT", null)); - cubeMeasures2.add(new ColumnMeasure(new FieldSchema("directMsr", "bigint", "fifth measure"), "Direct Measure", - null, "SUM", null)); - - cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("dim11", "string", "basedim"))); - cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("dim12", "int", "ref dim"), "Dim2 refer", - "dim2chain", "id", null, null, null)); // used as key in the chains - cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("dim22", "int", "ref dim"), "Dim2 refer", - "dim2chain", "id", null, null, null)); // not used as key in the chains - cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("dim13", "string", "basedim"))); - cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("userid", "int", "userid"))); - cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("xuserid", "int", "userid"))); - cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("yuserid", "int", "userid"))); - cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("user_id_added_in_past", "int", "user_id_added_in_past"))); - cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("user_id_added_far_future", "int", - "user_id_added_far_future"))); - cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("user_id_deprecated", "int", "user_id_deprecated"))); - - cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("xsports", "array<string>", ""), - "xuser sports", "xusersports", "name", null, null, null)); - cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("ysports", "array<string>", ""), - "yuser sports", "yusersports", "name", null, null, null)); - cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("sports", "array<string>", ""), - "user sports", "usersports", "name", null, null, null)); - cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("sportids", "array<int>", ""), - "user sports", "userInterestIds", "sport_id", null, null, null)); - cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("statecountry", "string", ""), - "state country", "cubestatecountry", "name", null, null, null)); - cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("citycountry", "string", ""), - "city country", "cubecitystatecountry", "name", null, null, null)); - List<ChainRefCol> refCols = new ArrayList<>(); - refCols.add(new ChainRefCol("cubeState", "countrycapital")); - refCols.add(new ChainRefCol("cubeCityStateCountry", "capital")); - cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("cubeCountryCapital", "String", "ref dim"), - "Country capital", refCols, null, null, null, null)); - Map<String, String> cubeProperties = new HashMap<>(); - cubeProperties.put(MetastoreUtil.getCubeTimedDimensionListKey(BASE_CUBE_NAME), - "d_time,pt,it,et,test_time_dim,test_time_dim2"); - cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "test_time_dim", "ttd"); - cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "test_time_dim2", "ttd2"); - cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "d_time", "dt"); - cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "it", "it"); - cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "et", "et"); - cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "processing_time", "pt"); - cubeProperties.put(MetastoreConstants.TIMEDIM_RELATION + "d_time", "processing_time+[-5 days,5 days]"); - cubeProperties.put(MetastoreConstants.TIMEDIM_RELATION + "processing_time", "test_time_dim+[-5 days,5 days]"); - cubeProperties.put(MetastoreConstants.CUBE_ALL_FIELDS_QUERIABLE, "false"); - - Map<String, JoinChain> joinChainMap = new HashMap<>(); - addCubeChains(joinChainMap, "basecube"); - // update new paths - joinChainMap.get("dim2chain").addPath(new ArrayList<TableReference>() { - { - add(new TableReference("basecube", "dim12")); - add(new TableReference("testdim2", "id")); - } - }); - joinChainMap.get("dim3chain").addPath(new ArrayList<TableReference>() { - { - add(new TableReference("basecube", "dim12")); - add(new TableReference("testdim2", "id")); - add(new TableReference("testdim2", "testdim3id")); - add(new TableReference("testdim3", "id")); - } - }); - joinChainMap.get("dim4chain").addPath(new ArrayList<TableReference>() { - { - add(new TableReference("basecube", "dim12")); - add(new TableReference("testdim2", "id")); - add(new TableReference("testdim2", "testdim3id")); - add(new TableReference("testdim3", "id")); - add(new TableReference("testdim3", "testdim4id")); - add(new TableReference("testdim4", "id")); - } - }); - Set<JoinChain> joinChains = Sets.newHashSet(joinChainMap.values()); - joinChains.add(new JoinChain("cityState", "city-state", "state thru city") { - { - addPath(new ArrayList<TableReference>() { - { - add(new TableReference("basecube", "cityid")); - add(new TableReference("citydim", "id")); - add(new TableReference("citydim", "stateid")); - add(new TableReference("statedim", "id")); - } - }); - addPath(new ArrayList<TableReference>() { - { - add(new TableReference("basecube", "cityid")); - add(new TableReference("citydim", "id")); - add(new TableReference("citydim", "statename")); - add(new TableReference("statedim", "name")); - } - }); - } - }); - joinChains.add(new JoinChain("cityZip", "city-zip", "zip thru city") { - { - addPath(new ArrayList<TableReference>() { - { - add(new TableReference("basecube", "cityid")); - add(new TableReference("citydim", "id")); - add(new TableReference("citydim", "zipcode")); - add(new TableReference("zipdim", "code")); - } - }); - } - }); - joinChains.add(new JoinChain("cubeStateCountry", "cube-state-country", "country through state") { - { - addPath(new ArrayList<TableReference>() { - { - add(new TableReference("basecube", "stateid")); - add(new TableReference("statedim", "id")); - add(new TableReference("statedim", "countryid")); - add(new TableReference("countrydim", "id")); - } - }); - } - }); - joinChains.add(new JoinChain("cubeCityStateCountry", "cube-city-state-country", "country through state thru city") { - { - addPath(new ArrayList<TableReference>() { - { - add(new TableReference("basecube", "cityid")); - add(new TableReference("citydim", "id")); - add(new TableReference("citydim", "stateid")); - add(new TableReference("statedim", "id")); - add(new TableReference("statedim", "countryid")); - add(new TableReference("countrydim", "id")); - } - }); - } - }); - joinChains.add(new JoinChain("userchain", "user-chain", "user chain") { - { - addPath(new ArrayList<TableReference>() { - { - add(new TableReference("basecube", "userid")); - add(new TableReference("userdim", "id")); - } - }); - } - }); - joinChains.add(new JoinChain("user_id_added_far_future_chain", "user_id_added_far_future_chain", - "user_id_added_far_future_chain") { - { - addPath(new ArrayList<TableReference>() { - { - add(new TableReference("basecube", "user_id_added_far_future")); - add(new TableReference("userdim", "user_id_added_far_future")); - } - }); - } - }); - joinChains.add(new JoinChain("userSports", "user-sports", "user sports") { - { - addPath(new ArrayList<TableReference>() { - { - add(new TableReference("basecube", "userid")); - add(new TableReference("userdim", "id")); - add(new TableReference("userdim", "id")); - add(new TableReference("user_interests", "user_id", true)); - add(new TableReference("user_interests", "sport_id")); - add(new TableReference("sports", "id")); - } - }); - } - }); - joinChains.add(new JoinChain("userInterestIds", "user-interestsIds", "user interest ids") { - { - addPath(new ArrayList<TableReference>() { - { - add(new TableReference("basecube", "userid")); - add(new TableReference("userdim", "id")); - add(new TableReference("userdim", "id")); - add(new TableReference("user_interests", "user_id", true)); - } - }); - } - }); - joinChains.add(new JoinChain("xuserSports", "xuser-sports", "xuser sports") { - { - addPath(new ArrayList<TableReference>() { - { - add(new TableReference("basecube", "xuserid")); - add(new TableReference("userdim", "id")); - add(new TableReference("userdim", "id")); - add(new TableReference("user_interests", "user_id", true)); - add(new TableReference("user_interests", "sport_id")); - add(new TableReference("sports", "id")); - } - }); - } - }); - joinChains.add(new JoinChain("yuserSports", "user-sports", "user sports") { - { - addPath(new ArrayList<TableReference>() { - { - add(new TableReference("basecube", "yuserid")); - add(new TableReference("userdim", "id")); - add(new TableReference("userdim", "id")); - add(new TableReference("user_interests", "user_id", true)); - add(new TableReference("user_interests", "sport_id")); - add(new TableReference("sports", "id")); - } - }); - } - }); - - // add ref dim through chain - cubeDimensions2.add( - new ReferencedDimAttribute(new FieldSchema("cityStateCapital", "string", "State's capital thru city"), - "State's capital thru city", "cityState", "capital", null, null, null)); - Set<ExprColumn> baseExprs = new HashSet<>(exprs); - baseExprs.add(new ExprColumn(new FieldSchema("substrsprorts", "String", "substr of sports"), "substr sports", - "substr(sports, 10)")); - baseExprs.add(new ExprColumn(new FieldSchema("xsports_abbr", "array<string>", ""), - "xuser sports", "substr(xsports, 3)")); - baseExprs.add(new ExprColumn(new FieldSchema("ysports_abbr", "array<string>", ""), - "yuser sports", "substr(ysports, 3)")); - baseExprs.add(new ExprColumn(new FieldSchema("sports_abbr", "array<string>", ""), - "user sports", "substr(sports, 3)")); - baseExprs.add(new ExprColumn(new FieldSchema("sportids_abbr", "array<string>", ""), - "user sports", "case when sportids == 1 then 'CKT' when sportids == 2 then 'FTB' else 'NON' end")); - baseExprs.add(new ExprColumn(new FieldSchema("directMsrExpr", "bigint", ""), - "Direct Measure", new ExprSpec("directMsr + 0", null, null), new ExprSpec("msr13 + msr14", null, null))); - client.createCube(BASE_CUBE_NAME, cubeMeasures2, cubeDimensions2, baseExprs, joinChains, cubeProperties); - - Map<String, String> derivedProperties = new HashMap<>(); - derivedProperties.put(MetastoreConstants.CUBE_ALL_FIELDS_QUERIABLE, "true"); - Set<String> measures = new HashSet<>(); - measures.add("msr1"); - measures.add("msr9"); - measures.add("msr11"); - Set<String> dimensions = new HashSet<>(); - dimensions.add("dim1"); - dimensions.add("dim11"); - dimensions.add("d_time"); - client.createDerivedCube(BASE_CUBE_NAME, DERIVED_CUBE_NAME1, measures, dimensions, derivedProperties, 5L); - - measures = new HashSet<>(); - measures.add("msr2"); - measures.add("msr12"); - measures.add("msr13"); - measures.add("msr14"); - measures.add("directmsr"); - dimensions = new HashSet<>(); - dimensions.add("cityid"); - dimensions.add("stateid"); - dimensions.add("userid"); - dimensions.add("xuserid"); - dimensions.add("yuserid"); - dimensions.add("dim1"); - dimensions.add("dim2"); - dimensions.add("dim2big1"); - dimensions.add("dim2big2"); - dimensions.add("dim2bignew"); - dimensions.add("dim11"); - dimensions.add("dim13"); - dimensions.add("dim12"); - dimensions.add("dim22"); - dimensions.add("d_time"); - dimensions.add("test_time_dim"); - dimensions.add("test_time_dim2"); - dimensions.add("test_time_dim_hour_id"); - dimensions.add("test_time_dim_day_id"); - dimensions.add("test_time_dim_hour_id2"); - dimensions.add("test_time_dim_day_id2"); - client.createDerivedCube(BASE_CUBE_NAME, DERIVED_CUBE_NAME2, measures, dimensions, derivedProperties, 10L); - measures = new HashSet<>(); - measures.add("msr3"); - measures.add("msr13"); - dimensions = new HashSet<>(); - dimensions.add("dim1"); - dimensions.add("location"); - dimensions.add("d_time"); - dimensions.add("test_time_dim"); - dimensions.add("test_time_dim2"); - dimensions.add("test_time_dim_hour_id"); - dimensions.add("test_time_dim_day_id"); - dimensions.add("test_time_dim_hour_id2"); - dimensions.add("test_time_dim_day_id2"); - client.createDerivedCube(BASE_CUBE_NAME, DERIVED_CUBE_NAME3, measures, dimensions, derivedProperties, 20L); - - // create base cube facts - createBaseCubeFacts(client); - } - - private void createBaseCubeFacts(CubeMetastoreClient client) throws HiveException, LensException { - - Map<String, Set<UpdatePeriod>> storageAggregatePeriods = new HashMap<String, Set<UpdatePeriod>>(); - Set<UpdatePeriod> updates = new HashSet<UpdatePeriod>(); - updates.add(MINUTELY); - updates.add(HOURLY); - updates.add(DAILY); - updates.add(MONTHLY); - updates.add(QUARTERLY); - updates.add(YEARLY); - - ArrayList<FieldSchema> partCols = new ArrayList<FieldSchema>(); - List<String> timePartCols = new ArrayList<String>(); - partCols.add(TestCubeMetastoreClient.getDatePartition()); - timePartCols.add(TestCubeMetastoreClient.getDatePartitionKey()); - - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s1.setPartCols(partCols); - s1.setTimePartCols(timePartCols); - - StorageTableDesc s2 = new StorageTableDesc(); - s2.setInputFormat(TextInputFormat.class.getCanonicalName()); - s2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - ArrayList<FieldSchema> s2PartCols = new ArrayList<FieldSchema>(); - s2PartCols.add(new FieldSchema("ttd", serdeConstants.STRING_TYPE_NAME, "test date partition")); - s2PartCols.add(new FieldSchema("ttd2", serdeConstants.STRING_TYPE_NAME, "test date partition")); - s2.setPartCols(s2PartCols); - s2.setTimePartCols(Arrays.asList("ttd", "ttd2")); - - storageAggregatePeriods.put(c1, updates); - storageAggregatePeriods.put(c2, updates); - storageAggregatePeriods.put(c3, updates); - storageAggregatePeriods.put(c4, updates); - - Map<String, StorageTableDesc> storageTables = new HashMap<String, StorageTableDesc>(); - storageTables.put(c1, s1); - storageTables.put(c4, s2); - storageTables.put(c2, s1); - storageTables.put(c3, s1); - - String factName = "testFact1_BASE"; - List<FieldSchema> factColumns = new ArrayList<FieldSchema>(cubeMeasures.size()); - for (CubeMeasure measure : cubeMeasures) { - factColumns.add(measure.getColumn()); - } - - // add dimensions of the cube - factColumns.add(new FieldSchema("d_time", "timestamp", "event time")); - factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time")); - factColumns.add(new FieldSchema("zipcode", "int", "zip")); - factColumns.add(new FieldSchema("cityid", "int", "city id")); - factColumns.add(new FieldSchema("stateid", "int", "state id")); - factColumns.add(new FieldSchema("userid", "int", "user id")); - factColumns.add(new FieldSchema("xuserid", "int", "user id")); - factColumns.add(new FieldSchema("yuserid", "int", "user id")); - factColumns.add(new FieldSchema("dim1", "string", "base dim")); - factColumns.add(new FieldSchema("dim11", "string", "base dim")); - factColumns.add(new FieldSchema("test_time_dim_hour_id", "int", "time id")); - - // create cube fact with materialized expressions - client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 5L, - factValidityProperties, storageTables); - - factName = "testFact5_BASE"; - factColumns = new ArrayList<>(cubeMeasures.size()); - for (CubeMeasure measure : cubeMeasures) { - factColumns.add(measure.getColumn()); - } - - // add dimensions of the cube - factColumns.add(new FieldSchema("d_time", "timestamp", "event time")); - factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time")); - factColumns.add(new FieldSchema("dim1", "string", "base dim")); - factColumns.add(new FieldSchema("booleancut", "boolean", "expr dim")); - - // create cube fact - client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 150L, - factValidityProperties, storageTables); - - // create fact only with extra measures - factName = "testFact2_BASE"; - factColumns = new ArrayList<FieldSchema>(); - factColumns.add(new FieldSchema("msr12", "float", "second measure")); - - // add dimensions of the cube - factColumns.add(new FieldSchema("d_time", "timestamp", "event time")); - factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time")); - factColumns.add(new FieldSchema("dim1", "string", "base dim")); - factColumns.add(new FieldSchema("dim11", "string", "base dim")); - factColumns.add(new FieldSchema("dim2", "int", "dim2 id")); - factColumns.add(new FieldSchema("userid", "int", "user id")); - factColumns.add(new FieldSchema("xuserid", "int", "user id")); - factColumns.add(new FieldSchema("yuserid", "int", "user id")); - // create cube fact - client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 5L, - factValidityProperties, storageTables); - Map<String, String> properties = Maps.newHashMap(factValidityProperties); - properties.put(MetastoreConstants.FACT_ABSOLUTE_END_TIME, DateUtil.relativeToAbsolute("now.day - 2 days")); - properties.put(MetastoreConstants.FACT_ABSOLUTE_START_TIME, DateUtil.relativeToAbsolute("now.day - 3 days")); - client.createCubeFactTable(BASE_CUBE_NAME, "testfact_deprecated", factColumns, storageAggregatePeriods, 5L, - properties, storageTables); - - // create fact only with extra measures - factName = "testFact3_BASE"; - factColumns = new ArrayList<FieldSchema>(); - factColumns.add(new FieldSchema("msr13", "double", "third measure")); - factColumns.add(new FieldSchema("msr14", "bigint", "fourth measure")); - - // add dimensions of the cube - factColumns.add(new FieldSchema("d_time", "timestamp", "event time")); - factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time")); - factColumns.add(new FieldSchema("dim1", "string", "base dim")); - factColumns.add(new FieldSchema("dim11", "string", "base dim")); - - // create cube fact - client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 5L, - factValidityProperties, storageTables); - - // create fact with materialized expression - factName = "testFact6_BASE"; - factColumns = new ArrayList<>(); - factColumns.add(new FieldSchema("msr13", "double", "third measure")); - factColumns.add(new FieldSchema("msr14", "bigint", "fourth measure")); - - // add dimensions of the cube - factColumns.add(new FieldSchema("d_time", "timestamp", "event time")); - factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time")); - factColumns.add(new FieldSchema("dim1", "string", "base dim")); - factColumns.add(new FieldSchema("booleancut", "boolean", "expr dim")); - - // create cube fact - client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 150L, - factValidityProperties, storageTables); - - // create raw fact only with extra measures - factName = "testFact2_RAW_BASE"; - factColumns = new ArrayList<FieldSchema>(); - factColumns.add(new FieldSchema("msr11", "int", "first measure")); - factColumns.add(new FieldSchema("msr12", "float", "second measure")); - factColumns.add(new FieldSchema("msr9", "bigint", "ninth measure")); - - // add dimensions of the cube - factColumns.add(new FieldSchema("d_time", "timestamp", "event time")); - factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time")); - factColumns.add(new FieldSchema("dim1", "string", "base dim")); - factColumns.add(new FieldSchema("dim11", "string", "base dim")); - factColumns.add(new FieldSchema("dim13", "string", "base dim")); - factColumns.add(new FieldSchema("dim12", "string", "base dim")); - factColumns.add(new FieldSchema("dim22", "string", "base dim")); - factColumns.add(new FieldSchema("cityid", "int", "city id")); - - storageAggregatePeriods = new HashMap<String, Set<UpdatePeriod>>(); - updates = new HashSet<UpdatePeriod>(); - updates.add(HOURLY); - storageAggregatePeriods.put(c1, updates); - - storageTables = new HashMap<String, StorageTableDesc>(); - storageTables.put(c1, s1); - - // create cube fact - properties.clear(); - properties.putAll(factValidityProperties); - properties.put(MetastoreConstants.FACT_AGGREGATED_PROPERTY, "false"); - properties.put(MetastoreConstants.FACT_DATA_COMPLETENESS_TAG, "f2"); - - client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 100L, properties, - storageTables); - - // create raw fact only with extra measures - factName = "testFact3_RAW_BASE"; - factColumns = new ArrayList<FieldSchema>(); - factColumns.add(new FieldSchema("msr13", "double", "third measure")); - factColumns.add(new FieldSchema("msr14", "bigint", "fourth measure")); - - // add dimensions of the cube - factColumns.add(new FieldSchema("d_time", "timestamp", "event time")); - factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time")); - factColumns.add(new FieldSchema("dim1", "string", "base dim")); - factColumns.add(new FieldSchema("dim11", "string", "base dim")); - factColumns.add(new FieldSchema("dim12", "string", "base dim")); - - storageAggregatePeriods = new HashMap<String, Set<UpdatePeriod>>(); - updates = new HashSet<UpdatePeriod>(); - updates.add(HOURLY); - storageAggregatePeriods.put(c1, updates); - - storageTables = new HashMap<String, StorageTableDesc>(); - storageTables.put(c1, s1); - properties.put(MetastoreConstants.FACT_COL_START_TIME_PFX.concat("user_id_added_in_past"), "2016-01-01"); - properties.put(MetastoreConstants.FACT_COL_END_TIME_PFX.concat("user_id_deprecated"), "2016-01-01"); - properties.put(MetastoreConstants.FACT_COL_START_TIME_PFX.concat("user_id_added_far_future"), "2099-01-01"); - client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 100L, properties, - storageTables); - - factName = "testFact4_RAW_BASE"; - factColumns = new ArrayList<FieldSchema>(); - factColumns.add(new FieldSchema("msr13", "double", "third measure")); - factColumns.add(new FieldSchema("msr14", "bigint", "fourth measure")); - - // add dimensions of the cube - factColumns.add(new FieldSchema("d_time", "timestamp", "event time")); - factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time")); - factColumns.add(new FieldSchema("dim1", "string", "base dim")); - factColumns.add(new FieldSchema("user_id_added_in_past", "int", "user id")); - factColumns.add(new FieldSchema("user_id_added_far_future", "int", "user id")); - factColumns.add(new FieldSchema("user_id_deprecated", "int", "user id")); - - storageTables = new HashMap<String, StorageTableDesc>(); - storageTables.put(c1, s1); - properties.put(MetastoreConstants.FACT_COL_START_TIME_PFX.concat("user_id_added_in_past"), "2016-01-01"); - properties.put(MetastoreConstants.FACT_COL_END_TIME_PFX.concat("user_id_deprecated"), "2016-01-01"); - properties.put(MetastoreConstants.FACT_COL_START_TIME_PFX.concat("user_id_added_far_future"), "2099-01-01"); - client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 100L, properties, - storageTables); - - factName = "testFact5_RAW_BASE"; - factColumns = new ArrayList<FieldSchema>(); - factColumns.add(new FieldSchema("msr9", "bigint", "ninth measure")); - - // add dimensions of the cube - factColumns.add(new FieldSchema("d_time", "timestamp", "event time")); - factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time")); - factColumns.add(new FieldSchema("dim1", "string", "base dim")); - - properties.clear(); - properties.putAll(factValidityProperties); - properties.put(MetastoreConstants.FACT_AGGREGATED_PROPERTY, "false"); - properties.put(MetastoreConstants.FACT_DATA_COMPLETENESS_TAG, "f2"); - client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 100L, properties, - storageTables); - - CubeFactTable fact = client.getFactTable(factName); - // Add all hourly partitions for two days - Calendar cal = Calendar.getInstance(); - cal.setTime(TWODAYS_BACK); - Date temp = cal.getTime(); - while (!(temp.after(NOW))) { - Map<String, Date> timeParts = new HashMap<String, Date>(); - timeParts.put("dt", temp); - StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY); - client.addPartition(sPartSpec, c1, CubeTableType.FACT); - cal.add(HOUR_OF_DAY, 1); - temp = cal.getTime(); - } - } - - private void createCubeContinuousFact(CubeMetastoreClient client) throws Exception { - // create continuous raw fact only with extra measures - String factName = "testFact_CONTINUOUS"; - List<FieldSchema> factColumns = new ArrayList<FieldSchema>(); - factColumns.add(new FieldSchema("msr11", "double", "third measure")); - factColumns.add(new FieldSchema("msr15", "int", "fifteenth measure")); - - // add dimensions of the cube - factColumns.add(new FieldSchema("d_time", "timestamp", "event time")); - factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time")); - factColumns.add(new FieldSchema("dim1", "string", "base dim")); - factColumns.add(new FieldSchema("dim11", "string", "base dim")); - factColumns.add(new FieldSchema("dim12", "string", "base dim")); - - Map<String, Set<UpdatePeriod>> storageAggregatePeriods = new HashMap<String, Set<UpdatePeriod>>(); - Set<UpdatePeriod> updates = new HashSet<UpdatePeriod>(); - updates.add(CONTINUOUS); - storageAggregatePeriods.put(c0, updates); - - StorageTableDesc s0 = new StorageTableDesc(); - s0.setInputFormat(TextInputFormat.class.getCanonicalName()); - s0.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - - Map<String, StorageTableDesc> storageTables = new HashMap<String, StorageTableDesc>(); - storageTables.put(c0, s0); - Map<String, String> properties = Maps.newHashMap(factValidityProperties); - properties.put(MetastoreConstants.FACT_ABSOLUTE_START_TIME, DateUtil.relativeToAbsolute("now.day - 3 days")); - - client.createCubeFactTable(TEST_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 100L, properties, - storageTables); - } -- - private void createCubeFact(CubeMetastoreClient client) throws Exception { + private void assertTestFactTimelineClass(CubeMetastoreClient client) throws Exception { String factName = "testFact"; - List<FieldSchema> factColumns = new ArrayList<FieldSchema>(cubeMeasures.size()); - for (CubeMeasure measure : cubeMeasures) { - if (!measure.getColumn().getName().equals("msr15")) { //do not add msr15 - factColumns.add(measure.getColumn()); - } - } - factColumns.add(new FieldSchema("msr5", "double", "msr5")); - - // add dimensions of the cube - factColumns.add(new FieldSchema("zipcode", "int", "zip")); - factColumns.add(new FieldSchema("cityid", "int", "city id")); - factColumns.add(new FieldSchema("cityid1", "int", "city id")); - factColumns.add(new FieldSchema("stateid", "int", "city id")); - factColumns.add(new FieldSchema("test_time_dim_day_id", "int", "time id")); - factColumns.add(new FieldSchema("test_time_dim_day_id2", "int", "time id")); - factColumns.add(new FieldSchema("ambigdim1", "string", "used in" + " testColumnAmbiguity")); - - Map<String, Set<UpdatePeriod>> storageAggregatePeriods = new HashMap<String, Set<UpdatePeriod>>(); - Set<UpdatePeriod> updates = new HashSet<UpdatePeriod>(); - updates.add(MINUTELY); - updates.add(HOURLY); - updates.add(DAILY); - updates.add(MONTHLY); - updates.add(QUARTERLY); - updates.add(YEARLY); - - ArrayList<FieldSchema> partCols = new ArrayList<FieldSchema>(); - List<String> timePartCols = new ArrayList<String>(); - partCols.add(TestCubeMetastoreClient.getDatePartition()); - timePartCols.add(TestCubeMetastoreClient.getDatePartitionKey()); - - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s1.setPartCols(partCols); - s1.setTimePartCols(timePartCols); - - StorageTableDesc s2 = new StorageTableDesc(); - s2.setInputFormat(TextInputFormat.class.getCanonicalName()); - s2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - ArrayList<FieldSchema> s2PartCols = new ArrayList<FieldSchema>(); - s2PartCols.add(new FieldSchema("ttd", serdeConstants.STRING_TYPE_NAME, "test date partition")); - s2PartCols.add(new FieldSchema("ttd2", serdeConstants.STRING_TYPE_NAME, "test date partition")); - s2.setPartCols(s2PartCols); - s2.setTimePartCols(Arrays.asList("ttd", "ttd2")); - - StorageTableDesc s3 = new StorageTableDesc(); - s3.setInputFormat(TextInputFormat.class.getCanonicalName()); - s3.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s3.setPartCols(partCols); - s3.setTimePartCols(timePartCols); - s3.getTblProps().put(MetastoreUtil.getStoragetableStartTimesKey(), "now.day - 90 days"); - s3.getTblProps().put(MetastoreUtil.getStoragetableEndTimesKey(), "now.day - 10 days"); - - StorageTableDesc s5 = new StorageTableDesc(); - s5.setInputFormat(TextInputFormat.class.getCanonicalName()); - s5.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s5.setPartCols(partCols); - s5.setTimePartCols(timePartCols); - s5.getTblProps().put(MetastoreUtil.getStoragetableStartTimesKey(), "now.day - 10 days"); - - storageAggregatePeriods.put(c1, updates); - storageAggregatePeriods.put(c2, updates); - storageAggregatePeriods.put(c3, updates); - storageAggregatePeriods.put(c4, updates); - storageAggregatePeriods.put(c5, updates); - - Map<String, StorageTableDesc> storageTables = new HashMap<String, StorageTableDesc>(); - storageTables.put(c1, s1); - storageTables.put(c4, s2); - storageTables.put(c2, s1); - storageTables.put(c3, s3); - storageTables.put(c5, s5); - - //add storage with continuous update period - updates.add(CONTINUOUS); - storageAggregatePeriods.put(c0, updates); - StorageTableDesc s0 = new StorageTableDesc(); - s0.setInputFormat(TextInputFormat.class.getCanonicalName()); - s0.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - storageTables.put(c0, s0); - - // create cube fact - client.createCubeFactTable(TEST_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 5L, - factValidityProperties, storageTables); + client.getTimelines(factName, c1, null, null); client.getTimelines(factName, c4, null, null); + client.clearHiveTableCache(); + CubeFactTable fact = client.getFactTable(factName); Table table = client.getTable(MetastoreUtil.getStorageTableName(fact.getName(), Storage.getPrefix(c1))); assertEquals(table.getParameters().get(MetastoreUtil.getPartitionTimelineCachePresenceKey()), "true"); http://git-wip-us.apache.org/repos/asf/lens/blob/8868b063/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java ---------------------------------------------------------------------- diff --cc lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java index abaae5b,24660e1..194b380 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java @@@ -226,10 -232,24 +226,8 @@@ public class CubeMetastoreServiceImpl e */ @Override public void createDimensionTable(LensSessionHandle sessionid, XDimensionTable xDimTable) throws LensException { -- String dimTblName = xDimTable.getTableName(); - List<FieldSchema> columns = JAXBUtils.fieldSchemaListFromColumns(xDimTable.getColumns()); - Map<String, UpdatePeriod> updatePeriodMap = - JAXBUtils.dumpPeriodsFromStorageTables(xDimTable.getStorageTables()); - - Map<String, String> properties = JAXBUtils.mapFromXProperties(xDimTable.getProperties()); - Map<String, StorageTableDesc> storageDesc = JAXBUtils.tableDescPrefixMapFromXStorageTables( - xDimTable.getStorageTables()); -- try (SessionContext ignored = new SessionContext(sessionid)){ - log.info("# Columns: " + columns); - getClient(sessionid).createCubeDimensionTable(xDimTable.getDimensionName(), - dimTblName, - columns, - xDimTable.getWeight(), - updatePeriodMap, - properties, - storageDesc); + getClient(sessionid).createCubeDimensionTable(xDimTable); log.info("Dimension Table created " + xDimTable.getTableName()); } }
